path: root/libbuild2/file.cxx

// file      : libbuild2/file.cxx -*- C++ -*-
// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
// license   : MIT; see accompanying LICENSE file

#include <libbuild2/file.hxx>

#include <iostream> // cin

#include <libbuild2/scope.hxx>
#include <libbuild2/target.hxx>
#include <libbuild2/context.hxx>
#include <libbuild2/filesystem.hxx>   // exists()
#include <libbuild2/prerequisite.hxx>
#include <libbuild2/diagnostics.hxx>

#include <libbuild2/token.hxx>
#include <libbuild2/lexer.hxx>
#include <libbuild2/parser.hxx>

using namespace std;
using namespace butl;

namespace build2
{
  // Standard and alternative build file/directory naming schemes.
  //
  const dir_path std_build_dir     ("build");
  const dir_path std_root_dir      (dir_path (std_build_dir) /= "root");
  const dir_path std_bootstrap_dir (dir_path (std_build_dir) /= "bootstrap");

  const path std_root_file      (std_build_dir     / "root.build");
  const path std_bootstrap_file (std_build_dir     / "bootstrap.build");
  const path std_src_root_file  (std_bootstrap_dir / "src-root.build");
  const path std_out_root_file  (std_bootstrap_dir / "out-root.build");
  const path std_export_file    (std_build_dir     / "export.build");

  const string std_build_ext        ("build");
  const path   std_buildfile_file   ("buildfile");
  const path   std_buildignore_file (".buildignore");

  //

  const dir_path alt_build_dir     ("build2");
  const dir_path alt_root_dir      (dir_path (alt_build_dir) /= "root");
  const dir_path alt_bootstrap_dir (dir_path (alt_build_dir) /= "bootstrap");

  const path alt_root_file      (alt_build_dir     / "root.build2");
  const path alt_bootstrap_file (alt_build_dir     / "bootstrap.build2");
  const path alt_src_root_file  (alt_bootstrap_dir / "src-root.build2");
  const path alt_out_root_file  (alt_bootstrap_dir / "out-root.build2");
  const path alt_export_file    (alt_build_dir     / "export.build2");

  const string alt_build_ext        ("build2");
  const path   alt_buildfile_file   ("build2file");
  const path   alt_buildignore_file (".build2ignore");

  ostream&
  operator<< (ostream& os, const subprojects& sps)
  {
    for (auto b (sps.begin ()), i (b); os && i != sps.end (); ++i)
    {
      // See find_subprojects() for details.
      //
      const project_name& n (
        path::traits_type::is_separator (i->first.string ().back ())
        ? empty_project_name
        : i->first);

      os << (i != b ? " " : "") << n << '@' << i->second;
    }

    return os;
  }

  // Check if the standard/alternative file/directory exists, returning empty
  // path if it does not.
  //
  template <typename T>
  static T
  exists (const dir_path& d, const T& s, const T& a, optional<bool>& altn)
  {
    T p;
    bool e;

    if (altn)
    {
      p = d / (*altn ? a : s);
      e = exists (p);
    }
    else
    {
      // Check the alternative name first since it is more specific.
      //
      p = d / a;

      if ((e = exists (p)))
        altn = true;
      else
      {
        p = d / s;

        if ((e = exists (p)))
          altn = false;
      }
    }

    return e ? p : T ();
  }

  bool
  is_src_root (const dir_path& d, optional<bool>& altn)
  {
    // We can't have root without bootstrap.build.
    //
    return !exists (d, std_bootstrap_file, alt_bootstrap_file, altn).empty ();
  }

  bool
  is_out_root (const dir_path& d, optional<bool>& altn)
  {
    return !exists (d, std_src_root_file, alt_src_root_file, altn).empty ();
  }

  dir_path
  find_src_root (const dir_path& b, optional<bool>& altn)
  {
    for (dir_path d (b); !d.root () && d != home; d = d.directory ())
    {
      if (is_src_root (d, altn))
        return d;
    }

    return dir_path ();
  }

  pair<dir_path, bool>
  find_out_root (const dir_path& b, optional<bool>& altn)
  {
    for (dir_path d (b); !d.root () && d != home; d = d.directory ())
    {
      bool s;
      if ((s = is_src_root (d, altn)) || is_out_root (d, altn))
        return make_pair (move (d), s);
    }

    return make_pair (dir_path (), false);
  }

  // Remap the src_root variable value if it is inside old_src_root.
  //
  static inline void
  remap_src_root (context& ctx, value& v)
  {
    if (!ctx.old_src_root.empty ())
    {
      dir_path& d (cast<dir_path> (v));

      if (d.sub (ctx.old_src_root))
        d = ctx.new_src_root / d.leaf (ctx.old_src_root);
    }
  }

  static void
  source (scope& root, scope& base, const path& bf, bool boot)
  {
    tracer trace ("source");

    try
    {
      bool sin (bf.string () == "-");

      ifdstream ifs;

      if (!sin)
        ifs.open (bf);
      else
        cin.exceptions (ifdstream::failbit | ifdstream::badbit);

      istream& is (sin ? cin : ifs);

      l5 ([&]{trace << "sourcing " << bf;});

      parser p (root.ctx, boot);
      p.parse_buildfile (is, bf, root, base);
    }
    catch (const io_error& e)
    {
      fail << "unable to read buildfile " << bf << ": " << e;
    }
  }

  void
  source (scope& root, scope& base, const path& bf)
  {
    source (root, base, bf, false);
  }

  bool
  source_once (scope& root, scope& base, const path& bf, scope& once)
  {
    tracer trace ("source_once");

    if (!once.buildfiles.insert (bf).second)
    {
      l5 ([&]{trace << "skipping already sourced " << bf;});
      return false;
    }

    source (root, base, bf);
    return true;
  }

  // Source (once) pre-*.build (pre is true) or post-*.build (otherwise) hooks
  // from the specified directory (build/{bootstrap,root}/ of out_root) which
  // must exist.
  //
  static void
  source_hooks (scope& root, const dir_path& d, bool pre)
  {
    // While we could have used the wildcard pattern matching functionality,
    // our needs are pretty basic and performance is quite important, so let's
    // handle this ourselves.
    //
    try
    {
      for (const dir_entry& de: dir_iterator (d, false /* ignore_dangling */))
      {
        // If this is a link, then type() will try to stat() it. And if the
        // link is dangling or points to something inaccessible, it will fail.
        // So let's first check that the name matches and only then check the
        // type.
        //
        const path& n (de.path ());

        if (n.string ().compare (0,
                                 pre ? 4 : 5,
                                 pre ? "pre-" : "post-") != 0 ||
            n.extension () != root.root_extra->build_ext)
          continue;

        path f (d / n);

        try
        {
          if (de.type () != entry_type::regular)
            continue;
        }
        catch (const system_error& e)
        {
          fail << "unable to read buildfile " << f << ": " << e;
        }

        source_once (root, root, f);
      }
    }
    catch (const system_error& e)
    {
      fail << "unable to iterate over " << d << ": " << e;
    }
  }

  scope_map::iterator
  create_root (scope& s, const dir_path& out_root, const dir_path& src_root)
  {
    auto i (s.ctx.scopes.rw (s).insert (out_root, true /* root */));
    scope& rs (i->second);

    context& ctx (rs.ctx);

    // Set out_path. Note that src_path is set in setup_root() below.
    //
    if (rs.out_path_ != &i->first)
    {
      assert (rs.out_path_ == nullptr);
      rs.out_path_ = &i->first;
    }

    // If this is already a root scope, verify that things are consistent.
    //
    {
      value& v (rs.assign (ctx.var_out_root));

      if (!v)
        v = out_root;
      else
      {
        const dir_path& p (cast<dir_path> (v));

        if (p != out_root)
          fail << "new out_root " << out_root << " does not match "
               << "existing " << p;
      }
    }

    if (!src_root.empty ())
    {
      value& v (rs.assign (ctx.var_src_root));

      if (!v)
        v = src_root;
      else
      {
        const dir_path& p (cast<dir_path> (v));

        if (p != src_root)
          fail << "new src_root " << src_root << " does not match "
               << "existing " << p;
      }
    }

    return i;
  }

  void
  setup_root (scope& s, bool forwarded)
  {
    context& ctx (s.ctx);

    // The caller must have made sure src_root is set on this scope.
    //
    value& v (s.assign (ctx.var_src_root));
    assert (v);
    const dir_path& d (cast<dir_path> (v));

    if (s.src_path_ == nullptr)
      s.src_path_ = &d;
    else
      assert (s.src_path_ == &d);

    s.assign (ctx.var_forwarded) = forwarded;
  }

  scope&
  setup_base (scope_map::iterator i,
              const dir_path& out_base,
              const dir_path& src_base)
  {
    scope& s (i->second);
    context& ctx (s.ctx);

    // Set src/out_base variables.
    //
    value& ov (s.assign (ctx.var_out_base));

    if (!ov)
      ov = out_base;
    else
      assert (cast<dir_path> (ov) == out_base);

    value& sv (s.assign (ctx.var_src_base));

    if (!sv)
      sv = src_base;
    else
      assert (cast<dir_path> (sv) == src_base);

    // Set src/out_path. The key (i->first) is out_base.
    //
    if (s.out_path_ == nullptr)
      s.out_path_ = &i->first;
    else
      assert (*s.out_path_ == out_base);

    if (s.src_path_ == nullptr)
      s.src_path_ = &cast<dir_path> (sv);
    else
      assert (*s.src_path_ == src_base);

    return s;
  }

  pair<scope&, scope*>
  switch_scope (scope& root, const dir_path& p)
  {
    // First, enter the scope into the map and see if it is in any project. If
    // it is not, then there is nothing else to do.
    //
    auto i (root.ctx.scopes.rw (root).insert (p));
    scope& base (i->second);
    scope* rs (base.root_scope ());

    if (rs != nullptr)
    {
      // Path p can be src_base or out_base. Figure out which one it is.
      //
      dir_path out_base (p.sub (rs->out_path ()) ? p : src_out (p, *rs));

      // Create and bootstrap root scope(s) of subproject(s) that this scope
      // may belong to. If any were created, load them. Note that we need to
      // do this before figuring out src_base since we may switch the root
      // project (and src_root with it).
      //
      {
        scope* nrs (&create_bootstrap_inner (*rs, out_base));

        if (rs != nrs)
          rs = nrs;
      }

      // Switch to the new root scope.
      //
      if (rs != &root)
        load_root (*rs); // Load new root(s) recursively.

      // Now we can figure out src_base and finish setting the scope.
      //
      dir_path src_base (src_out (out_base, *rs));
      setup_base (i, move (out_base), move (src_base));
    }

    return pair<scope&, scope*> (base, rs);
  }

  dir_path
  bootstrap_fwd (context& ctx, const dir_path& src_root, optional<bool>& altn)
  {
    path f (exists (src_root, std_out_root_file, alt_out_root_file, altn));

    if (f.empty ())
      return src_root;

    // We cannot just source the buildfile since there is no scope to do
    // this on yet.
    //
    auto p (extract_variable (ctx, f, *ctx.var_out_root));

    if (!p.second)
      fail << "variable out_root expected as first line in " << f;

    try
    {
      return convert<dir_path> (move (p.first));
    }
    catch (const invalid_argument& e)
    {
      fail << "invalid out_root value in " << f << ": " << e << endf;
    }
  }

  static void
  setup_root_extra (scope& root, optional<bool>& altn)
  {
    assert (altn && root.root_extra == nullptr);
    bool a (*altn);

    root.root_extra.reset (
      new scope::root_extra_type {
        a,
        a ? alt_build_ext        : std_build_ext,
        a ? alt_build_dir        : std_build_dir,
        a ? alt_buildfile_file   : std_buildfile_file,
        a ? alt_buildignore_file : std_buildignore_file,
        a ? alt_root_dir         : std_root_dir,
        a ? alt_bootstrap_dir    : std_bootstrap_dir,
        a ? alt_bootstrap_file   : std_bootstrap_file,
        a ? alt_root_file        : std_root_file,
        a ? alt_export_file      : std_export_file,
        a ? alt_src_root_file    : std_src_root_file,
        a ? alt_out_root_file    : std_out_root_file,
        {}, /* meta_operations */
        {}, /* operations */
        {}, /* modules */
        {}, /* override_cache */
        {}} /* target_types */);

    // Enter built-in meta-operation and operation names. Loading of
    // modules (via the src bootstrap; see below) can result in
    // additional meta/operations being added.
    //
    root.insert_meta_operation (noop_id,    mo_noop);
    root.insert_meta_operation (perform_id, mo_perform);
    root.insert_meta_operation (info_id,    mo_info);

    root.insert_operation (default_id, op_default);
    root.insert_operation (update_id,  op_update);
    root.insert_operation (clean_id,   op_clean);
  }

  void
  bootstrap_out (scope& root, optional<bool>& altn)
  {
    const dir_path& out_root (root.out_path ());

    path f (exists (out_root, std_src_root_file, alt_src_root_file, altn));

    if (f.empty ())
      return;

    if (root.root_extra == nullptr)
      setup_root_extra (root, altn);

    //@@ TODO: if bootstrap files can source other bootstrap files (for
    //   example, as a way to express dependecies), then we need a way to
    //   prevent multiple sourcing. We handle it here but we still need
    //   something like source_once (once [scope] source) in buildfiles.
    //
    source_once (root, root, f);
  }

  pair<value, bool>
  extract_variable (context& ctx, const path& bf, const variable& var)
  {
    try
    {
      ifdstream ifs (bf);

      lexer lex (ifs, bf);
      token t (lex.next ());
      token_type tt;

      if (t.type != token_type::word || t.value != var.name ||
          ((tt = lex.next ().type) != token_type::assign &&
           tt != token_type::prepend &&
           tt != token_type::append))
      {
        return make_pair (value (), false);
      }

      parser p (ctx);
      temp_scope tmp (ctx.global_scope.rw ());
      p.parse_variable (lex, tmp, var, tt);

      value* v (tmp.vars.find_to_modify (var).first);
      assert (v != nullptr);

      // Steal the value, the scope is going away.
      //
      return make_pair (move (*v), true);
    }
    catch (const io_error& e)
    {
      fail << "unable to read buildfile " << bf << ": " << e << endf;
    }
  }

  // Extract the project name from bootstrap.build.
  //
  static project_name
  find_project_name (context& ctx,
                     const dir_path& out_root,
                     const dir_path& fallback_src_root,
                     optional<bool> out_src, // True if out_root is src_root.
                     optional<bool>& altn)
  {
    tracer trace ("find_project_name");

    // First check if the root scope for this project has already been setup
    // in which case we will have src_root and maybe even the name.
    //
    const dir_path* src_root (nullptr);
    const scope& s (ctx.scopes.find (out_root));

    if (s.root_scope () == &s && s.out_path () == out_root)
    {
      if (s.root_extra != nullptr)
      {
        if (!altn)
          altn = s.root_extra->altn;
        else
          assert (*altn == s.root_extra->altn);
      }

      if (lookup l = s.vars[ctx.var_project])
        return cast<project_name> (l);

      src_root = s.src_path_;
    }

    // Load the project name. If this subdirectory is the subproject's
    // src_root, then we can get directly to that. Otherwise, we first have to
    // discover its src_root.
    //
    value src_root_v; // Need it to live until the end.

    if (src_root == nullptr)
    {
      if (out_src ? *out_src : is_src_root (out_root, altn))
        src_root = &out_root;
      else
      {
        path f (exists (out_root, std_src_root_file, alt_src_root_file, altn));

        if (f.empty ())
        {
          // Note: the same diagnostics as in main().
          //
          if (fallback_src_root.empty ())
            fail << "no bootstrapped src_root for " << out_root <<
              info << "consider reconfiguring this out_root";

          src_root = &fallback_src_root;
        }
        else
        {
          auto p (extract_variable (ctx, f, *ctx.var_src_root));

          if (!p.second)
            fail << "variable src_root expected as first line in " << f;

          src_root_v = move (p.first);
          remap_src_root (ctx, src_root_v); // Remap if inside old_src_root.
          src_root = &cast<dir_path> (src_root_v);

          l5 ([&]{trace << "extracted src_root " << *src_root
                        << " for " << out_root;});
        }
      }
    }

    project_name name;
    {
      path f (exists (*src_root, std_bootstrap_file, alt_bootstrap_file, altn));

      if (f.empty ())
        fail << "no build/bootstrap.build in " << *src_root;

      auto p (extract_variable (ctx, f, *ctx.var_project));

      if (!p.second)
        fail << "variable " << ctx.var_project->name << " expected "
             << "as a first line in " << f;

      name = cast<project_name> (move (p.first));
    }

    l5 ([&]{trace << "extracted project name '" << name << "' for "
                  << *src_root;});
    return name;
  }

  // Scan the specified directory for any subprojects. If a subdirectory
  // is a subproject, then enter it into the map, handling the duplicates.
  //
  static void
  find_subprojects (context& ctx,
                    subprojects& sps,
                    const dir_path& d,
                    const dir_path& root,
                    bool out)
  {
    tracer trace ("find_subprojects");

    try
    {
      for (const dir_entry& de: dir_iterator (d, true /* ignore_dangling */))
      {
        if (de.type () != entry_type::directory)
          continue;

        dir_path sd (d / path_cast<dir_path> (de.path ()));

        bool src (false);
        optional<bool> altn;

        if (!((out && is_out_root (sd, altn)) ||
              (src =  is_src_root (sd, altn))))
        {
          // We used to scan for subproject recursively but this is probably
          // too loose (think of some tests laying around). In the future we
          // should probably allow specifying something like extra/* or
          // extra/** in subprojects.
          //
          //find_subprojects (sps, sd, root, out);
          //
          continue;
        }

        // Calculate relative subdirectory for this subproject.
        //
        dir_path dir (sd.leaf (root));
        l5 ([&]{trace << "subproject " << sd << " as " << dir;});

        // Load its name. Note that here we don't use fallback src_root
        // since this function is used to scan both out_root and src_root.
        //
        project_name name (
          find_project_name (ctx, sd, dir_path (), src, altn));

        // If the name is empty, then is is an unnamed project. While the
        // 'project' variable stays empty, here we come up with a surrogate
        // name for a key. The idea is that such a key should never conflict
        // with a real project name. We ensure this by using the project's
        // sub-directory and appending a trailing directory separator to it.
        //
        if (name.empty ())
          name = project_name (dir.posix_string () + '/',
                               project_name::raw_string);

        // @@ Can't use move() because we may need the values in diagnostics
        // below. Looks like C++17 try_emplace() is what we need.
        //
        auto rp (sps.emplace (name, dir));

        // Handle duplicates.
        //
        if (!rp.second)
        {
          const dir_path& dir1 (rp.first->second);

          if (dir != dir1)
            fail << "inconsistent subproject directories for " << name <<
              info << "first alternative: " << dir1 <<
              info << "second alternative: " << dir;

          l6 ([&]{trace << "skipping duplicate";});
        }
      }
    }
    catch (const system_error& e)
    {
      fail << "unable to iterate over " << d << ": " << e;
    }
  }

  bool
  bootstrap_src (scope& rs, optional<bool>& altn)
  {
    tracer trace ("bootstrap_src");

    context& ctx (rs.ctx);

    bool r (false);

    const dir_path& out_root (rs.out_path ());
    const dir_path& src_root (rs.src_path ());

    {
      path f (exists (src_root, std_bootstrap_file, alt_bootstrap_file, altn));

      if (rs.root_extra == nullptr)
      {
        // If nothing so far has indicated the naming, assume standard.
        //
        if (!altn)
          altn = false;

        setup_root_extra (rs, altn);
      }

      if (!f.empty ())
      {
        // We assume that bootstrap out cannot load this file explicitly. It
        // feels wrong to allow this since that makes the whole bootstrap
        // process hard to reason about. But we may try to bootstrap the same
        // root scope multiple time.
        //
        if (rs.buildfiles.insert (f).second)
          source (rs, rs, f, true);
        else
          l5 ([&]{trace << "skipping already sourced " << f;});

        r = true;
      }
    }

    // See if we are a part of an amalgamation. There are two key players: the
    // outer root scope which may already be present (i.e., we were loaded as
    // part of an amalgamation) and the amalgamation variable that may or may
    // not be set by the user (in bootstrap.build) or by an earlier call to
    // this function for the same scope. When set by the user, the empty
    // special value means that the project shall not be amalgamated (and
    // which we convert to NULL below). When calculated, the NULL value
    // indicates that we are not amalgamated.
    //
    // Note: the amalgamation variable value is always a relative directory.
    //
    {
      auto rp (rs.vars.insert (*ctx.var_amalgamation)); // Set NULL by default.
      value& v (rp.first);

      if (v && v.empty ()) // Convert empty to NULL.
        v = nullptr;

      if (scope* ars = rs.parent_scope ()->root_scope ())
      {
        const dir_path& ad (ars->out_path ());
        dir_path rd (ad.relative (out_root));

        // If we already have the amalgamation variable set, verify
        // that aroot matches its value.
        //
        if (!rp.second)
        {
          if (!v)
          {
            fail << out_root << " cannot be amalgamated" <<
              info << "amalgamated by " << ad;
          }
          else
          {
            const dir_path& vd (cast<dir_path> (v));

            if (vd != rd)
            {
              fail << "inconsistent amalgamation of " << out_root <<
                info << "specified: " << vd <<
                info << "actual: " << rd << " by " << ad;
            }
          }
        }
        else
        {
          // Otherwise, use the outer root as our amalgamation.
          //
          l5 ([&]{trace << out_root << " amalgamated as " << rd;});
          v = move (rd);
        }
      }
      else if (rp.second)
      {
        // If there is no outer root and the amalgamation variable
        // hasn't been set, then we need to check if any of the
        // outer directories is a project's out_root. If so, then
        // that's our amalgamation.
        //
        optional<bool> altn;
        const dir_path& ad (find_out_root (out_root.directory (), altn).first);

        if (!ad.empty ())
        {
          dir_path rd (ad.relative (out_root));
          l5 ([&]{trace << out_root << " amalgamated as " << rd;});
          v = move (rd);
        }
      }
    }

    // See if we have any subprojects. In a sense, this is the other
    // side/direction of the amalgamation logic above. Here, the subprojects
    // variable may or may not be set by the user (in bootstrap.build) or by
    // an earlier call to this function for the same scope. When set by the
    // user, the empty special value means that there are no subproject and
    // none should be searched for (and which we convert to NULL below).
    // Otherwise, it is a list of [project@]directory pairs. The directory
    // must be relative to our out_root. If the project name is not specified,
    // then we have to figure it out. When subprojects are calculated, the
    // NULL value indicates that we found no subprojects.
    //
    {
      auto rp (rs.vars.insert (*ctx.var_subprojects)); // Set NULL by default.
      value& v (rp.first);

      if (rp.second)
      {
        // No subprojects set so we need to figure out if there are any.
        //
        // First we are going to scan our out_root and find all the
        // pre-configured subprojects. Then, if out_root != src_root,
        // we are going to do the same for src_root. Here, however,
        // we need to watch out for duplicates.
        //
        subprojects sps;

        if (exists (out_root))
        {
          l5 ([&]{trace << "looking for subprojects in " << out_root;});
          find_subprojects (rs.ctx, sps, out_root, out_root, true);
        }

        if (out_root != src_root)
        {
          l5 ([&]{trace << "looking for subprojects in " << src_root;});
          find_subprojects (rs.ctx, sps, src_root, src_root, false);
        }

        if (!sps.empty ()) // Keep it NULL if no subprojects.
          v = move (sps);
      }
      else if (v)
      {
        // Convert empty to NULL.
        //
        if (v.empty ())
          v = nullptr;
        else
        {
          // Scan the (untyped) value and convert it to the "canonical" form,
          // that is, a list of name@dir pairs.
          //
          subprojects sps;
          names& ns (cast<names> (v));

          for (auto i (ns.begin ()); i != ns.end (); ++i)
          {
            // Project name.
            //
            project_name n;
            if (i->pair)
            {
              if (i->pair != '@')
                fail << "unexpected pair style in variable subprojects";

              try
              {
                n = convert<project_name> (move (*i));

                if (n.empty ())
                  fail << "empty project name in variable subprojects";
              }
              catch (const invalid_argument&)
              {
                fail << "expected project name instead of '" << *i << "' in "
                     << "variable subprojects";
              }

              ++i; // Got to have the second half of the pair.
            }

            // Directory.
            //
            dir_path d;
            try
            {
              d = convert<dir_path> (move (*i));

              if (d.empty ())
                fail << "empty directory in variable subprojects";
            }
            catch (const invalid_argument&)
            {
              fail << "expected directory instead of '" << *i << "' in "
                   << "variable subprojects";
            }

            // Figure out the project name if the user didn't specify one.
            //
            if (n.empty ())
            {
              optional<bool> altn;

              // Pass fallback src_root since this is a subproject that was
              // specified by the user so it is most likely in our src.
              //
              n = find_project_name (rs.ctx,
                                     out_root / d,
                                     src_root / d,
                                     nullopt /* out_src */,
                                     altn);

              // See find_subprojects() for details on unnamed projects.
              //
              if (n.empty ())
                n = project_name (d.posix_string () + '/',
                                  project_name::raw_string);
            }

            sps.emplace (move (n), move (d));
          }

          // Change the value to the typed map.
          //
          v = move (sps);
        }
      }
    }

    return r;
  }

  void
  bootstrap_pre (scope& root, optional<bool>& altn)
  {
    const dir_path& out_root (root.out_path ());

    // This test is a bit loose in a sense that there can be a stray
    // build/bootstrap/ directory that will make us mis-treat a project as
    // following the standard naming scheme (the other way, while also
    // possible, is a lot less likely). If this does becomes a problem, we can
    // always tighten the test by also looking for a hook file with the
    // correct extension.
    //
    dir_path d (exists (out_root, std_bootstrap_dir, alt_bootstrap_dir, altn));

    if (!d.empty ())
    {
      if (root.root_extra == nullptr)
        setup_root_extra (root, altn);

      source_hooks (root, d, true /* pre */);
    }
  }

  void
  bootstrap_post (scope& root)
  {
    const dir_path& out_root (root.out_path ());

    dir_path d (out_root / root.root_extra->bootstrap_dir);

    if (exists (d))
      source_hooks (root, d, false /* pre */);
  }

  bool
  bootstrapped (scope& rs)
  {
    // Use the subprojects variable set by bootstrap_src() as an indicator.
    // It should either be NULL or typed (so we assume that the user will
    // never set it to NULL).
    //
    auto l (rs.vars[rs.ctx.var_subprojects]);
    return l.defined () && (l->null || l->type != nullptr);
  }

  // Return true if the inner/outer project (identified by out/src_root) of
  // the 'origin' project (identified by orig) should be forwarded.
  //
  static inline bool
  forwarded (const scope& orig,
             const dir_path& out_root,
             const dir_path& src_root,
             optional<bool>& altn)
  {
    context& ctx (orig.ctx);

    // The conditions are:
    //
    // 1. Origin is itself forwarded.
    //
    // 2. Inner/outer src_root != out_root.
    //
    // 3. Inner/outer out-root.build exists in src_root and refers out_root.
    //
    return (out_root != src_root                            &&
            cast_false<bool> (orig.vars[ctx.var_forwarded]) &&
            bootstrap_fwd (ctx, src_root, altn) == out_root);
  }

  void
  create_bootstrap_outer (scope& root)
  {
    context& ctx (root.ctx);

    auto l (root.vars[ctx.var_amalgamation]);

    if (!l)
      return;

    const dir_path& d (cast<dir_path> (l));
    dir_path out_root (root.out_path () / d);
    out_root.normalize (); // No need to actualize (d is a bunch of ..)

    // src_root is a bit more complicated. Here we have three cases:
    //
    // 1. Amalgamation's src_root is "parallel" to the sub-project's.
    // 2. Amalgamation's src_root is the same as its out_root.
    // 3. Some other pre-configured (via src-root.build) src_root.
    //
    // So we need to try all these cases in some sensible order. #3 should
    // probably be tried first since that src_root was explicitly configured
    // by the user. After that, #2 followed by #1 seems reasonable.
    //
    scope& rs (create_root (root, out_root, dir_path ())->second);

    bool bstrapped (bootstrapped (rs));

    optional<bool> altn;
    if (!bstrapped)
    {
      bootstrap_out (rs, altn); // #3 happens here (or it can be #1).

      value& v (rs.assign (ctx.var_src_root));

      if (!v)
      {
        if (is_src_root (out_root, altn)) // #2
          v = out_root;
        else // #1
        {
          dir_path src_root (root.src_path () / d);
          src_root.normalize (); // No need to actualize (as above).
          v = move (src_root);
        }
      }
      else
        remap_src_root (ctx, v); // Remap if inside old_src_root.

      setup_root (rs, forwarded (root, out_root, v.as<dir_path> (), altn));
      bootstrap_pre (rs, altn);
      bootstrap_src (rs, altn);
      // bootstrap_post() delayed until after create_bootstrap_outer().
    }
    else
    {
      altn = rs.root_extra->altn;

      if (forwarded (root, rs.out_path (), rs.src_path (), altn))
        rs.assign (ctx.var_forwarded) = true; // Only upgrade (see main()).
    }

    create_bootstrap_outer (rs);

    if (!bstrapped)
      bootstrap_post (rs);

    // Check if we are strongly amalgamated by this outer root scope.
    //
    if (root.src_path ().sub (rs.src_path ()))
      root.strong_ = rs.strong_scope (); // Itself or some outer scope.
  }

  scope&
  create_bootstrap_inner (scope& root, const dir_path& out_base)
  {
    context& ctx (root.ctx);

    scope* r (&root);

    if (auto l = root.vars[ctx.var_subprojects])
    {
      for (const auto& p: cast<subprojects> (l))
      {
        dir_path out_root (root.out_path () / p.second);

        if (!out_base.empty () && !out_base.sub (out_root))
          continue;

        // The same logic to src_root as in create_bootstrap_outer().
        //
        scope& rs (create_root (root, out_root, dir_path ())->second);

        optional<bool> altn;
        if (!bootstrapped (rs))
        {
          bootstrap_out (rs, altn);

          value& v (rs.assign (ctx.var_src_root));

          if (!v)
          {
            v = is_src_root (out_root, altn)
              ? out_root
              : (root.src_path () / p.second);
          }
          else
            remap_src_root (ctx, v); // Remap if inside old_src_root.

          setup_root (rs, forwarded (root, out_root, v.as<dir_path> (), altn));
          bootstrap_pre (rs, altn);
          bootstrap_src (rs, altn);
          bootstrap_post (rs);
        }
        else
        {
          altn = rs.root_extra->altn;
          if (forwarded (root, rs.out_path (), rs.src_path (), altn))
            rs.assign (ctx.var_forwarded) = true; // Only upgrade (see main()).
        }

        // Check if we strongly amalgamated this inner root scope.
        //
        if (rs.src_path ().sub (root.src_path ()))
          rs.strong_ = root.strong_scope (); // Itself or some outer scope.

        // See if there are more inner roots.
        //
        r = &create_bootstrap_inner (rs, out_base);

        if (!out_base.empty ())
          break; // We have found our subproject.
      }
    }

    return *r;
  }

  void
  load_root (scope& root)
  {
    tracer trace ("load_root");

    const dir_path& out_root (root.out_path ());
    const dir_path& src_root (root.src_path ());

    // As an optimization, check if we have already loaded root.build. If
    // that's the case, then we have already been called for this project.
    //
    path f (src_root / root.root_extra->root_file);

    if (root.buildfiles.find (f) != root.buildfiles.end ())
      return;

    // First load outer roots, if any.
    //
    if (scope* rs = root.parent_scope ()->root_scope ())
      load_root (*rs);

    // Finish off initializing bootstrapped modules.
    //
    for (auto& p: root.root_extra->modules)
    {
      module_state& s (p.second);

      if (s.boot && s.first)
        load_module (root, root, p.first, s.loc);
    }

    for (auto& p: root.root_extra->modules)
    {
      module_state& s (p.second);

      if (s.boot && !s.first)
        load_module (root, root, p.first, s.loc);
    }

    // Load hooks and root.build.
    //
    // We can load the pre hooks before finishing off loading the bootstrapped
    // modules (which, in case of config would load config.build) or after and
    // one can come up with a plausible use-case for either approach. Note,
    // however, that one can probably achieve adequate pre-modules behavior
    // with a post-bootstrap hook.
    //
    dir_path hd (out_root / root.root_extra->root_dir);
    bool he (exists (hd));

    if (he)         source_hooks (root, hd, true /* pre */);
    if (exists (f)) source_once  (root, root, f);
    if (he)         source_hooks (root, hd, false /* pre */);
  }

  scope&
  load_project (scope& s,
                const dir_path& out_root,
                const dir_path& src_root,
                bool forwarded,
                bool load)
  {
    assert (!forwarded || out_root != src_root);

    context& ctx (s.ctx);

    auto i (create_root (s, out_root, src_root));
    scope& rs (i->second);

    if (!bootstrapped (rs))
    {
      optional<bool> altn;
      bootstrap_out (rs, altn);
      setup_root (rs, forwarded);
      bootstrap_pre (rs, altn);
      bootstrap_src (rs, altn);
      bootstrap_post (rs);
    }
    else
    {
      if (forwarded)
        rs.assign (ctx.var_forwarded) = true; // Only upgrade (see main()).
    }

    if (load)
    {
      load_root (rs);
      setup_base (i, out_root, src_root); // Setup as base.
    }

    return rs;
  }

  pair<name, dir_path>
  import_search (scope& ibase, name target, const location& loc, bool subp)
  {
    tracer trace ("import_search");

    // If there is no project specified for this target, then our run will be
    // short and sweet: we simply return it as empty-project-qualified and
    // let someone else (e.g., a rule) take a stab at it.
    //
    if (target.unqualified ())
    {
      target.proj = project_name ();
      return make_pair (move (target), dir_path ());
    }

    context& ctx (ibase.ctx);

    // Otherwise, get the project name and convert the target to unqualified.
    //
    project_name proj (move (*target.proj));
    target.proj = nullopt;

    scope& iroot (*ibase.root_scope ());

    // Figure out the imported project's out_root.
    //
    dir_path out_root;

    // First try the config.import.* mechanism. The idea is that if the user
    // explicitly told us the project's location, then we should prefer that
    // over anything that we may discover. In particular, we will prefer it
    // over any bundled subprojects.
    //
    auto& vp (ibase.ctx.var_pool.rw (iroot));

    for (;;) // Break-out loop.
    {
      string n ("config.import." + proj.variable ());

      auto skip = [&target, &proj, &trace] ()
      {
        target.proj = move (proj);
        l5 ([&]{trace << "skipping " << target;});
        return make_pair (move (target), dir_path ());
      };

      // config.import.<proj>
      //
      {
        // Note: pattern-typed in context ctor as an overridable variable of
        // type abs_dir_path (path auto-completion).
        //
        const variable& var (vp.insert (n));

        if (auto l = iroot[var])
        {
          out_root = cast<dir_path> (l);      // Normalized and actualized.

          // Mark as part of config.
          //
          if (config_save_variable != nullptr)
            config_save_variable (iroot, var, 0 /* flags */);

          // Empty config.import.* value means don't look in subprojects or
          // amalgamations and go straight to the rule-specific import (e.g.,
          // to use system-installed).
          //
          if (out_root.empty ())
            return skip ();

          break;
        }
      }

      // config.import.<proj>.<name>.<type>
      // config.import.<proj>.<name>
      //
      // For example: config.import.build2.b.exe=/opt/build2/bin/b
      //
      if (!target.value.empty ())
      {
        auto lookup = [&iroot, &vp, &loc] (string name) -> path
        {
          // Note: pattern-typed in context ctor as an overridable variable of
          // type path.
          //
          const variable& var (vp.insert (move (name)));

          path r;
          if (auto l = iroot[var])
          {
            r = cast<path> (l);

            if (r.empty ())
              fail (loc) << "empty path in " << var.name;

            if (config_save_variable != nullptr)
              config_save_variable (iroot, var, 0 /* flags */);
          }

          return r;
        };

        // First try .<name>.<type>, then just .<name>.
        //
        path p;
        if (target.typed ())
          p = lookup (n + '.' + target.value + '.' + target.type);

        if (p.empty ())
          p = lookup (n + '.' + target.value);

        if (!p.empty ())
        {
          // If the path is relative, then keep it project-qualified assuming
          // import phase 2 knows what to do with it. Think:
          //
          // config.import.build2.b=b-boot
          //
          if (p.relative ())
            target.proj = move (proj);

          target.dir = p.directory ();
          target.value = p.leaf ().string ();

          return make_pair (move (target), dir_path ());
        }
      }

      // import.build2
      //
      if (proj == "build2")
      {
        // Note that this variable can be set to NULL to disable relying on
        // the built-in path. We use this in our tests to make sure we are
        // importing and testing the build system being built and not the one
        // doing the building.
        //
        if (auto l = iroot[ctx.var_import_build2])
        {
          out_root = cast<dir_path> (l);

          if (out_root.empty ())
            return skip ();

          break;
        }
      }

      // Otherwise search subprojects, starting with our root and then trying
      // outer roots for as long as we are inside an amalgamation.
      //
      if (subp)
      {
        for (scope* r (&iroot);; r = r->parent_scope ()->root_scope ())
        {
          l5 ([&]{trace << "looking in " << *r;});

          // First check the amalgamation itself.
          //
          if (r != &iroot &&
              cast<project_name> (r->vars[ctx.var_project]) == proj)
          {
            out_root = r->out_path ();
            break;
          }

          if (auto l = r->vars[ctx.var_subprojects])
          {
            const auto& m (cast<subprojects> (l));
            auto i (m.find (proj));

            if (i != m.end ())
            {
              const dir_path& d ((*i).second);
              out_root = r->out_path () / d;
              break;
            }
          }

          if (!r->vars[ctx.var_amalgamation])
            break;
        }
      }

      break;
    }

    // Add the qualification back to the target (import_load() will remove it
    // again).
    //
    target.proj = move (proj);

    return make_pair (move (target), move (out_root));
  }

  pair<names, const scope&>
  import_load (context& ctx, pair<name, dir_path> x, const location& loc)
  {
    tracer trace ("import_load");

    name target (move (x.first));
    dir_path out_root (move (x.second));

    assert (target.proj);
    project_name proj (move (*target.proj));
    target.proj = nullopt;

    // Bootstrap the imported root scope. This is pretty similar to what we do
    // in main() except that here we don't try to guess src_root.
    //
    // The user can also specify the out_root of the amalgamation that contains
    // our project. For now we only consider top-level sub-projects.
    //
    scope* root;
    dir_path src_root;

    // See if this is a forwarded configuration. For top-level project we want
    // to use the same logic as in main() while for inner subprojects -- as in
    // create_bootstrap_inner().
    //
    bool fwd (false);
    optional<bool> altn;
    if (is_src_root (out_root, altn))
    {
      src_root = move (out_root);
      out_root = bootstrap_fwd (ctx, src_root, altn);
      fwd = (src_root != out_root);
    }

    scope& gs (ctx.global_scope.rw ());

    for (const scope* proot (nullptr); ; proot = root)
    {
      bool top (proot == nullptr);

      root = &create_root (gs, out_root, src_root)->second;

      bool bstrapped (bootstrapped (*root));

      if (!bstrapped)
      {
        bootstrap_out (*root, altn);

        // Check that the bootstrap process set src_root.
        //
        auto l (root->vars[*ctx.var_src_root]);
        if (l)
        {
          // Note that unlike main() here we fail hard. The idea is that if
          // the project we are importing is misconfigured, then it should be
          // fixed first.
          //
          const dir_path& p (cast<dir_path> (l));

          if (!src_root.empty () && p != src_root)
            fail (loc) << "configured src_root " << p << " does not match "
                       << "discovered " << src_root;
        }
        else
          fail (loc) << "unable to determine src_root for imported " << proj <<
            info << "consider configuring " << out_root;

        setup_root (*root,
                    (top
                     ? fwd
                     : forwarded (*proot, out_root, l->as<dir_path> (), altn)));

        bootstrap_pre (*root, altn);
        bootstrap_src (*root, altn);
        if (!top)
          bootstrap_post (*root);
      }
      else
      {
        altn = root->root_extra->altn;

        if (src_root.empty ())
          src_root = root->src_path ();

        if (top ? fwd : forwarded (*proot, out_root, src_root, altn))
          root->assign (ctx.var_forwarded) = true; // Only upgrade (see main()).
      }

      if (top)
      {
        create_bootstrap_outer (*root);

        if (!bstrapped)
          bootstrap_post (*root);
      }

      // Now we know this project's name as well as all its subprojects.
      //
      if (cast<project_name> (root->vars[ctx.var_project]) == proj)
        break;

      if (auto l = root->vars[ctx.var_subprojects])
      {
        const auto& m (cast<subprojects> (l));
        auto i (m.find (proj));

        if (i != m.end ())
        {
          const dir_path& d ((*i).second);
          altn = nullopt;
          out_root = root->out_path () / d;
          src_root = is_src_root (out_root, altn) ? out_root : dir_path ();
          continue;
        }
      }

      fail (loc) << out_root << " is not out_root for " << proj;
    }

    // Load the imported root scope.
    //
    load_root (*root);

    // Use a temporary scope so that the export stub doesn't mess anything up.
    //
    temp_scope ts (gs);

    // "Pass" the imported project's roots to the stub.
    //
    ts.assign (ctx.var_out_root) = move (out_root);
    ts.assign (ctx.var_src_root) = move (src_root);

    // Also pass the target being imported in the import.target variable.
    //
    {
      value& v (ts.assign (ctx.var_import_target));

      if (!target.empty ()) // Otherwise leave NULL.
        v = target; // Can't move (need for diagnostics below).
    }

    // Load the export stub. Note that it is loaded in the context
    // of the importing project, not the imported one. The export
    // stub will normally switch to the imported root scope at some
    // point.
    //
    path es (root->src_path () / root->root_extra->export_file);

    try
    {
      ifdstream ifs (es);

      l5 ([&]{trace << "importing " << es;});

      // @@ Should we verify these are all unqualified names? Or maybe
      // there is a use-case for the export stub to return a qualified
      // name?
      //
      parser p (ctx);
      names v (p.parse_export_stub (ifs, es, gs, ts));

      // If there were no export directive executed in an export stub, assume
      // the target is not exported.
      //
      if (v.empty () && !target.empty ())
        fail (loc) << "target " << target << " is not exported by project "
                   << proj;

      return pair<names, const scope&> (move (v), *root);
    }
    catch (const io_error& e)
    {
      fail (loc) << "unable to read buildfile " << es << ": " << e << endf;
    }
  }

  names
  import (scope& base, name target, const location& loc)
  {
    tracer trace ("import");

    l5 ([&]{trace << target << " from " << base;});

    pair<name, dir_path> r (import_search (base, move (target), loc));

    // If we couldn't find the project, return to let someone else (e.g., a
    // rule) take a stab at it.
    //
    if (r.second.empty ())
    {
      l5 ([&]{trace << "postponing " << r.first;});
      return names {move (r.first)};
    }

    return import_load (base.ctx, move (r), loc).first;
  }

  const target*
  import (context& ctx, const prerequisite_key& pk, bool existing)
  {
    tracer trace ("import");

    assert (pk.proj);
    const project_name& proj (*pk.proj);

    // Target type-specific search.
    //
    const target_key& tk (pk.tk);
    const target_type& tt (*tk.type);

    // Try to find the executable in PATH (or CWD if relative).
    //
    if (tt.is_a<exe> ())
    {
      path n (*tk.dir);
      n /= *tk.name;
      if (tk.ext)
      {
        n += '.';
        n += *tk.ext;
      }

      // Only search in PATH (or CWD).
      //
      process_path pp (process::try_path_search (n, true, dir_path (), true));

      if (!pp.empty ())
      {
        path& p (pp.effect);
        assert (!p.empty ()); // We searched for a simple name.

        const exe* t (
          !existing
          ? &ctx.targets.insert<exe> (tt,
                                      p.directory (),
                                      dir_path (),    // No out (not in project).
                                      p.leaf ().base ().string (),
                                      p.extension (), // Always specified.
                                      trace)
          : ctx.targets.find<exe> (tt,
                                   p.directory (),
                                   dir_path (),
                                   p.leaf ().base ().string (),
                                   p.extension (),
                                   trace));

        if (t != nullptr)
        {
          if (!existing)
            t->path (move (p));
          else
            assert (t->path () == p);

          return t;
        }
      }
    }

    if (existing)
      return nullptr;

    // @@ We no longer have location. This is especially bad for the
    //    empty case, i.e., where do I need to specify the project
    //    name)? Looks like the only way to do this is to keep location
    //    in name and then in prerequisite. Perhaps one day...
    //
    diag_record dr;
    dr << fail << "unable to import target " << pk;

    if (proj.empty ())
      dr << info << "consider adding its installation location" <<
        info << "or explicitly specify its project name";
    else
      dr << info << "use config.import." << proj.variable ()
         << " command line variable to specify its project out_root";

    dr << endf;
  }
}