Add support for configuration exporting and importing

The new config.export variable specifies the alternative file to write the
configuration to as part of the configure meta-operation. For example:

$ b configure: proj/ config.export=proj-config.build

The config.export value "applies" only to the projects on whose root scope it
is specified or if it is a global override (the latter is a bit iffy but we
allow it, for example, to dump everything to stdout). This means that in order
to save a subproject's configuration we will have to use a scope-specific
override (since the default will apply to the outermost amalgamation). For
example:

$ b configure: subproj/ subproj/config.export=.../subproj-config.build

This could be somewhat unnatural but then it will be the amalgamation whose
configuration we normally want to export.

The new config.import variable specifies additional configuration files to be
loaded after the project's default config.build, if any. For example:

$ b create: cfg/,cc config.import=my-config.build

Similar to config.export, the config.import value "applies" only to the
project on whose root scope it is specified or if it is a global override.
This allows the use of the standard override "positioning" machinery (i.e.,
where the override applies) to decide where the extra configuration files are
loaded. The resulting semantics is quite natural and consistent with command
line variable overrides, for example:

$ b   config.import=.../config.build  # outermost amalgamation
$ b ./config.import=.../config.build  # this project
$ b  !config.import=.../config.build  # every project

Both config.export and config.import recognize the special `-` file name as an
instruction to write/read to/from stdout/stdin, respectively. For example:

$ b configure: src-prj/ config.export=- | b configure: dst-prj/ config.import=-

3 files changed, 244 insertions, 130 deletions

diff --git a/libbuild2/config/init.cxx b/libbuild2/config/init.cxx
index 6998017..4e1890a 100644
--- a/libbuild2/config/init.cxx
+++ b/libbuild2/config/init.cxx
@@ -6,6 +6,7 @@
 
 #include <libbuild2/file.hxx>
 #include <libbuild2/rule.hxx>
+#include <libbuild2/lexer.hxx>
 #include <libbuild2/scope.hxx>
 #include <libbuild2/context.hxx>
 #include <libbuild2/filesystem.hxx>  // exists()
@@ -32,9 +33,18 @@ namespace build2
       const string& mname (rs.ctx.current_mname);
       const string& oname (rs.ctx.current_oname);
 
-      // Only create the module if we are configuring or creating. This is a
-      // bit tricky since the build2 core may not yet know if this is the
-      // case. But we know.
+      // While config.import (see below) could theoretically be specified in a
+      // buildfile, config.export is expected to always be specified as a
+      // command line override.
+      //
+      // Note: must be entered during bootstrap since we need it in
+      // configure_execute().
+      //
+      vp.insert<path> ("config.export", true /* ovr */);
+
+      // Only create the module if we are configuring or creating or if it was
+      // forced with config.module (useful if we need to call $config.export()
+      // during other meta-operations).
       //
       if ((                   mname == "configure" || mname == "create") ||
           (mname.empty () && (oname == "configure" || oname == "create")))
@@ -80,42 +90,80 @@ namespace build2
 
       assert (config_hints.empty ()); // We don't known any hints.
 
+      // Note that the config.<name>* variables belong to the module <name>.
+      // So the only "special" variables we can allocate in config.* are
+      // config.config.*, names that have been "gifted" to us by other modules
+      // (like config.version) as well as names that we have reserved to not
+      // be valid module names (build, import, export).
+      //
       auto& vp (rs.ctx.var_pool.rw (rs));
 
-      // Load config.build if one exists (we don't need to worry about
-      // disfigure since we will never be init'ed).
+      auto& c_v (vp.insert<uint64_t> ("config.version", false /*ovr*/));
+      auto& c_i (vp.insert<paths> ("config.import", true /* ovr */));
+
+      // Load config.build if one exists followed by extra files specified in
+      // config.import (we don't need to worry about disfigure since we will
+      // never be init'ed).
       //
-      const variable& c_v (vp.insert<uint64_t> ("config.version", false));
+      auto load_config = [&rs, &c_v] (const path& f, const location& l)
+      {
+        // Check the config version. We assume that old versions cannot
+        // understand new configs and new versions are incompatible with old
+        // configs.
+        //
+        // We extract the value manually instead of loading and then checking
+        // in order to be able to fixup/migrate the file which we may want to
+        // do in the future.
+        //
+
+        // This is tricky for stdin since we cannot reopen it (or put more
+        // than one character back). So what we are going to do is continue
+        // reading after extracting the variable. One side effect of this is
+        // that we won't have the config.version variable entered in the scope
+        // but that is harmless (we could do it manually if necessary).
+        //
+        ifdstream ifs;
+        lexer lex (open_file_or_stdin (f, ifs), f);
+
+        // Assume missing version is 0.
+        //
+        auto p (extract_variable (rs.ctx, lex, c_v));
+        uint64_t v (p.second ? cast<uint64_t> (p.first) : 0);
+
+        if (v != module::version)
+          fail (l) << "incompatible config file " << f <<
+            info << "config file version   " << v
+                   << (p.second ? "" : " (missing)") <<
+            info << "config module version " << module::version <<
+            info << "consider reconfiguring " << project (rs) << '@'
+                   << rs.out_path ();
+
+        source (rs, rs, lex);
+      };
 
       {
         path f (config_file (rs));
 
         if (exists (f))
+          load_config (f, l);
+      }
+
+      if (lookup l = rs[c_i])
+      {
+        // Only load files that were specified on our root scope as well as
+        // global overrides. This way we can use our override "positioning"
+        // machinery (i.e., where the override applies) to decide where the
+        // extra config is loaded. The resulting semantics feels quite natural
+        // and consistent with command line variable overrides:
+        //
+        // b   config.import=.../config.build  # outermost amalgamation
+        // b ./config.import=.../config.build  # this project
+        // b  !config.import=.../config.build  # every project
+        //
+        if (l.belongs (rs) || l.belongs (rs.ctx.global_scope))
         {
-          // Check the config version. We assume that old versions cannot
-          // understand new configs and new versions are incompatible with old
-          // configs.
-          //
-          // We extract the value manually instead of loading and then
-          // checking in order to be able to fixup/migrate the file which we
-          // may want to do in the future.
-          //
-          {
-            // Assume missing version is 0.
-            //
-            auto p (extract_variable (rs.ctx, f, c_v));
-            uint64_t v (p.second ? cast<uint64_t> (p.first) : 0);
-
-            if (v != module::version)
-              fail (l) << "incompatible config file " << f <<
-                info << "config file version   " << v
-                         << (p.second ? "" : " (missing)") <<
-                info << "config module version " << module::version <<
-                info << "consider reconfiguring " << project (rs) << '@'
-                         << out_root;
-          }
-
-          source (rs, rs, f);
+          for (const path& f: cast<paths> (l))
+            load_config (f, location (&f));
         }
       }
 
diff --git a/libbuild2/config/module.cxx b/libbuild2/config/module.cxx
index 7e9b765..b43f1d9 100644
--- a/libbuild2/config/module.cxx
+++ b/libbuild2/config/module.cxx
@@ -30,7 +30,7 @@ namespace build2
         i = sm.insert (string (n, 0, n.find ('.', 7)));
       }
 
-      // Don't insert duplicates. The config.import vars are particularly
+      // Don't insert duplicates. The config.import.* vars are particularly
       // susceptible to duplication.
       //
       saved_variables& sv (i->second);
diff --git a/libbuild2/config/operation.cxx b/libbuild2/config/operation.cxx
index 535018e..9050645 100644
--- a/libbuild2/config/operation.cxx
+++ b/libbuild2/config/operation.cxx
@@ -87,35 +87,45 @@ namespace build2
 
     using project_set = set<const scope*>; // Use pointers to get comparison.
 
+    // If inherit is false, then don't rely on inheritance from outer scopes
+    // (used for config.export).
+    //
     static void
-    save_config (const scope& rs, const project_set& projects)
+    save_config (const scope& rs,
+                 const path& f,
+                 bool inherit,
+                 const project_set& projects)
     {
       context& ctx (rs.ctx);
 
-      path f (config_file (rs));
+      const module& mod (*rs.lookup_module<const module> (module::name));
 
-      if (verb)
-        text << (verb >= 2 ? "cat >" : "save ") << f;
+      const string& df (f.string () != "-" ? f.string () : "<stdout>");
 
-      const module& mod (*rs.lookup_module<const module> (module::name));
+      if (verb)
+        text << (verb >= 2 ? "cat >" : "save ") << df;
 
       try
       {
-        ofdstream ofs (f);
+        ofdstream ofs;
+        ostream& os (open_file_or_stdout (f, ofs));
 
-        ofs << "# Created automatically by the config module, but feel " <<
+        os << "# Created automatically by the config module, but feel " <<
           "free to edit." << endl
-            << "#" << endl;
+           << "#" << endl;
 
-        ofs << "config.version = " << module::version << endl;
+        os << "config.version = " << module::version << endl;
 
-        if (auto l = rs.vars[ctx.var_amalgamation])
+        if (inherit)
         {
-          const dir_path& d (cast<dir_path> (l));
+          if (auto l = rs.vars[ctx.var_amalgamation])
+          {
+            const dir_path& d (cast<dir_path> (l));
 
-          ofs << endl
-              << "# Base configuration inherited from " << d << endl
-              << "#" << endl;
+            os << endl
+               << "# Base configuration inherited from " << d << endl
+               << "#" << endl;
+          }
         }
 
         // Save config variables.
@@ -146,6 +156,11 @@ namespace build2
             if (!l.defined ())
               continue;
 
+            // Handle inherited from outer scope values.
+            //
+            // Note that we keep this logic (with warnings and all) even if
+            // inherit is false to make things easier to reason about.
+            //
             if (!(l.belongs (rs) || l.belongs (ctx.global_scope)))
             {
               // This is presumably an inherited value. But it could also be
@@ -205,42 +220,50 @@ namespace build2
                 }
               }
 
-              if (found) // Inherited.
-                continue;
-
-              location loc (&f);
-
-              // If this value is not defined in a project's root scope, then
-              // something is broken.
-              //
-              if (r == nullptr)
-                fail (loc) << "inherited variable " << var << " value "
-                           << "is not from a root scope";
-
-              // If none of the outer project's configurations use this value,
-              // then we warn and save as our own. One special case where we
-              // don't want to warn the user is if the variable is overriden.
-              //
-              if (org.first == ovr.first)
+              if (found)
               {
-                diag_record dr;
-                dr << warn (loc) << "saving previously inherited variable "
-                   << var;
-
-                dr << info (loc) << "because project " << *r
-                   << " no longer uses it in its configuration";
-
-                if (verb >= 2)
+                // Inherited.
+                //
+                if (inherit)
+                  continue;
+              }
+              else
+              {
+                location loc (&f);
+
+                // If this value is not defined in a project's root scope,
+                // then something is broken.
+                //
+                if (r == nullptr)
+                  fail (loc) << "inherited variable " << var << " value "
+                             << "is not from a root scope";
+
+                // If none of the outer project's configurations use this
+                // value, then we warn and save as our own. One special case
+                // where we don't want to warn the user is if the variable is
+                // overriden.
+                //
+                if (org.first == ovr.first)
                 {
-                  dr << info (loc) << "variable value: ";
+                  diag_record dr;
+                  dr << warn (loc) << "saving previously inherited variable "
+                     << var;
 
-                  if (*l)
+                  dr << info (loc) << "because project " << *r
+                     << " no longer uses it in its configuration";
+
+                  if (verb >= 2)
                   {
-                    storage.clear ();
-                    dr << "'" << reverse (*l, storage) << "'";
+                    dr << info (loc) << "variable value: ";
+
+                    if (*l)
+                    {
+                      storage.clear ();
+                      dr << "'" << reverse (*l, storage) << "'";
+                    }
+                    else
+                      dr << "[null]";
                   }
-                  else
-                    dr << "[null]";
                 }
               }
             }
@@ -264,7 +287,7 @@ namespace build2
             //
             if (first)
             {
-              ofs << endl;
+              os << endl;
               first = false;
             }
 
@@ -274,7 +297,7 @@ namespace build2
                 org.first == ovr.first &&                     // Not overriden.
                 (sv.flags & save_commented) == save_commented)
             {
-              ofs << '#' << n << " =" << endl;
+              os << '#' << n << " =" << endl;
               continue;
             }
 
@@ -283,20 +306,20 @@ namespace build2
               storage.clear ();
               names_view ns (reverse (v, storage));
 
-              ofs << n;
+              os << n;
 
               if (ns.empty ())
-                ofs << " =";
+                os << " =";
               else
               {
-                ofs << " = ";
-                to_stream (ofs, ns, true, '@'); // Quote.
+                os << " = ";
+                to_stream (os, ns, true, '@'); // Quote.
               }
 
-              ofs << endl;
+              os << endl;
             }
             else
-              ofs << n << " = [null]" << endl;
+              os << n << " = [null]" << endl;
           }
         }
 
@@ -304,12 +327,15 @@ namespace build2
       }
       catch (const io_error& e)
       {
-        fail << "unable to write " << f << ": " << e;
+        fail << "unable to write " << df << ": " << e;
       }
     }
 
     static void
-    configure_project (action a, const scope& rs, project_set& projects)
+    configure_project (action a,
+                       const scope& rs,
+                       const variable* c_e, // config.export
+                       project_set& projects)
     {
       tracer trace ("configure_project");
 
@@ -332,8 +358,7 @@ namespace build2
         mkdir (out_root / rs.root_extra->bootstrap_dir, 2);
       }
 
-      // We distinguish between a complete configure and operation-
-      // specific.
+      // We distinguish between a complete configure and operation-specific.
       //
       if (a.operation () == default_id)
       {
@@ -341,15 +366,48 @@ namespace build2
 
         // Save src-root.build unless out_root is the same as src.
         //
-        if (out_root != src_root)
+        if (c_e == nullptr && out_root != src_root)
           save_src_root (rs);
 
-        // Save config.build.
+        // Save config.build unless an alternative is specified with
+        // config.export. Similar to config.import we will only save to that
+        // file if it is specified on our root scope or as a global override
+        // (the latter is a bit iffy but let's allow it, for example, to dump
+        // everything to stdout). Note that to save a subproject's config we
+        // will have to use a scope-specific override (since the default will
+        // apply to the amalgamation):
         //
-        save_config (rs, projects);
+        // b configure: subproj/ subproj/config.export=.../config.build
+        //
+        // Could be confusing but then normally it will be the amalgamation
+        // whose configuration we want to export.
+        //
+        // Note also that if config.export is specified we do not rewrite
+        // config.build files (say, of subprojects) as well as src-root.build
+        // above. Failed that, if we are running in a disfigured project, we
+        // may end up leaving it in partially configured state.
+        //
+        if (c_e == nullptr)
+          save_config (rs, config_file (rs), true /* inherit */, projects);
+        else
+        {
+          lookup l (rs[*c_e]);
+          if (l && (l.belongs (rs) || l.belongs (ctx.global_scope)))
+          {
+            // While writing the complete configuration seems like a natural
+            // default, there might be a desire to take inheritance into
+            // account (if, say, we are exporting at multiple levels). One can
+            // of course just copy the relevant config.build files, but we may
+            // still want to support this mode somehow in the future (maybe
+            // using `+` as a modifier, say config.export=+.../config.build).
+            //
+            save_config (rs, cast<path> (l), false /* inherit */, projects);
+          }
+        }
       }
       else
       {
+        fail << "operation-specific configuration not yet supported";
       }
 
       // Configure subprojects that have been loaded.
@@ -368,7 +426,7 @@ namespace build2
           if (nrs.out_path () != out_nroot) // This subproject not loaded.
             continue;
 
-          configure_project (a, nrs, projects);
+          configure_project (a, nrs, c_e, projects);
         }
       }
     }
@@ -513,6 +571,15 @@ namespace build2
     {
       bool fwd (forward (params));
 
+      context& ctx (fwd ? ts[0].as<scope> ().ctx : ts[0].as<target> ().ctx);
+
+      const variable* c_e (ctx.var_pool.find ("config.export"));
+
+      if (c_e->overrides == nullptr)
+        c_e = nullptr;
+      else if (fwd)
+        fail << "config.export specified for forward configuration";
+
       project_set projects;
 
       for (const action_target& at: ts)
@@ -521,53 +588,52 @@ namespace build2
         {
           // Forward configuration.
           //
-          const scope& rs (*static_cast<const scope*> (at.target));
+          const scope& rs (at.as<scope> ());
           configure_forward (rs, projects);
-          continue;
         }
+        else
+        {
+          // Normal configuration.
+          //
+          // Match rules to configure every operation supported by each
+          // project. Note that we are not calling operation_pre/post()
+          // callbacks here since the meta operation is configure and we know
+          // what we are doing.
+          //
+          // Note that we cannot do this in parallel. We cannot parallelize
+          // the outer loop because we should match for a single action at a
+          // time. And we cannot swap the loops because the list of operations
+          // is target-specific. However, inside match(), things can proceed
+          // in parallel.
+          //
+          const target& t (at.as<target> ());
+          const scope* rs (t.base_scope ().root_scope ());
 
-        // Normal configuration.
-        //
-        // Match rules to configure every operation supported by each project.
-        // Note that we are not calling operation_pre/post() callbacks here
-        // since the meta operation is configure and we know what we are
-        // doing.
-        //
-        // Note that we cannot do this in parallel. We cannot parallelize the
-        // outer loop because we should match for a single action at a time.
-        // And we cannot swap the loops because the list of operations is
-        // target-specific. However, inside match(), things can proceed in
-        // parallel.
-        //
-        const target& t (at.as_target ());
-        const scope* rs (t.base_scope ().root_scope ());
-
-        if (rs == nullptr)
-          fail << "out of project target " << t;
-
-        context& ctx (t.ctx);
+          if (rs == nullptr)
+            fail << "out of project target " << t;
 
-        const operations& ops (rs->root_extra->operations);
+          const operations& ops (rs->root_extra->operations);
 
-        for (operation_id id (default_id + 1); // Skip default_id.
-             id < ops.size ();
-             ++id)
-        {
-          if (const operation_info* oif = ops[id])
+          for (operation_id id (default_id + 1); // Skip default_id.
+               id < ops.size ();
+               ++id)
           {
-            // Skip aliases (e.g., update-for-install).
-            //
-            if (oif->id != id)
-              continue;
+            if (const operation_info* oif = ops[id])
+            {
+              // Skip aliases (e.g., update-for-install).
+              //
+              if (oif->id != id)
+                continue;
 
-            ctx.current_operation (*oif);
+              ctx.current_operation (*oif);
 
-            phase_lock pl (ctx, run_phase::match);
-            match (action (configure_id, id), t);
+              phase_lock pl (ctx, run_phase::match);
+              match (action (configure_id, id), t);
+            }
           }
-        }
 
-        configure_project (a, *rs, projects);
+          configure_project (a, *rs, c_e, projects);
+        }
       }
     }
 
@@ -802,7 +868,7 @@ namespace build2
       //
       for (const action_target& at: ts)
       {
-        const scope& rs (*static_cast<const scope*> (at.target));
+        const scope& rs (at.as<scope> ());
 
         if (!(fwd
               ? disfigure_forward (   rs, projects)