Dist module/meta-operation initial implementation

6 files changed, 670 insertions, 0 deletions

diff --git a/build/dist/module b/build/dist/module
new file mode 100644
index 0000000..9fad86f
--- /dev/null
+++ b/build/dist/module
@@ -0,0 +1,21 @@
+// file      : build/dist/module -*- C++ -*-
+// copyright : Copyright (c) 2014-2015 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BUILD_DIST_MODULE
+#define BUILD_DIST_MODULE
+
+#include <build/types>
+#include <build/module>
+
+namespace build
+{
+  namespace dist
+  {
+    extern "C" void
+    dist_init (
+      scope&, scope&, const location&, std::unique_ptr<module>&, bool);
+  }
+}
+
+#endif // BUILD_DIST_MODULE
diff --git a/build/dist/module.cxx b/build/dist/module.cxx
new file mode 100644
index 0000000..7b9b340
--- /dev/null
+++ b/build/dist/module.cxx
@@ -0,0 +1,130 @@
+// file      : build/dist/module.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2015 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <build/dist/module>
+
+#include <build/scope>
+#include <build/file>
+#include <build/diagnostics>
+
+#include <build/config/utility>
+
+#include <build/dist/rule>
+#include <build/dist/operation>
+
+using namespace std;
+using namespace butl;
+
+namespace build
+{
+  namespace dist
+  {
+    static rule rule_;
+
+    extern "C" void
+    dist_init (scope& r,
+               scope& b,
+               const location& l,
+               std::unique_ptr<module>&,
+               bool first)
+    {
+      tracer trace ("dist::init");
+
+      if (&r != &b)
+        fail (l) << "dist module must be initialized in bootstrap.build";
+
+      if (!first)
+      {
+        warn (l) << "multiple dist module initializations";
+        return;
+      }
+
+      const dir_path& out_root (r.path ());
+      level4 ([&]{trace << "for " << out_root;});
+
+      // Register meta-operation.
+      //
+      r.meta_operations.insert (dist_id, dist);
+
+      // Register our wildcard rule. Do it explicitly for the alias
+      // to prevent something like insert<target>(dist_id, test_id)
+      // taking precedence.
+      //
+      r.rules.insert<target> (dist_id, 0, "dist", rule_);
+      r.rules.insert<alias> (dist_id, 0, "dist", rule_);
+
+      // Enter module variables.
+      //
+      if (first)
+      {
+        variable_pool.find ("dist.package", string_type);
+
+        variable_pool.find ("dist.root", dir_path_type);
+        variable_pool.find ("config.dist.root", dir_path_type);
+
+        //@@ VAR type
+        //
+        variable_pool.find ("dist.cmd", string_type);
+        variable_pool.find ("config.dist.cmd", string_type);
+
+        variable_pool.find ("dist.archives", strings_type);
+        variable_pool.find ("config.dist.archives", strings_type);
+      }
+
+      // Configuration.
+      //
+      // Note that we don't use any defaults for root -- the location
+      // must be explicitly specified or we will complain if and when
+      // we try to dist.
+      //
+      using namespace config;
+
+      bool s (specified (r, "config.dist"));
+
+      // dist.root
+      //
+      {
+        value& v (r.assign ("dist.root"));
+
+        if (s)
+        {
+          const value& cv (optional_absolute (r, "config.dist.root"));
+
+          if (cv && !cv.empty ())
+            v = cv;
+        }
+      }
+
+      // dist.cmd
+      //
+      {
+        value& v (r.assign ("dist.cmd"));
+
+        if (s)
+        {
+          const value& cv (required (r, "config.dist.cmd", "install").first);
+
+          if (cv && !cv.empty ())
+            v = cv;
+        }
+        else
+          v = "install";
+      }
+
+      // dist.archives
+      //
+      {
+        value& v (r.assign ("dist.archives"));
+
+        if (s)
+        {
+          const value& cv (optional (r, "config.dist.archives"));
+
+          if (cv && !cv.empty ())
+            v = cv;
+        }
+      }
+    }
+  }
+}
diff --git a/build/dist/operation b/build/dist/operation
new file mode 100644
index 0000000..c7aa014
--- /dev/null
+++ b/build/dist/operation
@@ -0,0 +1,18 @@
+// file      : build/dist/operation -*- C++ -*-
+// copyright : Copyright (c) 2014-2015 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BUILD_DIST_OPERATION
+#define BUILD_DIST_OPERATION
+
+#include <build/operation>
+
+namespace build
+{
+  namespace dist
+  {
+    extern meta_operation_info dist;
+  }
+}
+
+#endif // BUILD_DIST_OPERATION
diff --git a/build/dist/operation.cxx b/build/dist/operation.cxx
new file mode 100644
index 0000000..3c6ec98
--- /dev/null
+++ b/build/dist/operation.cxx
@@ -0,0 +1,423 @@
+// file      : build/dist/operation.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2015 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <build/dist/operation>
+
+#include <cassert>
+
+#include <butl/process>
+#include <butl/filesystem>
+
+#include <build/file>
+#include <build/dump>
+#include <build/scope>
+#include <build/target>
+#include <build/context>
+#include <build/algorithm>
+#include <build/diagnostics>
+
+using namespace std;
+using namespace butl;
+
+namespace build
+{
+  namespace dist
+  {
+    static void
+    dist_meta_operation_pre ()
+    {
+      // Reset the dependency state so that we don't end up with stray
+      // files from previous batches.
+      //
+      // @@ This is called too late, after we have bootstrapped the
+      //    project.
+      //
+      //reset ();
+    }
+
+    static operation_id
+    dist_operation_pre (operation_id o)
+    {
+      if (o != default_id)
+        fail << "explicit operation specified for dist meta-operation";
+
+      return o;
+    }
+
+    static void
+    dist_match (action, action_targets&)
+    {
+      // Don't match anything -- see execute ().
+    }
+
+    // install -d <dir>
+    //
+    static void
+    install (const string& cmd, const dir_path&);
+
+    // install <file> <dir>
+    //
+    static void
+    install (const string& cmd, file&, const dir_path&);
+
+    // cd <root> && tar|zip ... <pkg>.<ext> <pkg>
+    //
+    static void
+    archive (const dir_path& root, const string& pkg, const string& ext);
+
+    static void
+    dist_execute (action, const action_targets& ts, bool)
+    {
+      tracer trace ("dist_execute");
+
+      // For now we assume all the targets are from the same project.
+      //
+      target& t (*static_cast<target*> (ts[0]));
+      scope* rs (t.base_scope ().root_scope ());
+
+      if (rs == nullptr)
+        fail << "out of project target " << t;
+
+      const dir_path& out_root (rs->path ());
+      const dir_path& src_root (rs->src_path ());
+
+      if (out_root == src_root)
+        fail << "in-tree distribution of target " << t <<
+          info << "distribution requires out-of-tree build";
+
+      // Make sure we have the necessary configuration before
+      // we get down to business.
+      //
+      auto l (rs->vars["dist.root"]);
+
+      if (!l || l->empty ())
+        fail << "unknown root distribution directory" <<
+          info << "did you forget to specify config.dist.root?";
+
+      const dir_path& dist_root (as<dir_path> (*l));
+
+      if (!dir_exists (dist_root))
+        fail << "root distribution directory " << dist_root
+             << " does not exist";
+
+      l = rs->vars["dist.package"];
+
+      if (!l || l->empty ())
+        fail << "unknown distribution package name" <<
+          info << "did you forget to set dist.package?";
+
+      const string& dist_package (as<string> (*l));
+      const string& dist_cmd (as<string> (*rs->vars["dist.cmd"]));
+
+      // Get the list of operations supported by this project. Skip
+      // default_id.
+      //
+      for (operations::size_type id (default_id + 1);
+           id < rs->operations.size ();
+           ++id)
+      {
+        const operation_info* oi (rs->operations[id]);
+        if (oi == nullptr)
+          continue;
+
+        // Note that we are not calling operation_pre/post() callbacks
+        // here since the meta operation is dist and we know what we
+        // are doing.
+        //
+        current_inner_oif = oi;
+        current_outer_oif = nullptr;
+        current_mode = oi->mode;
+        dependency_count = 0;
+
+        action a (dist_id, id);
+
+        if (verb >= 5)
+          dump (a);
+
+        for (void* v: ts)
+        {
+          target& t (*static_cast<target*> (v));
+
+          if (rs != t.base_scope ().root_scope ())
+            fail << "out of project target " << t;
+
+          level4 ([&]{trace << diag_doing (a, t);});
+
+          match (a, t);
+        }
+
+        if (verb >= 5)
+          dump (a);
+      }
+
+      // Add buildfiles that are not normally loaded as part of the
+      // project, for example, the export stub. They will still be
+      // ignored on the next step if the user explicitly marked them
+      // nodist.
+      //
+      auto add_adhoc = [&src_root, &trace] (const char* f)
+        {
+          path p (src_root / path (f));
+          if (file_exists (p))
+          {
+            const char* e (p.extension ());
+            targets.insert<buildfile> (
+              p.directory (),
+              p.leaf ().base ().string (),
+              &extension_pool.find (e == nullptr ? "" : e), // Specified.
+              trace);
+          }
+        };
+
+      add_adhoc ("build/export.build");
+
+      // Collect the files. We want to take the snapshot of targets
+      // since updating some of them may result in more targets being
+      // entered.
+      //
+      action_targets files;
+
+      for (const auto& pt: targets)
+      {
+        file* ft (pt->is_a<file> ());
+
+        if (ft == nullptr) // Not a file.
+          continue;
+
+        if (ft->dir.sub (src_root))
+        {
+          // Include unless explicitly excluded.
+          //
+          files.push_back (ft);
+          continue;
+        }
+
+        if (ft->dir.sub (out_root))
+        {
+          // Exclude unless explicitly included.
+          //
+          //files.push_back (*ft);
+          continue;
+        }
+      }
+
+      // Make sure what we need to distribute is up to date.
+      //
+      {
+        if (perform.meta_operation_pre != nullptr)
+          perform.meta_operation_pre ();
+
+        current_mif = &perform;
+
+        if (perform.operation_pre != nullptr)
+          perform.operation_pre (update_id);
+
+        current_inner_oif = &update;
+        current_outer_oif = nullptr;
+        current_mode = update.mode;
+        dependency_count = 0;
+
+        action a (perform_id, update_id);
+
+        perform.match (a, files);
+        perform.execute (a, files, true); // Run quiet.
+
+        if (perform.operation_post != nullptr)
+          perform.operation_post (update_id);
+
+        if (perform.meta_operation_post != nullptr)
+          perform.meta_operation_post ();
+      }
+
+      dir_path td (dist_root / dir_path (dist_package));
+
+      // Clean up the target directory.
+      //
+      // @@ Not for incremental dist?
+      //
+      if (build::rmdir_r (td) == rmdir_status::not_empty)
+        fail << "unable to clean target directory " << td;
+
+      install (dist_cmd, td);
+
+      // Copy over all the files.
+      //
+      for (void* v: files)
+      {
+        file& t (*static_cast<file*> (v));
+
+        // Figure out where this file is inside the target directory.
+        //
+        dir_path d (td);
+        d /= t.dir.sub (src_root)
+          ? t.dir.leaf (src_root)
+          : t.dir.leaf (out_root);
+
+        if (!dir_exists (d))
+          install (dist_cmd, d);
+
+        install (dist_cmd, t, d);
+      }
+
+      // Archive if requested.
+      //
+      if (auto l = rs->vars["dist.archives"])
+      {
+        for (const string& e: as<strings> (*l))
+          archive (dist_root, dist_package, e);
+      }
+    }
+
+    // install -d <dir>
+    //
+    static void
+    install (const string& cmd, const dir_path& d)
+    {
+      path reld (relative (d));
+
+      cstrings args {cmd.c_str (), "-d"};
+
+      args.push_back ("-m");
+      args.push_back ("755");
+      args.push_back (reld.string ().c_str ());
+      args.push_back (nullptr);
+
+      if (verb)
+        print_process (args);
+      else
+        text << "dist -d " << d;
+
+      try
+      {
+        process pr (args.data ());
+
+        if (!pr.wait ())
+          throw failed ();
+      }
+      catch (const process_error& e)
+      {
+        error << "unable to execute " << args[0] << ": " << e.what ();
+
+        if (e.child ())
+          exit (1);
+
+        throw failed ();
+      }
+    }
+
+    // install <file> <dir>
+    //
+    static void
+    install (const string& cmd, file& t, const dir_path& d)
+    {
+      path reld (relative (d));
+      path relf (relative (t.path ()));
+
+      cstrings args {cmd.c_str ()};
+
+      // Preserve timestamps. This could becomes important if, for
+      // example, we have pre-generated sources. Note that the
+      // install-sh script doesn't support this option, while both
+      // Linux and BSD install's do.
+      //
+      args.push_back ("-p");
+
+      // Assume the file is executable if the owner has execute
+      // permission, in which case we make it executable for
+      // everyone.
+      //
+      args.push_back ("-m");
+      args.push_back (
+        (path_permissions (t.path ()) & permissions::xu) == permissions::xu
+        ? "755"
+        : "644");
+
+      args.push_back (relf.string ().c_str ());
+      args.push_back (reld.string ().c_str ());
+      args.push_back (nullptr);
+
+      if (verb)
+        print_process (args);
+      else
+        text << "dist " << t;
+
+      try
+      {
+        process pr (args.data ());
+
+        if (!pr.wait ())
+          throw failed ();
+      }
+      catch (const process_error& e)
+      {
+        error << "unable to execute " << args[0] << ": " << e.what ();
+
+        if (e.child ())
+          exit (1);
+
+        throw failed ();
+      }
+    }
+
+    static void
+    archive (const dir_path& root, const string& pkg, const string& e)
+    {
+      string a (pkg + '.' + e);
+
+      // Delete old archive for good measure.
+      //
+      path ap (root / path (a));
+      if (file_exists (ap))
+        rmfile (ap);
+
+      // Use zip for .zip archives. Everything else goes to tar in the
+      // auto-compress mode (-a).
+      //
+      cstrings args;
+      if (e == "zip")
+        args = {"zip", "-rq", a.c_str (), pkg.c_str (), nullptr};
+      else
+        args = {"tar", "-a", "-cf", a.c_str (), pkg.c_str (), nullptr};
+
+      if (verb)
+        print_process (args);
+      else
+        text << args[0] << " " << ap;
+
+      try
+      {
+        // Change child's working directory to dist_root.
+        //
+        process pr (root.string ().c_str (), args.data ());
+
+        if (!pr.wait ())
+          throw failed ();
+      }
+      catch (const process_error& e)
+      {
+        error << "unable to execute " << args[0] << ": " << e.what ();
+
+        if (e.child ())
+          exit (1);
+
+        throw failed ();
+      }
+    }
+
+    meta_operation_info dist {
+      "dist",
+      "distribute",
+      "distributing",
+      "has nothing to distribute", // We cannot "be distributed".
+      &dist_meta_operation_pre,
+      &dist_operation_pre,
+      &load,   // normal load
+      &search, // normal search
+      &dist_match,
+      &dist_execute,
+      nullptr, // operation post
+      nullptr  // meta-operation post
+    };
+  }
+}
diff --git a/build/dist/rule b/build/dist/rule
new file mode 100644
index 0000000..ffb2227
--- /dev/null
+++ b/build/dist/rule
@@ -0,0 +1,29 @@
+// file      : build/dist/rule -*- C++ -*-
+// copyright : Copyright (c) 2014-2015 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef BUILD_DIST_RULE
+#define BUILD_DIST_RULE
+
+#include <build/rule>
+#include <build/types>
+#include <build/target>
+#include <build/operation>
+
+namespace build
+{
+  namespace dist
+  {
+    class rule: public build::rule
+    {
+    public:
+      virtual match_result
+      match (action, target&, const std::string&) const;
+
+      virtual recipe
+      apply (action, target&, const match_result&) const;
+    };
+  }
+}
+
+#endif // BUILD_DIST_RULE
diff --git a/build/dist/rule.cxx b/build/dist/rule.cxx
new file mode 100644
index 0000000..bda2ef7
--- /dev/null
+++ b/build/dist/rule.cxx
@@ -0,0 +1,49 @@
+// file      : build/dist/rule.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2015 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <build/dist/rule>
+
+#include <build/scope>
+#include <build/target>
+#include <build/algorithm>
+#include <build/diagnostics>
+
+using namespace std;
+
+namespace build
+{
+  namespace dist
+  {
+    match_result rule::
+    match (action, target& t, const std::string&) const
+    {
+      return t; // We always match.
+    }
+
+    recipe rule::
+    apply (action a, target& t, const match_result&) const
+    {
+      const dir_path& out_root (t.root_scope ().path ());
+
+      for (prerequisite_member p: group_prerequisite_members (a, t))
+      {
+        // Skip prerequisites imported from other projects.
+        //
+        if (p.proj () != nullptr)
+          continue;
+
+        // @@ This is where we will handle dist/nodist.
+
+        target& pt (p.search ());
+
+        // Don't match targets that are outside our project.
+        //
+        if (pt.dir.sub (out_root))
+          build::match (a, pt);
+      }
+
+      return noop_recipe; // We will never be executed.
+    }
+  }
+}