Add support for installation manifest

5 files changed, 485 insertions, 19 deletions

diff --git a/libbuild2/install/init.cxx b/libbuild2/install/init.cxx
index d4b7f86..0f8b1be 100644
--- a/libbuild2/install/init.cxx
+++ b/libbuild2/install/init.cxx
@@ -459,6 +459,91 @@ namespace build2
           config::unsave_variable (rs, v);
         }
 
+        // config.install.manifest
+        //
+        // Installation manifest. Valid values are a file path or `-` to dump
+        // the manifest to stdout.
+        //
+        // If specified during the install operation, then write the
+        // information about all the filesystem entries being installed into
+        // the manifest. If specified during uninstall, then remove the
+        // filesystem entries according to the manifest as opposed to the
+        // current build state. In particular, this functionality can be used
+        // to avoid surprising (and potentially lengthy) updates during
+        // uninstall that may happen because of changes to system-installed
+        // dependencies (for example, the compiler or standard library).
+        //
+        // @@ TODO: manifest uninstall is still TODO.
+        //
+        // Note: there is a single manifest per operation and thus this
+        // variable can only be specified as a global override. (While it
+        // could be handy to save this varible in config.build in some
+        // situations, supporting this will complicate the global override
+        // case).
+        //
+        // Note also that the manifest is produced even in the dry-run mode.
+        // However, in this case no directory creation is tracked.
+        //
+        // The format of the installation manifest is "JSON lines", that is,
+        // each line is a JSON text (this makes it possible to reverse the
+        // order of lines without loading the entire file into memory). For
+        // example (indented lines indicate line continuations):
+        //
+        // {"type":"directory","path":"/tmp/install","mode":"755"}
+        // {"type":"target","name":"/tmp/libhello/libs{hello}",
+        //   "entries":[
+        //     {"type":"file","path":"/tmp/install/lib/libhello-1.0.so","mode":"755"},
+        //     {"type":"symlink","path":"/tmp/install/lib/libhello.so","target":"libhello-1.0.so"}]}
+        //
+        // Each line is a serialization of one of the following non-abstract
+        // C++ structs:
+        //
+        // struct entry // abstract
+        // {
+        //   enum {directory, file, symlink, target} type;
+        // };
+        //
+        // struct filesystem_entry: entry // abstract
+        // {
+        //   path path;
+        // };
+        //
+        // struct directory_entry: filesystem_entry
+        // {
+        //   string mode;
+        // };
+        //
+        // struct file_entry: filesystem_entry
+        // {
+        //   string mode;
+        // };
+        //
+        // struct symlink_entry: filesystem_entry
+        // {
+        //   path target;
+        // };
+        //
+        // struct target_entry: entry
+        // {
+        //   string name;
+        //   vector<filesystem_entry*> entries;
+        // };
+        //
+        {
+          auto& v (vp.insert<path> ("config.install.manifest"));
+
+          // If specified, verify it is a global override.
+          //
+          if (lookup l = rs[v])
+          {
+            if (!l.belongs (rs.global_scope ()))
+              fail << "config.install.manifest must be a global override" <<
+                info << "specify !config.install.manifest=...";
+          }
+
+          config::unsave_variable (rs, v);
+        }
+
         // Support for private install (aka poor man's Flatpack).
         //
         const dir_path* p;
diff --git a/libbuild2/install/operation.cxx b/libbuild2/install/operation.cxx
index 52e8c94..32da60f 100644
--- a/libbuild2/install/operation.cxx
+++ b/libbuild2/install/operation.cxx
@@ -3,6 +3,11 @@
 
 #include <libbuild2/install/operation.hxx>
 
+#include <sstream>
+
+#include <libbuild2/scope.hxx>
+#include <libbuild2/target.hxx>
+#include <libbuild2/context.hxx>
 #include <libbuild2/variable.hxx>
 
 using namespace std;
@@ -12,25 +17,300 @@ namespace build2
 {
   namespace install
   {
+#ifndef BUILD2_BOOTSTRAP
+    install_context_data::
+    install_context_data (const path* mf)
+        : manifest_file (mf),
+          manifest_os (mf != nullptr
+                       ? open_file_or_stdout (manifest_file, manifest_ofs)
+                       : manifest_ofs),
+          manifest_autorm (manifest_ofs.is_open () ? *mf : path ()),
+          manifest_json (manifest_os, 0 /* indentation */)
+    {
+    }
+
+    // Serialize current target and, if tgt is not NULL, start the new target.
+    //
+    // Note that we always serialize directories as top-level entries. And
+    // theoretically we can end up "splitting" a target with a directory
+    // creation. For example, if some files that belong to the target are
+    // installed into subdirectories that have not yet been created. So we
+    // have to cache the information for the current target in memory and only
+    // flush it once we see the next target (or the end).
+    //
+    // You may be wondering why not just serialize directories as target
+    // entries. While we could do that, it's not quite correct conceptually,
+    // since this would be the first of potentially many targets that caused
+    // the directory's creation. To put it another way, while files and
+    // symlinks belong to tragets, directories do not.
+    //
+    static void
+    manifest_flush_target (install_context_data& d, const target* tgt)
+    {
+      if (d.manifest_target != nullptr)
+      {
+        assert (!d.manifest_target_entries.empty ());
+
+        // Target name format is the same as in the structured result output.
+        //
+        ostringstream os;
+        stream_verb (os, stream_verbosity (1, 0));
+        os << *d.manifest_target;
+
+        try
+        {
+          auto& s (d.manifest_json);
+
+          s.begin_object ();
+          s.member ("type", "target");
+          s.member ("name", os.str ());
+          s.member_name ("entries");
+          s.begin_array ();
+
+          for (const auto& e: d.manifest_target_entries)
+          {
+            s.begin_object ();
+
+            if (e.target.empty ())
+            {
+              s.member ("type", "file");
+              s.member ("path", e.path);
+              s.member ("mode", e.mode);
+            }
+            else
+            {
+              s.member ("type", "symlink");
+              s.member ("path", e.path);
+              s.member ("target", e.target);
+            }
+
+            s.end_object ();
+          }
+
+          s.end_array ();  // entries member
+          s.end_object (); // target object
+        }
+        catch (const json::invalid_json_output& e)
+        {
+          fail << "invalid " << d.manifest_file << " json output: " << e;
+        }
+        catch (const io_error& e)
+        {
+          fail << "unable to write to " << d.manifest_file << ": " << e;
+        }
+
+        d.manifest_target_entries.clear ();
+      }
+
+      d.manifest_target = tgt;
+    }
+
+    void install_context_data::
+    manifest_install_d (context& ctx,
+                        const target& tgt,
+                        const dir_path& dir,
+                        const string& mode)
+    {
+      auto& d (
+        *static_cast<install_context_data*> (ctx.current_inner_odata.get ()));
+
+      if (d.manifest_file.path != nullptr)
+      {
+        try
+        {
+          auto& s (d.manifest_json);
+
+          // If we moved to the next target, flush the current one.
+          //
+          if (d.manifest_target != &tgt)
+            manifest_flush_target (d, nullptr);
+
+          s.begin_object ();
+          s.member ("type", "directory");
+          s.member ("path", dir.string ());
+          s.member ("mode", mode);
+          s.end_object ();
+        }
+        catch (const json::invalid_json_output& e)
+        {
+          fail << "invalid " << d.manifest_file << " json output: " << e;
+        }
+        catch (const io_error& e)
+        {
+          fail << "unable to write to " << d.manifest_file << ": " << e;
+        }
+      }
+    }
+
+    void install_context_data::
+    manifest_install_f (context& ctx,
+                        const target& tgt,
+                        const dir_path& dir,
+                        const path& name,
+                        const string& mode)
+    {
+      auto& d (
+        *static_cast<install_context_data*> (ctx.current_inner_odata.get ()));
+
+      if (d.manifest_file.path != nullptr)
+      {
+        if (d.manifest_target != &tgt)
+          manifest_flush_target (d, &tgt);
+
+        d.manifest_target_entries.push_back (
+          manifest_target_entry {(dir / name).string (), mode, ""});
+      }
+    }
+
+    void install_context_data::
+    manifest_install_l (context& ctx,
+                        const target& tgt,
+                        const path& link_target,
+                        const dir_path& dir,
+                        const path& link)
+    {
+      auto& d (
+        *static_cast<install_context_data*> (ctx.current_inner_odata.get ()));
+
+      if (d.manifest_file.path != nullptr)
+      {
+        if (d.manifest_target != &tgt)
+          manifest_flush_target (d, &tgt);
+
+        d.manifest_target_entries.push_back (
+          manifest_target_entry {
+            (dir / link).string (), "", link_target.string ()});
+      }
+    }
+
+    static void
+    manifest_close (context& ctx)
+    {
+      auto& d (
+        *static_cast<install_context_data*> (ctx.current_inner_odata.get ()));
+
+      if (d.manifest_file.path != nullptr)
+      {
+        try
+        {
+          manifest_flush_target (d, nullptr);
+
+          d.manifest_os << '\n'; // Final newline.
+
+          if (d.manifest_ofs.is_open ())
+          {
+            d.manifest_ofs.close ();
+            d.manifest_autorm.cancel ();
+          }
+        }
+        catch (const json::invalid_json_output& e)
+        {
+          fail << "invalid " << d.manifest_file << " json output: " << e;
+        }
+        catch (const io_error& e)
+        {
+          fail << "unable to write to " << d.manifest_file << ": " << e;
+        }
+      }
+    }
+#else
+    install_context_data::
+    install_context_data (const path*)
+    {
+    }
+
+    void install_context_data::
+    manifest_install_d (context&,
+                        const target&,
+                        const dir_path&,
+                        const string&)
+    {
+    }
+
+    void install_context_data::
+    manifest_install_f (context&,
+                        const target&,
+                        const dir_path&,
+                        const path&,
+                        const string&)
+    {
+    }
+
+    void install_context_data::
+    manifest_install_l (context&,
+                        const target&,
+                        const path&,
+                        const dir_path&,
+                        const path&)
+    {
+    }
+
+    static void
+    manifest_close (context&)
+    {
+    }
+#endif
+
     static operation_id
-    install_pre (context&,
-                 const values& params,
+    pre_install (context&,
+                 const values&,
                  meta_operation_id mo,
-                 const location& l)
+                 const location&)
     {
-      if (!params.empty ())
-        fail (l) << "unexpected parameters for operation install";
+      // Run update as a pre-operation, unless we are disfiguring.
+      //
+      return mo != disfigure_id ? update_id : 0;
+    }
 
+    static operation_id
+    pre_uninstall (context&,
+                   const values&,
+                   meta_operation_id mo,
+                   const location&)
+    {
       // Run update as a pre-operation, unless we are disfiguring.
       //
       return mo != disfigure_id ? update_id : 0;
     }
 
+    static void
+    install_pre (context& ctx,
+                 const values& params,
+                 bool inner,
+                 const location& l)
+    {
+      if (!params.empty ())
+        fail (l) << "unexpected parameters for operation install";
+
+      if (inner)
+      {
+        // See if we need to write the installation manifest.
+        //
+        // Note: go straight for the public variable pool.
+        //
+        const variable& var (*ctx.var_pool.find ("config.install.manifest"));
+        const path* mf (cast_null<path> (ctx.global_scope[var]));
+
+        ctx.current_inner_odata = context::current_data_ptr (
+          new install_context_data (mf),
+          [] (void* p) {delete static_cast<install_context_data*> (p);});
+      }
+    }
+
+    static void
+    install_post (context& ctx, const values&, bool inner)
+    {
+      if (inner)
+        manifest_close (ctx);
+    }
+
     // Note that we run both install and uninstall serially. The reason for
     // this is all the fuzzy things we are trying to do like removing empty
     // outer directories if they are empty. If we do this in parallel, then
     // those things get racy. Also, since all we do here is creating/removing
     // files, there is not going to be much speedup from doing it in parallel.
+    // There is also now the installation manifest, which relies on us
+    // installing all the filesystem entries of a target serially.
 
     const operation_info op_install {
       install_id,
@@ -42,8 +322,10 @@ namespace build2
       "has nothing to install", // We cannot "be installed".
       execution_mode::first,
       0 /* concurrency */,      // Run serially.
-      &install_pre,
+      &pre_install,
       nullptr,
+      &install_pre,
+      &install_post,
       nullptr,
       nullptr
     };
@@ -67,7 +349,9 @@ namespace build2
       "is not installed",
       execution_mode::last,
       0 /* concurrency */,      // Run serially
-      &install_pre,
+      &pre_uninstall,
+      nullptr,
+      nullptr,
       nullptr,
       nullptr,
       nullptr
@@ -87,6 +371,8 @@ namespace build2
       op_update.concurrency,
       op_update.pre_operation,
       op_update.post_operation,
+      op_update.operation_pre,
+      op_update.operation_post,
       op_update.adhoc_match,
       op_update.adhoc_apply
     };
diff --git a/libbuild2/install/operation.hxx b/libbuild2/install/operation.hxx
index c1f5416..71bdcba 100644
--- a/libbuild2/install/operation.hxx
+++ b/libbuild2/install/operation.hxx
@@ -7,7 +7,12 @@
 #include <libbuild2/types.hxx>
 #include <libbuild2/utility.hxx>
 
+#ifndef BUILD2_BOOTSTRAP
+#  include <libbutl/json/serializer.hxx>
+#endif
+
 #include <libbuild2/operation.hxx>
+#include <libbuild2/filesystem.hxx> // auto_rmfile
 
 namespace build2
 {
@@ -16,6 +21,59 @@ namespace build2
     extern const operation_info op_install;
     extern const operation_info op_uninstall;
     extern const operation_info op_update_for_install;
+
+    // Set as context::current_inner_odata during the install inner operation.
+    //
+    struct install_context_data
+    {
+#ifndef BUILD2_BOOTSTRAP
+      path_name manifest_file;
+      ofdstream manifest_ofs;
+      ostream&  manifest_os;
+      auto_rmfile manifest_autorm;
+      butl::json::stream_serializer manifest_json;
+      const target* manifest_target = nullptr; // Target being installed.
+      struct manifest_target_entry
+      {
+        string path;
+        string mode;
+        string target;
+      };
+      vector<manifest_target_entry> manifest_target_entries;
+#endif
+
+      explicit
+      install_context_data (const path* manifest);
+
+      // The following manifest_install_[dfl]() functions correspond to (and
+      // are called from) file_rule::install_[dfl]().
+
+      // install -d -m <mode> <dir>
+      //
+      static void
+      manifest_install_d (context&,
+                          const target&,
+                          const dir_path& dir,
+                          const string& mode);
+
+      // install -m <mode> <file> <dir>/<name>
+      //
+      static void
+      manifest_install_f (context&,
+                          const target& file,
+                          const dir_path& dir,
+                          const path& name,
+                          const string& mode);
+
+      // install -l <link_target> <dir>/<link>
+      //
+      static void
+      manifest_install_l (context&,
+                          const target&,
+                          const path& link_target,
+                          const dir_path& dir,
+                          const path& link);
+    };
   }
 }
 
diff --git a/libbuild2/install/rule.cxx b/libbuild2/install/rule.cxx
index 5ff4703..a3fa5ee 100644
--- a/libbuild2/install/rule.cxx
+++ b/libbuild2/install/rule.cxx
@@ -13,6 +13,8 @@
 #include <libbuild2/filesystem.hxx>
 #include <libbuild2/diagnostics.hxx>
 
+#include <libbuild2/install/operation.hxx>
+
 using namespace std;
 using namespace butl;
 
@@ -775,6 +777,7 @@ namespace build2
     install_d (const scope& rs,
                const install_dir& base,
                const dir_path& d,
+               const file& t,
                uint16_t verbosity)
     {
       context& ctx (rs.ctx);
@@ -789,6 +792,9 @@ namespace build2
       // with uninstall since the directories won't be empty (because we don't
       // actually uninstall any files).
       //
+      // Note that this also means we won't have the directory entries in the
+      // manifest created with dry-run. Probably not a big deal.
+      //
       if (ctx.dry_run)
         return;
 
@@ -816,7 +822,7 @@ namespace build2
         dir_path pd (d.directory ());
 
         if (pd != base.dir)
-          install_d (rs, base, pd, verbosity);
+          install_d (rs, base, pd, t, verbosity);
       }
 
       cstrings args;
@@ -853,6 +859,8 @@ namespace build2
       run (ctx,
            pp, args,
            verb >= verbosity ? 1 : verb_never /* finish_verbosity */);
+
+      install_context_data::manifest_install_d (ctx, t, d, *base.dir_mode);
     }
 
     void file_rule::
@@ -915,13 +923,21 @@ namespace build2
         run (ctx,
              pp, args,
              verb >= verbosity ? 1 : verb_never /* finish_verbosity */);
+
+      install_context_data::manifest_install_f (
+        ctx,
+        t,
+        base.dir,
+        name.empty () ? f.leaf () : name,
+        *base.mode);
     }
 
     void file_rule::
     install_l (const scope& rs,
                const install_dir& base,
-               const path& target,
                const path& link,
+               const file& target,
+               const path& link_target,
                uint16_t verbosity)
     {
       context& ctx (rs.ctx);
@@ -942,7 +958,7 @@ namespace build2
         base.sudo != nullptr ? base.sudo->c_str () : nullptr,
         "ln",
         "-sf",
-        target.string ().c_str (),
+        link_target.string ().c_str (),
         rell.string ().c_str (),
         nullptr};
 
@@ -960,7 +976,7 @@ namespace build2
           // a link. FreeBSD install(1) has the -l flag with the appropriate
           // semantics. For consistency, we also pass -d above.
           //
-          print_diag ("install -l", target, chd / link);
+          print_diag ("install -l", link_target, chd / link);
         }
       }
 
@@ -979,15 +995,15 @@ namespace build2
       if (verb >= verbosity)
       {
         if (verb >= 2)
-          text << "ln -sf " << target.string () << ' ' << rell.string ();
+          text << "ln -sf " << link_target.string () << ' ' << rell.string ();
         else if (verb)
-          print_diag ("install -l", target, chd / link);
+          print_diag ("install -l", link_target, chd / link);
       }
 
       if (!ctx.dry_run)
       try
       {
-        mkanylink (target, rell, true /* copy */);
+        mkanylink (link_target, rell, true /* copy */);
       }
       catch (const pair<entry_type, system_error>& e)
       {
@@ -999,6 +1015,13 @@ namespace build2
         fail << "unable to make " << w << ' ' << rell << ": " << e.second;
       }
 #endif
+
+      install_context_data::manifest_install_l (
+        ctx,
+        target,
+        link_target,
+        base.dir,
+        link);
     }
 
     target_state file_rule::
@@ -1047,7 +1070,7 @@ namespace build2
         // sudo, etc).
         //
         for (auto i (ids.begin ()), j (i); i != ids.end (); j = i++)
-          install_d (rs, *j, i->dir, verbosity); // install -d
+          install_d (rs, *j, i->dir, t, verbosity); // install -d
 
         install_dir& id (ids.back ());
 
@@ -1336,8 +1359,8 @@ namespace build2
     bool file_rule::
     uninstall_l (const scope& rs,
                  const install_dir& base,
-                 const path& /*target*/,
                  const path& link,
+                 const path& /*link_target*/,
                  uint16_t verbosity)
     {
       dir_path chd (chroot_path (rs, base.dir));
diff --git a/libbuild2/install/rule.hxx b/libbuild2/install/rule.hxx
index 98d2d0d..eb8addf 100644
--- a/libbuild2/install/rule.hxx
+++ b/libbuild2/install/rule.hxx
@@ -188,10 +188,16 @@ namespace build2
       //
       // install -d <dir>
       //
+      // Note: <dir> is expected to be absolute.
+      //
+      // Note that the target argument only specifies which target caused
+      // this directory to be created.
+      //
       static void
       install_d (const scope& rs,
                  const install_dir& base,
                  const dir_path& dir,
+                 const file& target,
                  uint16_t verbosity = 1);
 
       // Install a file:
@@ -209,13 +215,21 @@ namespace build2
 
       // Install (make) a symlink:
       //
-      // ln -s <target> <base>/<link>
+      // install -l <link_target> <base>/<link>
+      //
+      // Which is essentially:
+      //
+      // ln -s <link_target> <base>/<link>
+      //
+      // Note that the target argument only specifies which target this
+      // symlink "belongs" to.
       //
       static void
       install_l (const scope& rs,
                  const install_dir& base,
-                 const path& target,
                  const path& link,
+                 const file& target,
+                 const path& link_target,
                  uint16_t verbosity = 1);
 
       // Uninstall (remove) a file or symlink:
@@ -241,8 +255,8 @@ namespace build2
       static bool
       uninstall_l (const scope& rs,
                    const install_dir& base,
-                   const path& target,
                    const path& link,
+                   const path& link_target,
                    uint16_t verbosity = 1);