Add default options loading and merging API

5 files changed, 583 insertions, 0 deletions

diff --git a/libbutl/default-options.mxx b/libbutl/default-options.mxx
new file mode 100644
index 0000000..01d32c1
--- /dev/null
+++ b/libbutl/default-options.mxx
@@ -0,0 +1,115 @@
+// file      : libbutl/default-options.mxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef __cpp_modules_ts
+#pragma once
+#endif
+
+#ifndef __cpp_lib_modules_ts
+#include <utility> // move()
+#endif
+
+// Other includes.
+
+#ifdef __cpp_modules_ts
+export module butl.default_options;
+#ifdef __cpp_lib_modules_ts
+import std.core;
+#endif
+import butl.path;
+import butl.optional;
+import butl.small_vector;
+
+import butl.git;
+import butl.filesystem;
+#else
+#include <libbutl/path.mxx>
+#include <libbutl/optional.mxx>
+#include <libbutl/small-vector.mxx>
+
+#include <libbutl/git.mxx>
+#include <libbutl/filesystem.mxx>
+#endif
+
+#include <libbutl/export.hxx>
+
+LIBBUTL_MODEXPORT namespace butl
+{
+  // Default options files helper implementation.
+  //
+  struct default_options_files
+  {
+    small_vector<path, 2> files;
+    optional<dir_path>    start_dir;
+  };
+
+  template <typename O>
+  struct default_options_entry
+  {
+    path file;
+    O    options;
+    bool remote;
+  };
+
+  template <typename O>
+  using default_options = small_vector<default_options_entry<O>, 4>;
+
+  // Search for and load (using scanner S and parsing in the U::fail mode for
+  // both options and arguments) the specified list of options files in the
+  // specified directories returning a vector of option class instances (O).
+  // Throw std::system_error on the underlying OS error and pass through
+  // exceptions thrown by the options scanner/parser.
+  //
+  // Search order:
+  //
+  // - sys_dir
+  // - home_dir
+  // - start_dir and outer until home_dir or root (both excluding)
+  //
+  // Except for sys_dir, the options files are looked for in the .build2/ and
+  // .build2/local/ subdirectories of each directory. For sys_dir they are
+  // looked for in the directory itself (e.g., /etc/build2/).
+  //
+  // Note that all the directories should be absolute and normalized.
+  //
+  // The presence of the .git filesystem entry causes the options files in
+  // this directory and any of its subdirectories to be considered remote
+  // (note that in the current implementation this is the case even for files
+  // from the .build2/local/ subdirectory since the mere location is not a
+  // sufficient ground to definititevly conclude that the file is not remote;
+  // to be sure we would need to query the VCS or some such).
+  //
+  template <typename O, typename S, typename U>
+  default_options<O>
+  load_default_options (const optional<dir_path>& sys_dir,
+                        const optional<dir_path>& home_dir,
+                        const default_options_files&);
+
+  // Merge the default options and the command line options.
+  //
+  // Note that this is the default implementation and in some cases you may
+  // want to provide an options class-specific version that verifies/sanitizes
+  // the default options (e.g., you may not want to allow certain options to
+  // be specified in the default options files) or warns/prompts about
+  // potentially dangerous options if they came from the remote options files.
+  //
+  template <typename O>
+  O
+  merge_default_options (const default_options<O>&, const O& cmd_ops);
+
+  // As above but pass each default option to the specified function prior to
+  // merging. The function signature is:
+  //
+  // void (const default_options_entry<O>&, const O& cmd_ops)
+  //
+  // This version can be used to verify the default options. For example, you
+  // may want to disallow certain options from being specified in the default
+  // options files.
+  //
+  template <typename O, typename F>
+  O
+  merge_default_options (const default_options<O>&, const O&, F&&);
+}
+
+#include <libbutl/default-options.txx>
diff --git a/libbutl/default-options.txx b/libbutl/default-options.txx
new file mode 100644
index 0000000..42dd585
--- /dev/null
+++ b/libbutl/default-options.txx
@@ -0,0 +1,166 @@
+// file      : libbutl/default-options.txx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+LIBBUTL_MODEXPORT namespace butl //@@ MOD Clang needs this for some reason.
+{
+  // Search for and parse the options files in the specified directory and
+  // its local/ subdirectory, if exists, and append the options to the
+  // resulting list.
+  //
+  // Note that we check for the local/ subdirectory even if we don't think it
+  // belongs to the remote directory; the user may move things around or it
+  // could be a VCS we don't yet recognize and there doesn't seem to be any
+  // harm in doing so.
+  //
+  // Also note that if the directory is remote, then for now the options files
+  // in both the directory itself and its local/ subdirectory are considered
+  // remote (see load_default_options() for details).
+  //
+  template <typename O, typename S, typename U>
+  void
+  load_default_options_files (const dir_path& d,
+                              bool remote,
+                              const small_vector<path, 2>& fs,
+                              default_options<O>& r)
+  {
+    auto load = [&fs, &r] (const dir_path& d, bool remote)
+    {
+      using namespace std;
+
+      for (const path& f: fs)
+      {
+        path p (d / f);
+
+        if (file_exists (p)) // Follows symlinks.
+        {
+          S s (p.string ());
+
+          // @@ Note that the potentially thrown exceptions (unknown option,
+          //    unexpected argument, etc) will not contain any location
+          //    information. Intercepting exception handling to add the file
+          //    attribution feels too hairy for now. Maybe we should support
+          //    this in CLI.
+          //
+          O o;
+          o.parse (s, U::fail, U::fail);
+
+          r.push_back (default_options_entry<O> {move (p), move (o), remote});
+        }
+      }
+    };
+
+    load (d, remote);
+
+    dir_path ld (d / dir_path ("local"));
+
+    if (dir_exists (ld))
+      load (ld, remote);
+  }
+
+  // Search for and parse the options files in the specified and outer
+  // directories until root/home directory (excluding) and append the options
+  // to the resulting list. Return true if the directory is "remote" (i.e.,
+  // belongs to a VCS repository).
+  //
+  template <typename O, typename S, typename U>
+  bool
+  load_default_options_files (const dir_path& start_dir,
+                              const optional<dir_path>& home_dir,
+                              const small_vector<path, 2>& fs,
+                              default_options<O>& r)
+  {
+    if (start_dir.root () || (home_dir && start_dir == *home_dir))
+      return false;
+
+    bool remote (load_default_options_files<O, S, U> (start_dir.directory (),
+                                                     home_dir,
+                                                     fs,
+                                                     r) ||
+                 git_repository (start_dir));
+
+    dir_path d (start_dir / dir_path (".build2"));
+
+    if (dir_exists (d))
+      load_default_options_files<O, S, U> (d, remote, fs, r);
+
+    return remote;
+  }
+
+  template <typename O, typename S, typename U>
+  default_options<O>
+  load_default_options (const optional<dir_path>& sys_dir,
+                        const optional<dir_path>& home_dir,
+                        const default_options_files& ofs)
+  {
+    default_options<O> r;
+
+    if (sys_dir)
+    {
+      assert (sys_dir->absolute () && sys_dir->normalized ());
+
+      if (dir_exists (*sys_dir))
+        load_default_options_files<O, S, U> (*sys_dir,
+                                             false /* remote */,
+                                             ofs.files,
+                                             r);
+    }
+
+    if (home_dir)
+    {
+      assert (home_dir->absolute () && home_dir->normalized ());
+
+      dir_path d (*home_dir / dir_path (".build2"));
+
+      if (dir_exists (d))
+        load_default_options_files<O, S, U> (d,
+                                             false /* remote */,
+                                             ofs.files,
+                                             r);
+    }
+
+    if (ofs.start_dir)
+    {
+      assert (ofs.start_dir->absolute () && ofs.start_dir->normalized ());
+
+      load_default_options_files<O, S, U> (*ofs.start_dir,
+                                           home_dir,
+                                           ofs.files,
+                                           r);
+    }
+
+    return r;
+  }
+
+  template <typename O, typename F>
+  O
+  merge_default_options (const default_options<O>& def_ops,
+                         const O& cmd_ops,
+                         F&& f)
+  {
+    // Optimize for the common case.
+    //
+    if (def_ops.empty ())
+      return cmd_ops;
+
+    O r;
+    for (const default_options_entry<O>& e: def_ops)
+    {
+      f (e, cmd_ops);
+      r.merge (e.options);
+    }
+
+    r.merge (cmd_ops);
+    return r;
+  }
+
+  template <typename O>
+  inline O
+  merge_default_options (const default_options<O>& def_ops, const O& cmd_ops)
+  {
+    return merge_default_options (
+      def_ops,
+      cmd_ops,
+      [] (const default_options_entry<O>&, const O&) {});
+  }
+}
diff --git a/tests/default-options/buildfile b/tests/default-options/buildfile
new file mode 100644
index 0000000..5530a10
--- /dev/null
+++ b/tests/default-options/buildfile
@@ -0,0 +1,7 @@
+# file      : tests/default-options/buildfile
+# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+import libs = libbutl%lib{butl}
+
+exe{driver}: {hxx cxx}{*} $libs testscript
diff --git a/tests/default-options/driver.cxx b/tests/default-options/driver.cxx
new file mode 100644
index 0000000..a2ed43d
--- /dev/null
+++ b/tests/default-options/driver.cxx
@@ -0,0 +1,186 @@
+// file      : tests/default-options/driver.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <cassert>
+
+#ifndef __cpp_lib_modules_ts
+#include <string>
+#include <vector>
+#include <iostream>
+#endif
+
+// Other includes.
+
+#ifdef __cpp_modules_ts
+#ifdef __cpp_lib_modules_ts
+import std.core;
+import std.io;
+#endif
+import butl.path;
+import butl.path_io;
+import butl.optional;
+import butl.fdstream;
+import butl.default_options;
+#else
+#include <libbutl/path.mxx>
+#include <libbutl/path-io.mxx>
+#include <libbutl/utility.mxx>         // eof()
+#include <libbutl/optional.mxx>
+#include <libbutl/fdstream.mxx>
+#include <libbutl/default-options.mxx>
+#endif
+
+using namespace std;
+using namespace butl;
+
+// Usage: argv[0] [-f <file>] [-d <start-dir>] [-s <sys-dir>] [-h <home-dir>]
+//                [-e] <cmd-options>
+//
+// Parse default options files, merge them with the command line options, and
+// print the resulting options to STDOUT one per line. Note that the options
+// instance is a vector of arbitrary strings.
+//
+// -f
+//    Default options file name. Can be specified multiple times.
+//
+// -d
+//    Directory to start the default options files search from.
+//
+// -s
+//    System directory.
+//
+// -h
+//    Home directory.
+//
+// -e
+//    Print the default options entries (rather than the merged options) to
+//    STDOUT one per line in the following format:
+//
+//    <file>,<space-separated-options>,<remote>
+//
+int
+main (int argc, const char* argv[])
+{
+  using butl::optional;
+
+  class scanner
+  {
+  public:
+    scanner (const string& f): ifs_ (f, fdopen_mode::in, ifdstream::badbit) {}
+
+    optional<string>
+    next ()
+    {
+      string s;
+      return !eof (getline (ifs_, s)) ? optional<string> (move (s)) : nullopt;
+    }
+
+  private:
+    ifdstream ifs_;
+  };
+
+  enum class unknow_mode
+  {
+    fail
+  };
+
+  class options: public vector<string>
+  {
+  public:
+    bool
+    parse (scanner& s, unknow_mode, unknow_mode)
+    {
+      bool r (false);
+      while (optional<string> o = s.next ())
+      {
+        push_back (move (*o));
+        r = true;
+      }
+      return r;
+    }
+
+    void
+    merge (const options& o)
+    {
+      insert (end (), o.begin (), o.end ());
+    }
+  };
+
+  // Parse and validate the arguments.
+  //
+  default_options_files fs;
+  optional<dir_path> sys_dir;
+  optional<dir_path> home_dir;
+  options cmd_ops;
+  bool print_entries (false);
+
+  for (int i (1); i != argc; ++i)
+  {
+    string op (argv[i]);
+
+    if (op == "-f")
+    {
+      assert (++i != argc);
+      fs.files.push_back (path (argv[i]));
+    }
+    else if (op == "-d")
+    {
+      assert (++i != argc);
+      fs.start_dir = dir_path (argv[i]);
+    }
+    else if (op == "-s")
+    {
+      assert (++i != argc);
+      sys_dir = dir_path (argv[i]);
+    }
+    else if (op == "-h")
+    {
+      assert (++i != argc);
+      home_dir = dir_path (argv[i]);
+    }
+    else if (op == "-e")
+    {
+      print_entries = true;
+    }
+    else
+      cmd_ops.push_back (argv[i]);
+  }
+
+  // Load and print the default options.
+  //
+  default_options<options> def_ops (
+    load_default_options<options, scanner, unknow_mode> (sys_dir,
+                                                         home_dir,
+                                                         fs));
+
+  if (print_entries)
+  {
+    for (const default_options_entry<options>& e: def_ops)
+    {
+      cout << e.file << ',';
+
+      for (const string& o: e.options)
+      {
+        if (&o != &e.options[0])
+          cout << ' ';
+
+        cout << o;
+      }
+
+      cout << (e.remote ? ",true" : ",false") << endl;
+    }
+  }
+
+  // Merge the options and print the result.
+  //
+  options ops (merge_default_options (def_ops, cmd_ops));
+
+  if (!print_entries)
+  {
+    for (const string& o: ops)
+      cout << o << endl;
+  }
+
+  return 0;
+}
diff --git a/tests/default-options/testscript b/tests/default-options/testscript
new file mode 100644
index 0000000..c0c3816
--- /dev/null
+++ b/tests/default-options/testscript
@@ -0,0 +1,109 @@
+# file      : tests/default-options/testscript
+# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+# Note that when cross-testing the driver may not be able to run the command
+# due to the meaningless program path.
+#
++if ($test.target != $build.host)
+  exit
+end
+
+sys_dir = $canonicalize([dir_path] $~/build2)
++mkdir -p $sys_dir/local
+
++echo 'sys-foo' >=$sys_dir/foo
++echo 'sys-bar' >=$sys_dir/bar
++echo 'sys-local-foo' >=$sys_dir/local/foo
++echo 'sys-local-bar' >=$sys_dir/local/bar
+
+home_dir = $canonicalize([dir_path] $~/home)
++mkdir -p $home_dir/.build2/local/
+
++echo 'home-foo'       >=$home_dir/.build2/foo
++echo 'home-bar'       >=$home_dir/.build2/bar
++echo 'home-local-foo' >=$home_dir/.build2/local/foo
++echo 'home-local-bar' >=$home_dir/.build2/local/bar
+
+: in-home
+:
+{
+  d = $home_dir/work/.build2
+  +mkdir -p $d/local/
+
+  +echo 'work-foo'       >=$d/foo
+  +echo 'work-bar'       >=$d/bar
+  +echo 'work-local-foo' >=$d/local/foo
+  +echo 'work-local-bar' >=$d/local/bar
+
+  d = $home_dir/work/project/.build2
+  +mkdir -p $d/local/
+  +touch $home_dir/work/project/.git
+
+  +echo 'project-foo'       >=$d/foo
+  +echo 'project-bar'       >=$d/bar
+  +echo 'project-local-foo' >=$d/local/foo
+  +echo 'project-local-bar' >=$d/local/bar
+
+  d = $home_dir/work/project/package/.build2
+  +mkdir -p $d/local/
+
+  +echo 'package-foo'       >=$d/foo
+  +echo 'package-bar'       >=$d/bar
+  +echo 'package-local-foo' >=$d/local/foo
+  +echo 'package-local-bar' >=$d/local/bar
+
+  start_dir = $canonicalize([dir_path] $home_dir/work/project/package)
+
+  : entries
+  :
+  $* -e -f foo -f bar -d $start_dir -s $sys_dir -h $home_dir cmd-foo cmd-bar >>/~%EOO%d
+    %\.+/build2/foo,sys-foo,false%
+    %\.+/build2/bar,sys-bar,false%
+    %\.+/build2/local/foo,sys-local-foo,false%
+    %\.+/build2/local/bar,sys-local-bar,false%
+    %\.+/home/.build2/foo,home-foo,false%
+    %\.+/home/.build2/bar,home-bar,false%
+    %\.+/home/.build2/local/foo,home-local-foo,false%
+    %\.+/home/.build2/local/bar,home-local-bar,false%
+    %\.+/home/work/.build2/foo,work-foo,false%
+    %\.+/home/work/.build2/bar,work-bar,false%
+    %\.+/home/work/.build2/local/foo,work-local-foo,false%
+    %\.+/home/work/.build2/local/bar,work-local-bar,false%
+    %\.+/home/work/project/.build2/foo,project-foo,true%
+    %\.+/home/work/project/.build2/bar,project-bar,true%
+    %\.+/home/work/project/.build2/local/foo,project-local-foo,true%
+    %\.+/home/work/project/.build2/local/bar,project-local-bar,true%
+    %\.+/home/work/project/package/.build2/foo,package-foo,true%
+    %\.+/home/work/project/package/.build2/bar,package-bar,true%
+    %\.+/home/work/project/package/.build2/local/foo,package-local-foo,true%
+    %\.+/home/work/project/package/.build2/local/bar,package-local-bar,true%
+    EOO
+
+  : merged
+  :
+  $* -f foo -f bar -d $start_dir -s $sys_dir -h $home_dir cmd-foo cmd-bar >>EOO
+    sys-foo
+    sys-bar
+    sys-local-foo
+    sys-local-bar
+    home-foo
+    home-bar
+    home-local-foo
+    home-local-bar
+    work-foo
+    work-bar
+    work-local-foo
+    work-local-bar
+    project-foo
+    project-bar
+    project-local-foo
+    project-local-bar
+    package-foo
+    package-bar
+    package-local-foo
+    package-local-bar
+    cmd-foo
+    cmd-bar
+    EOO
+}