Implement brep-clean

7 files changed, 589 insertions, 0 deletions

diff --git a/clean/.gitignore b/clean/.gitignore
new file mode 100644
index 0000000..8e11bbd
--- /dev/null
+++ b/clean/.gitignore
@@ -0,0 +1,2 @@
+*-options.?xx
+brep-clean
diff --git a/clean/buildfile b/clean/buildfile
new file mode 100644
index 0000000..b4feada
--- /dev/null
+++ b/clean/buildfile
@@ -0,0 +1,30 @@
+# file      : clean/buildfile
+# copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+import libs  = libodb%lib{odb}
+import libs += libodb-pgsql%lib{odb-pgsql}
+import libs += libbutl%lib{butl}
+import libs += libbbot%lib{bbot}
+
+include ../libbrep/
+
+exe{brep-clean}: {hxx ixx cxx}{* -clean-options} \
+                 {hxx ixx cxx}{clean-options}    \
+                 ../libbrep/lib{brep} $libs
+
+# Generated options parser.
+#
+if $cli.configured
+{
+  cli.cxx{clean-options}: cli{clean}
+
+  cli.options += -I $src_root --include-with-brackets --include-prefix clean \
+--guard-prefix CLEAN --generate-specifier --page-usage print_ --ansi-color   \
+--cxx-prologue "#include <clean/types-parsers.hxx>"                          \
+--long-usage
+
+  # Include generated cli files into the distribution.
+  #
+  cli.cxx{clean-options}: dist = true
+}
diff --git a/clean/clean.cli b/clean/clean.cli
new file mode 100644
index 0000000..3f4e2ea
--- /dev/null
+++ b/clean/clean.cli
@@ -0,0 +1,164 @@
+// file      : clean/clean.cli
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+include <vector>;
+include <string>;
+include <cstdint>; // uint16_t
+
+include <clean/options-types.hxx>;
+
+"\section=1"
+"\name=brep-clean"
+"\summary=clean brep build database"
+
+{
+  "<options> <buildtab>",
+
+  "\h|SYNOPSIS|
+
+   \cb{brep-clean --help}\n
+   \cb{brep-clean --version}\n
+   \c{\b{brep-clean} [<options>] <buildtab>}
+
+   \h|DESCRIPTION|
+
+   \cb{brep-clean} deletes expired package builds from the brep \cb{build}
+   database. The build is considered expired if the package version is not
+   in the \cb{package} database, or the configuration is not listed in the
+   <buildtab> file, or the timestamp is older than the one specified for
+   this build's toolchain (see \cb{--stale-timeout}).
+
+   Note that \cb{brep-clean} expects the \cb{build} and \cb{package} database
+   schemas to have already been created using \l{brep-migrate(1)}."
+}
+
+class options
+{
+  "\h|OPTIONS|"
+
+  brep::toolchain_timeouts --stale-timeout
+  {
+    "[<name>=]<days>",
+    "Number of days to wait before considering builds for the named toolchain
+     as stale. Specify zero for <days> to make builds for a toolchain never
+     expire. Omit <name> (including \cb{=}) to specify the default timeout.
+     It will apply to all the toolchains that don't have a timeout specified
+     explicitly. If unspecified, the default timeout is zero (never expire)."
+  }
+
+  std::string --build-db-user
+  {
+    "<user>",
+    "Build database user name. If not specified, then operating system (login)
+     name is used."
+  }
+
+  std::string --build-db-password
+  {
+    "<pass>",
+    "Build database password. If not specified, then login without password is
+     expected to work."
+  }
+
+  std::string --build-db-name = "brep_build"
+  {
+    "<name>",
+    "Build database name. If not specified, then \cb{brep_build} is used by
+     default."
+  }
+
+  std::string --build-db-host
+  {
+    "<host>",
+    "Build database host name, address, or socket. If not specified, then
+     connect to \cb{localhost} using the operating system-default mechanism
+     (Unix-domain socket, etc)."
+  }
+
+  std::uint16_t --build-db-port = 0
+  {
+    "<port>",
+    "Build database port number. If not specified, the default port is used."
+  }
+
+  std::string --package-db-user
+  {
+    "<user>",
+    "Package database user name. If not specified, then operating system
+     (login) name is used."
+  }
+
+  std::string --package-db-password
+  {
+    "<pass>",
+    "Package database password. If not specified, then login without password
+     is expected to work."
+  }
+
+  std::string --package-db-name = "brep_package"
+  {
+    "<name>",
+    "Package database name. If not specified, then \cb{brep_package} is used by
+     default."
+  }
+
+  std::string --package-db-host
+  {
+    "<host>",
+    "Package database host name, address, or socket. If not specified, then
+     connect to \cb{localhost} using the operating system-default mechanism
+     (Unix-domain socket, etc)."
+  }
+
+  std::uint16_t --package-db-port = 0
+  {
+    "<port>",
+    "Package database port number. If not specified, the default port is used."
+  }
+
+  std::string --pager // String to allow empty value.
+  {
+    "<path>",
+    "The pager program to be used to show long text. Commonly used pager
+     programs are \cb{less} and \cb{more}. You can also specify additional
+     options that should be passed to the pager program with
+     \cb{--pager-option}. If an empty string is specified as the pager
+     program, then no pager will be used. If the pager program is not
+     explicitly specified, then \cb{brep-clean} will try to use \cb{less}.
+     If it is not available, then no pager will be used."
+  }
+
+  std::vector<std::string> --pager-option
+  {
+    "<opt>",
+    "Additional option to be passed to the pager program. See \cb{--pager}
+     for more information on the pager program. Repeat this option to
+     specify multiple pager options."
+  }
+
+  bool --help {"Print usage information and exit."}
+  bool --version {"Print version and exit."}
+};
+
+"\h|EXIT STATUS|
+
+\dl|
+
+\li|\cb{0}
+
+Success.|
+
+\li|\cb{1}
+
+Fatal error.|
+
+\li|\cb{2}
+
+An instance of \cb{brep-clean} or \l{brep-migrate(1)} is already running. Try
+again.|
+
+\li|\cb{3}
+
+Recoverable database error. Try again.||
+"
diff --git a/clean/clean.cxx b/clean/clean.cxx
new file mode 100644
index 0000000..fbb4d23
--- /dev/null
+++ b/clean/clean.cxx
@@ -0,0 +1,287 @@
+// file      : clean/clean.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <set>
+#include <iostream>
+
+#include <odb/database.hxx>
+#include <odb/transaction.hxx>
+#include <odb/schema-catalog.hxx>
+
+#include <odb/pgsql/database.hxx>
+
+#include <libbutl/pager.hxx>
+
+#include <libbbot/build-config.hxx>
+
+#include <libbrep/build.hxx>
+#include <libbrep/build-odb.hxx>
+#include <libbrep/package.hxx>
+#include <libbrep/package-odb.hxx>
+#include <libbrep/database-lock.hxx>
+
+#include <clean/clean-options.hxx>
+
+using namespace std;
+using namespace bbot;
+using namespace brep;
+using namespace odb::core;
+
+// Operation failed, diagnostics has already been issued.
+//
+struct failed {};
+
+static const char* help_info (
+  "  info: run 'brep-clean --help' for more information");
+
+int
+main (int argc, char* argv[])
+try
+{
+  cli::argv_scanner scan (argc, argv, true);
+  options ops (scan);
+
+  // Version.
+  //
+  if (ops.version ())
+  {
+    cout << "brep-clean " << BREP_VERSION_ID << endl
+         << "libbrep " << LIBBREP_VERSION_ID << endl
+         << "libbbot " << LIBBBOT_VERSION_ID << endl
+         << "libbpkg " << LIBBPKG_VERSION_ID << endl
+         << "libbutl " << LIBBUTL_VERSION_ID << endl
+         << "Copyright (c) 2014-2017 Code Synthesis Ltd" << endl
+         << "This is free software released under the MIT license." << endl;
+
+    return 0;
+  }
+
+  // Help.
+  //
+  if (ops.help ())
+  {
+    butl::pager p ("brep-clean help",
+                   false,
+                   ops.pager_specified () ? &ops.pager () : nullptr,
+                   &ops.pager_option ());
+
+    print_usage (p.stream ());
+
+    // If the pager failed, assume it has issued some diagnostics.
+    //
+    return p.wait () ? 0 : 1;
+  }
+
+  const toolchain_timeouts& timeouts (ops.stale_timeout ());
+
+  auto i (timeouts.find (string ()));
+  timestamp default_timeout (i != timeouts.end ()
+                             ? i->second
+                             : timestamp_nonexistent);
+
+  // Load configurations names.
+  //
+  if (!scan.more ())
+  {
+    cerr << "error: configuration file expected" << endl
+         << help_info << endl;
+    return 1;
+  }
+
+  set<string> configs;
+  for (auto& c: parse_buildtab (path (scan.next ())))
+    configs.emplace (move (c.name));
+
+  if (scan.more ())
+  {
+    cerr << "error: unexpected argument encountered" << endl
+         << help_info << endl;
+    return 1;
+  }
+
+  odb::pgsql::database build_db (
+    ops.build_db_user (),
+    ops.build_db_password (),
+    ops.build_db_name (),
+    ops.build_db_host (),
+    ops.build_db_port (),
+    "options='-c default_transaction_isolation=serializable'");
+
+  odb::pgsql::database package_db (
+    ops.package_db_user (),
+    ops.package_db_password (),
+    ops.package_db_name (),
+    ops.package_db_host (),
+    ops.package_db_port (),
+    "options='-c default_transaction_isolation=serializable'");
+
+  // Prevent several brep-clean/migrate instances from updating build database
+  // simultaneously.
+  //
+  database_lock l (build_db);
+
+  // Check that the build and package database schemas match the current ones.
+  //
+  const string bs ("build");
+  if (schema_catalog::current_version (build_db, bs) !=
+      build_db.schema_version (bs))
+  {
+    cerr << "error: build database schema differs from the current one"
+         << endl << "  info: use brep-migrate to migrate the database" << endl;
+    return 1;
+  }
+
+  const string ps ("package");
+  if (schema_catalog::current_version (package_db, ps) !=
+      package_db.schema_version (ps))
+  {
+    cerr << "error: package database schema differs from the current one"
+         << endl << "  info: use brep-migrate to migrate the database" << endl;
+    return 1;
+  }
+
+  // Prepare the build prepared query.
+  //
+  // Query package builds in chunks in order not to hold locks for too long.
+  // Sort the result by package version to minimize number of queries to the
+  // package database.
+  //
+  using bld_query = query<build>;
+  using prep_bld_query = prepared_query<build>;
+
+  size_t offset (0);
+  bld_query bq ("ORDER BY" + bld_query::id.package.name +
+                order_by_version_desc (bld_query::id.package.version, false) +
+                "OFFSET" + bld_query::_ref (offset) + "LIMIT 100");
+
+  connection_ptr bld_conn (build_db.connection ());
+
+  prep_bld_query bld_prep_query (
+    bld_conn->prepare_query<build> ("build-query", bq));
+
+  // Prepare the package version query.
+  //
+  // Query package versions every time the new package name is encountered
+  // during iterating over the package builds. Such a query will be made once
+  // per package name due to the builds query sorting criteria (see above).
+  //
+  using pkg_query = query<package_version>;
+  using prep_pkg_query = prepared_query<package_version>;
+
+  string package_name;
+  set<version> package_versions;
+
+  pkg_query pq (pkg_query::package::id.name == pkg_query::_ref (package_name));
+
+  connection_ptr pkg_conn (package_db.connection ());
+
+  prep_pkg_query pkg_prep_query (
+    pkg_conn->prepare_query<package_version> ("package-version-query", pq));
+
+  while (true)
+  {
+    // Start the build database transaction.
+    //
+    transaction bt (bld_conn->begin ());
+
+    // Query builds.
+    //
+    auto builds (bld_prep_query.execute ());
+
+    if (!builds.empty ())
+    {
+      // Start the package database transaction.
+      //
+      transaction pt (pkg_conn->begin (), false);
+
+      for (const auto& b: builds)
+      {
+        auto i (timeouts.find (b.toolchain_name));
+
+        timestamp et (i != timeouts.end ()
+                      ? i->second
+                      : default_timeout);
+
+        bool cleanup (
+          // Check that the build is not stale.
+          //
+          b.timestamp <= et ||
+
+          // Check that the build configuration is still present.
+          //
+          // Note that we unable to detect configuration changes and rely on
+          // periodic rebuilds to take care of that.
+          //
+          configs.find (b.configuration) == configs.end ());
+
+        // Check that the build package still exists.
+        //
+        if (!cleanup)
+        {
+          if (package_name != b.package_name)
+          {
+            // Switch to the package database transaction.
+            //
+            transaction::current (pt);
+
+            package_name = b.package_name;
+            package_versions.clear ();
+
+            for (auto& v: pkg_prep_query.execute ())
+              package_versions.emplace (move (v.version));
+
+            // Switch back to the build database transaction.
+            //
+            transaction::current (bt);
+          }
+
+          cleanup = package_versions.find (b.package_version) ==
+            package_versions.end ();
+        }
+
+        if (cleanup)
+          build_db.erase (b);
+        else
+          ++offset;
+      }
+
+      // Commit the package database transaction.
+      //
+      pt.commit ();
+    }
+
+    bt.commit ();
+
+    if (builds.empty ())
+      break;
+  }
+
+  return 0;
+}
+catch (const database_locked&)
+{
+  cerr << "brep-clean or brep-migrate is running" << endl;
+  return 2;
+}
+catch (const recoverable& e)
+{
+  cerr << "recoverable database error: " << e << endl;
+  return 3;
+}
+catch (const cli::exception& e)
+{
+  cerr << "error: " << e << endl << help_info << endl;
+  return 1;
+}
+catch (const failed&)
+{
+  return 1; // Diagnostics has already been issued.
+}
+// Fully qualified to avoid ambiguity with odb exception.
+//
+catch (const std::exception& e)
+{
+  cerr << "error: " << e << endl;
+  return 1;
+}
diff --git a/clean/options-types.hxx b/clean/options-types.hxx
new file mode 100644
index 0000000..183c0df
--- /dev/null
+++ b/clean/options-types.hxx
@@ -0,0 +1,18 @@
+// file      : clean/options-types.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef CLEAN_OPTIONS_TYPES_HXX
+#define CLEAN_OPTIONS_TYPES_HXX
+
+#include <map>
+
+#include <libbrep/types.hxx>
+#include <libbrep/utility.hxx>
+
+namespace brep
+{
+  struct toolchain_timeouts: std::map<string, timestamp> {};
+}
+
+#endif // CLEAN_OPTIONS_TYPES_HXX
diff --git a/clean/types-parsers.cxx b/clean/types-parsers.cxx
new file mode 100644
index 0000000..26b9a02
--- /dev/null
+++ b/clean/types-parsers.cxx
@@ -0,0 +1,60 @@
+// file      : clean/types-parsers.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <chrono>
+#include <string> // strtoull()
+
+#include <clean/types-parsers.hxx>
+
+#include <clean/options-types.hxx>
+#include <clean/clean-options.hxx> // cli namespace
+
+using namespace std;
+using namespace brep;
+
+namespace cli
+{
+  void parser<toolchain_timeouts>::
+  parse (toolchain_timeouts& x, bool& xs, scanner& s)
+  {
+    const char* o (s.next ());
+
+    if (!s.more ())
+      throw missing_value (o);
+
+    string ov (s.next ());
+    size_t p (ov.find ('='));
+
+    timestamp now (timestamp::clock::now ());
+
+    // Convert timeout duration into the time point.
+    //
+    auto timeout = [o, &ov, &now] (const string& tm) -> timestamp
+    {
+      char* e (nullptr);
+      uint64_t t (strtoull (tm.c_str (), &e, 10));
+
+      if (*e != '\0' || tm.empty ())
+        throw invalid_value (o, ov);
+
+      if (t == 0)
+        return timestamp_nonexistent;
+
+      return now - chrono::duration<uint64_t, ratio<86400>> (t);
+    };
+
+    if (p == string::npos)
+      x[string ()] = timeout (ov); // Default timeout.
+    else
+    {
+      string k (ov, 0, p);
+      if (k.empty ())
+        throw invalid_value (o, ov);
+
+      x[k] = timeout (string (ov, p + 1));
+    }
+
+    xs = true;
+  }
+}
diff --git a/clean/types-parsers.hxx b/clean/types-parsers.hxx
new file mode 100644
index 0000000..46fd483
--- /dev/null
+++ b/clean/types-parsers.hxx
@@ -0,0 +1,28 @@
+// file      : clean/types-parsers.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2017 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+// CLI parsers, included into the generated source files.
+//
+
+#ifndef CLEAN_TYPES_PARSERS_HXX
+#define CLEAN_TYPES_PARSERS_HXX
+
+#include <clean/options-types.hxx>
+
+namespace cli
+{
+  class scanner;
+
+  template <typename T>
+  struct parser;
+
+  template <>
+  struct parser<brep::toolchain_timeouts>
+  {
+    static void
+    parse (brep::toolchain_timeouts&, bool&, scanner&);
+  };
+}
+
+#endif // CLEAN_TYPES_PARSERS_HXX