Add basic support for CI request handling

29 files changed, 1544 insertions, 336 deletions

diff --git a/INSTALL b/INSTALL
index 8c679ca..36b3465 100644
--- a/INSTALL
+++ b/INSTALL
@@ -4,8 +4,9 @@ you are using a systemd-based distribution. If not, then you will need to
 replace systemctl commands with the equivalent init.d ones.
 
 The below instructions include steps for setting up brep as the build2 build
-bot controller and package submission service. Both of these functionalities
-are optional and, if not needed, then the corresponding steps can be omitted.
+bot controller, package submission, and CI request services. All these
+functionalities are optional and, if not needed, then the corresponding steps
+can be omitted.
 
 1. Create 'brep' User
 
@@ -240,7 +241,26 @@ example:
 $ cp install/share/brep/www/submit.xhtml config/
 $ edit config/submit.xhtml # Add custom form fields, adjust CSS style, etc.
 
-For sample submission handler implementations see brep/submit/.
+For sample submission handler implementations see brep/handler/submit/.
+
+To enable the CI request functionality you will need to specify the ci-data
+directory in brep-module.conf. Note that this directory must exist and have
+read, write, and execute permissions granted to the www-data user. This, for
+example, can be achieved with the following commands:
+
+$ mkdir /home/brep/ci-data
+$ setfacl -m g:www-data:rwx /home/brep/ci-data
+
+To also enable the CI request submission web form set the ci-form option. You
+can use the installed sample CI form fragment or create a custom one if your
+CI request handler requires additional information (besides the repository URL
+and optional package name[/version]) to be supplied by the client. For
+example:
+
+$ cp install/share/brep/www/ci.xhtml config/
+$ edit config/ci.xhtml # Add custom form fields, adjust CSS style, etc.
+
+For sample CI request handler implementations see brep/handler/ci/.
 
 Here we assume you have setup an appropriate Apache2 virtual server. Open the
 corresponding Apache2 .conf file and add the following inside VirtualHost (you
diff --git a/brep/.gitignore b/brep/.gitignore
new file mode 100644
index 0000000..f6635ca
--- /dev/null
+++ b/brep/.gitignore
@@ -0,0 +1,3 @@
+# All our bash modules are generated from .in files so ignore them wholesale.
+#
+*.bash
diff --git a/brep/handler/.gitignore b/brep/handler/.gitignore
deleted file mode 100644
index e3cf716..0000000
--- a/brep/handler/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-handler.bash
diff --git a/brep/handler/buildfile b/brep/handler/buildfile
index 5223b5c..5f64251 100644
--- a/brep/handler/buildfile
+++ b/brep/handler/buildfile
@@ -5,6 +5,6 @@
 import mods  = libbutl.bash%bash{manifest-parser}
 import mods += libbutl.bash%bash{manifest-serializer}
 
-./: bash{handler} submit/
+./: bash{handler} submit/ ci/
 
 bash{handler}: in{handler} $mods
diff --git a/brep/handler/ci/.gitignore b/brep/handler/ci/.gitignore
new file mode 100644
index 0000000..f31b542
--- /dev/null
+++ b/brep/handler/ci/.gitignore
@@ -0,0 +1 @@
+brep-ci-dir
diff --git a/brep/handler/ci/buildfile b/brep/handler/ci/buildfile
new file mode 100644
index 0000000..c45a79b
--- /dev/null
+++ b/brep/handler/ci/buildfile
@@ -0,0 +1,11 @@
+# file      : brep/handler/ci/buildfile
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+./: exe{brep-ci-dir}
+
+include ../
+
+exe{brep-ci-dir}: in{ci-dir} bash{ci} ../bash{handler}
+
+bash{ci}: in{ci} ../bash{handler}
diff --git a/brep/handler/ci/ci-dir.in b/brep/handler/ci/ci-dir.in
new file mode 100644
index 0000000..6a4f0af
--- /dev/null
+++ b/brep/handler/ci/ci-dir.in
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+
+# file      : brep/handler/ci/ci-dir.in
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+# Simple package CI request handler with directory storage.
+#
+# Keep the CI request directory unless simulating. Write the CI result
+# manifest to stdout.
+#
+usage="usage: $0 <dir>"
+
+verbose= #true
+
+trap "{ exit 1; }" ERR
+set -o errtrace # Trap ERR in functions.
+
+@import brep/handler/handler@
+@import brep/handler/ci/ci@
+
+if [ "$#" != 1 ]; then
+  error "$usage"
+fi
+
+# CI request data directory (last and the only argument).
+#
+data_dir="${!#/}"
+
+if [ -z "$data_dir" ]; then
+  error "$usage"
+fi
+
+if [ ! -d "$data_dir" ]; then
+  error "'$data_dir' does not exist or is not a directory"
+fi
+
+reference="$(basename "$data_dir")"
+
+# Parse the CI request manifest and obtain the repository URL, package names
+# with optional versions, as well as the simulate value.
+#
+manifest_parser_start "$data_dir/request.manifest"
+
+repository=
+packages=()
+simulate=
+
+while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
+  case "$n" in
+    repository) repository="$v"  ;;
+    package)    packages+=("$v") ;;
+    simulate)   simulate="$v"    ;;
+  esac
+done
+
+manifest_parser_finish
+
+if [ -z "$repository" ]; then
+  error "repository manifest value expected"
+fi
+
+if [ -n "$simulate" -a "$simulate" != "success" ]; then
+  exit_with_manifest 400 "unrecognized simulation outcome '$simulate'"
+fi
+
+# Produce the bpkg-build(1)-like package spec for tracing.
+#
+spec=
+for p in "${packages[@]}"; do
+  if [ -n "$spec" ]; then
+    spec="$spec,"
+  fi
+  spec="$spec$p"
+done
+
+if [ -n "$spec" ]; then
+  spec="$spec@"
+fi
+
+spec="$spec$repository"
+
+if [ -n "$simulate" ]; then
+  rm -r "$data_dir"
+  trace "CI request for '$spec' is simulated"
+else
+  trace "CI request for '$spec' is queued"
+fi
+
+# The spec normally contains the full commit id and so feels too hairy for
+# including in the message.
+#
+exit_with_manifest 200 "CI request is queued"
diff --git a/brep/handler/ci/ci.bash.in b/brep/handler/ci/ci.bash.in
new file mode 100644
index 0000000..023e98e
--- /dev/null
+++ b/brep/handler/ci/ci.bash.in
@@ -0,0 +1,41 @@
+# file      : brep/handler/ci/ci.bash.in
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+# Utility functions useful for implementing CI request handlers.
+
+if [ "$brep_handler_ci" ]; then
+  return 0
+else
+  brep_handler_ci=true
+fi
+
+@import brep/handler/handler@
+
+# Serialize the CI result manifest to stdout and exit the (sub-)shell with the
+# zero status.
+#
+reference= # Should be assigned by the handler when becomes available.
+
+function exit_with_manifest () # <status> <message>
+{
+  trace_func "$@"
+
+  local sts="$1"
+  local msg="$2"
+
+  manifest_serializer_start
+
+  manifest_serialize ""        "1"    # Start of manifest.
+  manifest_serialize "status"  "$sts"
+  manifest_serialize "message" "$msg"
+
+  if [ -n "$reference" ]; then
+    manifest_serialize "reference" "$reference"
+  elif [ "$sts" == "200" ]; then
+    error "no reference for code $sts"
+  fi
+
+  manifest_serializer_finish
+  run exit 0
+}
diff --git a/brep/handler/submit/.gitignore b/brep/handler/submit/.gitignore
index ef91424..cbbd541 100644
--- a/brep/handler/submit/.gitignore
+++ b/brep/handler/submit/.gitignore
@@ -1,5 +1,2 @@
-submit.bash
-submit-git.bash
-
 brep-submit-dir
 brep-submit-git
diff --git a/brep/handler/submit/buildfile b/brep/handler/submit/buildfile
index b110a1d..fd9fe14 100644
--- a/brep/handler/submit/buildfile
+++ b/brep/handler/submit/buildfile
@@ -4,6 +4,8 @@
 
 ./: exe{brep-submit-dir} exe{brep-submit-git}
 
+include ../
+
 exe{brep-submit-dir}: in{submit-dir} bash{submit} ../bash{handler}
 
 exe{brep-submit-git}: in{submit-git} \
diff --git a/doc/manual.cli b/doc/manual.cli
index 322c414..4c10689 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -190,12 +190,14 @@ message: <string>
 
 The CI functionality allows submission of package CI requests as well as
 additional, repository-specific information via the HTTP \c{GET} and \c{POST}
-methods. The implementation in \c{brep} only handles reception as well as
-basic parameter verification expecting the rest of the CI logic to be handled
-by a separate entity according to the repository policy. Such an entity can be
-notified by \c{brep} about a new CI request as an invocation of the \i{handler
-program} (as part of the HTTP request) and/or via email. It could also be a
-separate process that monitors the CI data directory.
+methods using the \c{application/x-www-form-urlencoded} or
+\c{multipart/form-data} parameters encoding. The implementation in \c{brep}
+only handles reception as well as basic parameter verification expecting the
+rest of the CI logic to be handled by a separate entity according to the
+repository policy. Such an entity can be notified by \c{brep} about a new CI
+request as an invocation of the \i{handler program} (as part of the HTTP
+request) and/or via email. It could also be a separate process that monitors
+the CI data directory.
 
 The CI request without any parameters is treated as the CI form request. If
 \c{ci-form} is configured, then such a form is generated and returned.
@@ -208,11 +210,11 @@ For each CI request \c{brep} performs the following steps.
 
 \li|Verify the required \c{repository} and optional \c{package} parameters.
 
-The \c{repository} parameter is the repository URL that contains the packages
-to be tested. If one or more \c{package} parameters are present, then only the
-specified packages are tested. If no \c{package} parameters are specified,
-then all the packages present in the repository (but excluding complement
-repositories) are tested.
+The \c{repository} parameter is the remote \c{bpkg} repository location that
+contains the packages to be tested. If one or more \c{package} parameters are
+present, then only the specified packages are tested. If no \c{package}
+parameters are specified, then all the packages present in the repository (but
+excluding complement repositories) are tested.
 
 Each \c{package} parameter can specify either just the package name, in which
 case all the versions of this package present in the repository will be
@@ -322,7 +324,7 @@ timestamp: <date-time>
 [user-agent]: <string>
 \
 
-The \c{package} value can be repeated multiple time. The \c{timestamp} value
+The \c{package} value can be repeated multiple times. The \c{timestamp} value
 is in the ISO-8601 \c{<YYYY>-<MM>-<DD>T<hh>:<mm>:<ss>Z} form (always
 UTC). Note also that \c{client-ip} can be IPv4 or IPv6.
 
diff --git a/etc/brep-module.conf b/etc/brep-module.conf
index e5770fb..2dc738e 100644
--- a/etc/brep-module.conf
+++ b/etc/brep-module.conf
@@ -265,6 +265,51 @@ menu About=?about
 # submit-handler-timeout 60
 
 
+# The directory to save CI request data to. If unspecified, the package CI
+# functionality will be disabled.
+#
+# Note that the directory path must be absolute and the directory itself must
+# exist and have read, write, and execute permissions granted to the user that
+# runs the web server.
+#
+# ci-data
+
+
+# The package CI form fragment. If specified, then its contents are treated as
+# an XHTML5 fragment that is inserted into the <body> element of the CI page.
+# If unspecified, then no CI page will be displayed. Note that the file path
+# must be absolute.
+#
+# ci-form
+
+
+# The package CI email. If specified, the CI request and result manifests will
+# be sent to this address.
+#
+# ci-email
+
+
+# The handler program to be executed on CI request. The handler is executed as
+# part of the HTTP request and is passed additional arguments that can be
+# specified with ci-handler-argument followed by the absolute path to the CI
+# request directory. Note that the program path must be absolute.
+#
+# ci-handler
+
+
+# Additional arguments to be passed to the CI handler program (see ci-handler
+# for details). Repeat this option to specify multiple arguments.
+#
+# ci-handler-argument
+
+
+# The CI handler program timeout in seconds. If specified and the handler does
+# not exit in the allotted time, then it is killed and its termination is
+# treated as abnormal.
+#
+# ci-handler-timeout
+
+
 # Trace verbosity. Disabled by default.
 #
 # verbosity 0
diff --git a/mod/external-handler.cxx b/mod/external-handler.cxx
new file mode 100644
index 0000000..d3ea6e3
--- /dev/null
+++ b/mod/external-handler.cxx
@@ -0,0 +1,346 @@
+// file      : mod/external-handler.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <mod/external-handler.hxx>
+
+#include <sys/time.h>   // timeval
+#include <sys/select.h>
+
+#include <ratio>        // ratio_greater_equal
+#include <chrono>
+#include <sstream>
+#include <cstdlib>      // strtoul()
+#include <type_traits>  // static_assert
+#include <system_error> // error_code, generic_category()
+
+#include <libbutl/process.mxx>
+#include <libbutl/fdstream.mxx>
+#include <libbutl/process-io.mxx> // operator<<(ostream, process_args)
+
+using namespace std;
+using namespace butl;
+
+namespace brep
+{
+  namespace external_handler
+  {
+    optional<result_manifest>
+    run (const path& handler,
+         const strings& args,
+         const dir_path& data_dir,
+         size_t tm,
+         const basic_mark& error,
+         const basic_mark& warn,
+         const basic_mark* trace)
+    {
+      using parser  = manifest_parser;
+      using parsing = manifest_parsing;
+
+      using namespace chrono;
+
+      using time_point = system_clock::time_point;
+      using duration   = system_clock::duration;
+
+      // Make sure that the system clock has at least milliseconds resolution.
+      //
+      static_assert(
+        ratio_greater_equal<milliseconds::period, duration::period>::value,
+        "The system clock resolution is too low");
+
+      // For the sake of the documentation we will call the handler's normal
+      // exit with 0 code "successful termination".
+      //
+      // To make sure the handler process execution doesn't exceed the
+      // specified timeout we set the non-blocking mode for the process
+      // stdout-reading stream, try to read from it with the 10 milliseconds
+      // timeout and check the process execution time between the reads. We
+      // then kill the process if the execution time is exceeded.
+      //
+      optional<milliseconds> timeout;
+
+      if (tm != 0)
+        timeout = milliseconds (tm * 1000);
+
+      // Note that due to the non-blocking mode we cannot just pass the stream
+      // to the manifest parser constructor. So we buffer the data in the
+      // string stream and then parse that.
+      //
+      stringstream ss;
+
+      assert (!data_dir.empty ());
+
+      // Normally the data directory leaf component identifies the entity
+      // being handled. We will use it as a reference for logging.
+      //
+      string ref (data_dir.leaf ().string ());
+
+      for (;;) // Breakout loop.
+        try
+        {
+          fdpipe pipe (fdopen_pipe ()); // Can throw io_error.
+
+          // Redirect the diagnostics to the web server error log.
+          //
+          process pr (
+            process_start_callback ([&trace] (const char* args[], size_t n)
+                                    {
+                                      if (trace != nullptr)
+                                        *trace << process_args {args, n};
+                                    },
+                                    0     /* stdin  */,
+                                    pipe  /* stdout */,
+                                    2     /* stderr */,
+                                    handler,
+                                    args,
+                                    data_dir));
+          pipe.out.close ();
+
+          auto kill = [&pr, &warn, &handler, &ref] ()
+            {
+              // We may still end up well (see below), thus this is a warning.
+              //
+              warn << "ref " << ref << ": process " << handler
+              << " execution timeout expired";
+
+              pr.kill ();
+            };
+
+          try
+          {
+            ifdstream is (move (pipe.in), fdstream_mode::non_blocking);
+
+            const size_t nbuf (8192);
+            char buf[nbuf];
+
+            while (is.is_open ())
+            {
+              time_point start;
+              milliseconds wd (10); // Max time to wait for the data portion.
+
+              if (timeout)
+              {
+                start = system_clock::now ();
+
+                if (*timeout < wd)
+                  wd = *timeout;
+              }
+
+              timeval tm {wd.count () / 1000        /* seconds */,
+                          wd.count () % 1000 * 1000 /* microseconds */};
+
+              fd_set rd;
+              FD_ZERO (&rd);
+              FD_SET  (is.fd (), &rd);
+
+              int r (select (is.fd () + 1, &rd, nullptr, nullptr, &tm));
+
+              if (r == -1)
+              {
+                // Don't fail if the select() call was interrupted by the
+                // signal.
+                //
+                if (errno != EINTR)
+                  throw_system_ios_failure (errno, "select failed");
+              }
+              else if (r != 0) // Is data available?
+              {
+                assert (FD_ISSET (is.fd (), &rd));
+
+                // The only leagal way to read from non-blocking ifdstream.
+                //
+                streamsize n (is.readsome (buf, nbuf));
+
+                // Close the stream (and bail out) if the end of the data is
+                // reached. Otherwise cache the read data.
+                //
+                if (is.eof ())
+                  is.close ();
+                else
+                {
+                  // The data must be available.
+                  //
+                  // Note that we could keep reading until the readsome() call
+                  // returns 0. However, this way we could potentially exceed
+                  // the timeout significantly for some broken handler that
+                  // floods us with data. So instead, we will be checking the
+                  // process execution time after every data chunk read.
+                  //
+                  assert (n != 0);
+
+                  ss.write (buf, n);
+                }
+              }
+              else // Timeout occured.
+              {
+                // Normally, we don't expect timeout to occur on the pipe read
+                // operation if the process has terminated successfully, as
+                // all its output must already be buffered (including eof).
+                // However, there can be some still running handler's child
+                // that has inherited the parent's stdout. In this case we
+                // assume that we have read all the handler's output, close
+                // the stream, log the warning and bail out.
+                //
+                if (pr.exit)
+                {
+                  // We keep reading only upon successful handler termination.
+                  //
+                  assert (*pr.exit);
+
+                  is.close ();
+
+                  warn << "ref " << ref << ": process " << handler
+                       << " stdout is not closed after termination (possibly "
+                       << "handler's child still running)";
+                }
+              }
+
+              if (timeout)
+              {
+                time_point now (system_clock::now ());
+
+                // Assume we have waited the full amount if the time
+                // adjustment is detected.
+                //
+                duration d (now > start ? now - start : wd);
+
+                // If the timeout is not fully exhausted, then decrement it and
+                // try to read some more data from the handler' stdout.
+                // Otherwise, kill the process, if not done yet.
+                //
+                // Note that it may happen that we are killing an already
+                // terminated process, in which case kill() just sets the
+                // process exit information. On the other hand it's guaranteed
+                // that the process is terminated after the kill() call, and
+                // so the pipe is presumably closed on the write end (see
+                // above for details). Thus, if the process terminated
+                // successfully, we will continue reading until eof is
+                // reached or read timeout occurred. Yes, it may happen that
+                // we will succeed even with the kill.
+                //
+                if (*timeout > d)
+                  *timeout -= duration_cast<milliseconds> (d);
+                else if (!pr.exit)
+                {
+                  kill ();
+
+                  assert (pr.exit);
+
+                  // Close the stream (and bail out) if the process hasn't
+                  // terminate successfully.
+                  //
+                  if (!*pr.exit)
+                    is.close ();
+
+                  *timeout = milliseconds::zero ();
+                }
+              }
+            }
+
+            assert (!is.is_open ());
+
+            if (!timeout)
+              pr.wait ();
+
+            // If the process is not terminated yet, then wait for its
+            // termination for the remaining time. Kill it if the timeout has
+            // been exceeded and the process still hasn't terminate.
+            //
+            else if (!pr.exit && !pr.timed_wait (*timeout))
+              kill ();
+
+            assert (pr.exit); // The process must finally be terminated.
+
+            if (*pr.exit)
+              break; // Get out of the breakout loop.
+
+            error << "ref " << ref << ": process " << handler << " "
+                  << *pr.exit;
+
+            // Fall through.
+          }
+          catch (const io_error& e)
+          {
+            if (pr.wait ())
+              error << "ref " << ref << ": unable to read handler's output: "
+                    << e;
+
+            // Fall through.
+          }
+
+          return nullopt;
+        }
+      // Handle process_error and io_error (both derive from system_error).
+      //
+        catch (const system_error& e)
+        {
+          error << "ref " << ref << ": unable to execute '" << handler
+                << "': " << e;
+
+          return nullopt;
+        }
+
+      result_manifest r;
+
+      // Parse and verify the manifest.
+      //
+      try
+      {
+        parser p (ss, handler.leaf ().string ());
+        manifest_name_value nv (p.next ());
+
+        auto bad_value ([&p, &nv] (const string& d) {
+            throw parsing (p.name (), nv.value_line, nv.value_column, d);});
+
+        if (nv.empty ())
+          bad_value ("empty manifest");
+
+        const string& n (nv.name);
+        const string& v (nv.value);
+
+        // The format version pair is verified by the parser.
+        //
+        assert (n.empty () && v == "1");
+
+        // Save the format version pair.
+        //
+        r.values.push_back (move (nv));
+
+        // Get and verify the HTTP status.
+        //
+        nv = p.next ();
+        if (n != "status")
+          bad_value ("no status specified");
+
+        char* e (nullptr);
+        unsigned long c (strtoul (v.c_str (), &e, 10)); // Can't throw.
+
+        assert (e != nullptr);
+
+        if (!(*e == '\0' && c >= 100 && c < 600))
+          bad_value ("invalid HTTP status '" + v + "'");
+
+        // Save the HTTP status.
+        //
+        r.status = static_cast<uint16_t> (c);
+        r.values.push_back (move (nv));
+
+        // Save the remaining name/value pairs.
+        //
+        for (nv = p.next (); !nv.empty (); nv = p.next ())
+          r.values.push_back (move (nv));
+
+        // Save end of manifest.
+        //
+        r.values.push_back (move (nv));
+      }
+      catch (const parsing& e)
+      {
+        error << "ref " << ref << ": unable to parse handler's output: " << e;
+        return nullopt;
+      }
+
+      return optional<result_manifest> (move (r));
+    }
+  }
+}
diff --git a/mod/external-handler.hxx b/mod/external-handler.hxx
new file mode 100644
index 0000000..45de711
--- /dev/null
+++ b/mod/external-handler.hxx
@@ -0,0 +1,52 @@
+// file      : mod/external-handler.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef MOD_EXTERNAL_HANDLER_HXX
+#define MOD_EXTERNAL_HANDLER_HXX
+
+#include <libbutl/manifest-parser.mxx>
+
+#include <libbrep/types.hxx>
+#include <libbrep/utility.hxx>
+
+#include <mod/diagnostics.hxx>
+
+namespace brep
+{
+  // Utility for running external handler programs.
+  //
+  namespace external_handler
+  {
+    // Run an external handler program and, if it exited normally with the
+    // zero exit status, return the result manifest it is expected to write to
+    // stdout, containing at least the HTTP status value. Otherwise, log an
+    // error and return nullopt. Redirect the program stderr to the web server
+    // error log.
+    //
+    // If the timeout (in seconds) is not zero and the handler program does
+    // not exit in the allotted time, then it is killed and its termination is
+    // treated as abnormal.
+    //
+    // Note that warnings can be logged regardless of the program success. If
+    // the trace argument is not NULL, then trace records are also logged.
+    //
+    struct result_manifest
+    {
+      uint16_t status;
+      vector<butl::manifest_name_value> values; // Note: all values, including
+                                                // status.
+    };
+
+    optional<result_manifest>
+    run (const path& handler,
+         const strings& args,
+         const dir_path& data_dir,
+         size_t timeout,
+         const basic_mark& error,
+         const basic_mark& warn,
+         const basic_mark* trace);
+  }
+}
+
+#endif // MOD_EXTERNAL_HANDLER_HXX
diff --git a/mod/mod-ci.cxx b/mod/mod-ci.cxx
new file mode 100644
index 0000000..79472d0
--- /dev/null
+++ b/mod/mod-ci.cxx
@@ -0,0 +1,628 @@
+// file      : mod/mod-ci.cxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#include <mod/mod-ci.hxx>
+
+#include <ostream>
+
+#include <libbutl/uuid.hxx>
+#include <libbutl/sendmail.mxx>
+#include <libbutl/fdstream.mxx>
+#include <libbutl/timestamp.mxx>
+#include <libbutl/filesystem.mxx>
+#include <libbutl/process-io.mxx>          // operator<<(ostream, process_args)
+#include <libbutl/manifest-parser.mxx>
+#include <libbutl/manifest-serializer.mxx>
+
+#include <libbpkg/manifest.hxx>
+#include <libbpkg/package-name.hxx>
+
+#include <web/xhtml.hxx>
+#include <web/module.hxx>
+
+#include <mod/page.hxx>
+#include <mod/options.hxx>
+#include <mod/external-handler.hxx>
+
+using namespace std;
+using namespace butl;
+using namespace web;
+using namespace brep::cli;
+
+brep::ci::
+ci (const ci& r)
+    : handler (r),
+      options_ (r.initialized_ ? r.options_ : nullptr),
+      form_ (r.initialized_ || r.form_ == nullptr
+             ? r.form_
+             : make_shared<xhtml::fragment> (*r.form_))
+{
+}
+
+void brep::ci::
+init (scanner& s)
+{
+  HANDLER_DIAG;
+
+  options_ = make_shared<options::ci> (
+    s, unknown_mode::fail, unknown_mode::fail);
+
+  // Verify that the CI request handling is setup properly, if configured.
+  //
+  if (options_->ci_data_specified ())
+  {
+    // Verify the data directory satisfies the requirements.
+    //
+    const dir_path& d (options_->ci_data ());
+
+    if (d.relative ())
+      fail << "ci-data directory path must be absolute";
+
+    if (!dir_exists (d))
+      fail << "ci-data directory '" << d << "' does not exist";
+
+    // Parse XHTML5 form file, if configured.
+    //
+    if (options_->ci_form_specified ())
+    {
+      const path& ci_form (options_->ci_form ());
+
+      if (ci_form.relative ())
+        fail << "ci-form path must be absolute";
+
+      try
+      {
+        ifdstream is (ci_form);
+
+        form_ = make_shared<xhtml::fragment> (is.read_text (),
+                                              ci_form.string ());
+      }
+      catch (const xml::parsing& e)
+      {
+        fail << "unable to parse ci-form file: " << e;
+      }
+      catch (const io_error& e)
+      {
+        fail << "unable to read ci-form file '" << ci_form << "': " << e;
+      }
+    }
+
+    if (options_->ci_handler_specified () &&
+        options_->ci_handler ().relative ())
+      fail << "ci-handler path must be absolute";
+  }
+
+  if (options_->root ().empty ())
+    options_->root (dir_path ("/"));
+}
+
+bool brep::ci::
+handle (request& rq, response& rs)
+{
+  using namespace bpkg;
+  using namespace xhtml;
+
+  using serializer    = manifest_serializer;
+  using serialization = manifest_serialization;
+
+  HANDLER_DIAG;
+
+  const dir_path& root (options_->root ());
+
+  // We will respond with the manifest to the CI request submission protocol
+  // violations and with a plain text message on the internal errors. In the
+  // latter case we will always respond with the same neutral message for
+  // security reason, logging the error details. Note that descriptions of
+  // exceptions caught by the web server are returned to the client (see
+  // web/module.hxx for details), and we want to avoid this when there is a
+  // danger of exposing sensitive data.
+  //
+  // Also we will pass through exceptions thrown by the underlying API, unless
+  // we need to handle them or add details for the description, in which case
+  // we will fallback to one of the above mentioned response methods.
+  //
+  // Note that both respond_manifest() and respond_error() are normally called
+  // right before the end of the request handling. They both always return
+  // true to allow bailing out with a single line, for example:
+  //
+  // return respond_error (); // Request is handled with an error.
+  //
+  string request_id; // Will be set later.
+  auto respond_manifest = [&rs, &request_id] (status_code status,
+                                              const string& message) -> bool
+  {
+    serializer s (rs.content (status, "text/manifest;charset=utf-8"),
+                  "response");
+
+    s.next ("", "1");                      // Start of manifest.
+    s.next ("status", to_string (status));
+    s.next ("message", message);
+
+    if (!request_id.empty ())
+      s.next ("reference", request_id);
+
+    s.next ("", "");                       // End of manifest.
+    return true;
+  };
+
+  auto respond_error = [&rs] (status_code status = 500) -> bool
+  {
+    rs.content (status, "text/plain;charset=utf-8")
+      << "CI request submission handling failed" << endl;
+
+    return true;
+  };
+
+  // Check if the CI request functionality is enabled.
+  //
+  // Note that this is not a submission protocol violation but it feels right
+  // to respond with the manifest, to help the client a bit.
+  //
+  if (!options_->ci_data_specified ())
+    return respond_manifest (404, "CI request submission disabled");
+
+  // Parse the request form data.
+  //
+  const name_values& rps (rq.parameters (64 * 1024));
+
+  // If there is no request parameters then we respond with the CI form XHTML,
+  // if configured. Otherwise, will proceed as for the CI request and will fail
+  // (missing parameters).
+  //
+  if (rps.empty () && form_ != nullptr)
+  {
+    const string title ("CI");
+
+    xml::serializer s (rs.content (), title);
+
+    s << HTML
+      <<   HEAD
+      <<     TITLE << title << ~TITLE
+      <<     CSS_LINKS (path ("ci.css"), root)
+      <<   ~HEAD
+      <<   BODY
+      <<     DIV_HEADER (root, options_->logo (), options_->menu ())
+      <<     DIV(ID="content") << *form_ << ~DIV
+      <<   ~BODY
+      << ~HTML;
+
+    return true;
+  }
+
+  // Verify the CI request parameters we expect. The unknown ones will be
+  // serialized to the CI request manifest.
+  //
+  params::ci params;
+
+  try
+  {
+    name_value_scanner s (rps);
+    params = params::ci (s, unknown_mode::skip, unknown_mode::skip);
+  }
+  catch (const cli::exception&)
+  {
+    return respond_manifest (400, "invalid parameter");
+  }
+
+  const string& simulate (params.simulate ());
+
+  if (simulate == "internal-error-text")
+    return respond_error ();
+  else if (simulate == "internal-error-html")
+  {
+    const string title ("Internal Error");
+    xml::serializer s (rs.content (500), title);
+
+    s << HTML
+      <<   HEAD << TITLE << title << ~TITLE << ~HEAD
+      <<   BODY << "CI request submission handling failed" << ~BODY
+      << ~HTML;
+
+    return true;
+  }
+
+  // Parse and verify the remote repository location.
+  //
+  repository_location rl;
+
+  try
+  {
+    const repository_url& u (params.repository ());
+
+    if (u.empty () || u.scheme == repository_protocol::file)
+      throw invalid_argument ("");
+
+    rl = repository_location (u, guess_type (u, false /* local */));
+  }
+  catch (const invalid_argument&)
+  {
+    return respond_manifest (400, "invalid repository location");
+  }
+
+  // Verify the package name[/version] arguments.
+  //
+  for (const string& s: params.package())
+  {
+    //  Let's skip the potentially unfilled package form fields.
+    //
+    if (s.empty ())
+      continue;
+
+    try
+    {
+      size_t p (s.find ('/'));
+
+      if (p != string::npos)
+      {
+        package_name (string (s, 0, p));
+
+        // Not to confuse with module::version.
+        //
+        bpkg::version (string (s, p + 1));
+      }
+      else
+        package_name p (s); // Not to confuse with the s variable declaration.
+    }
+    catch (const invalid_argument&)
+    {
+      return respond_manifest (400, "invalid package " + s);
+    }
+  }
+
+  // Verify that unknown parameter values satisfy the requirements (contain
+  // only ASCII printable characters plus '\r', '\n', and '\t').
+  //
+  // Actually, the expected ones must satisfy too, so check them as well.
+  //
+  auto printable = [] (const string& s) -> bool
+  {
+    for (char c: s)
+    {
+      if (!((c >= 0x20 && c <= 0x7E) || c == '\n' || c == '\r' || c == '\t'))
+        return false;
+    }
+    return true;
+  };
+
+  for (const name_value& nv: rps)
+  {
+    if (nv.value && !printable (*nv.value))
+      return respond_manifest (400, "invalid parameter " + nv.name);
+  }
+
+  try
+  {
+    // Note that from now on the result manifest we respond with will contain
+    // the reference value.
+    //
+    request_id = uuid::generate ().string ();
+  }
+  catch (const system_error& e)
+  {
+    error << "unable to generate request id: " << e;
+    return respond_error ();
+  }
+
+  // Create the submission data directory.
+  //
+  dir_path dd (options_->ci_data () / dir_path (request_id));
+
+  try
+  {
+    // It's highly unlikely but still possible that the directory already
+    // exists. This can only happen if the generated uuid is not unique.
+    //
+    if (try_mkdir (dd) == mkdir_status::already_exists)
+      throw_generic_error (EEXIST);
+  }
+  catch (const system_error& e)
+  {
+    error << "unable to create directory '" << dd << "': " << e;
+    return respond_error ();
+  }
+
+  auto_rmdir ddr (dd);
+
+  // Serialize the CI request manifest to a stream. On the serialization error
+  // respond to the client with the manifest containing the bad request (400)
+  // code and return false, on the stream error pass through the io_error
+  // exception, otherwise return true.
+  //
+  timestamp ts (system_clock::now ());
+
+  auto rqm = [&request_id,
+              &rl,
+              &ts,
+              &simulate,
+              &rq,
+              &rps,
+              &params,
+              &respond_manifest]
+             (ostream& os) -> bool
+  {
+    try
+    {
+      serializer s (os, "request");
+
+      // Serialize the submission manifest header.
+      //
+      s.next ("", "1");                // Start of manifest.
+      s.next ("id", request_id);
+      s.next ("repository", rl.string ());
+
+      for (const string& p: params.package())
+      {
+        if (!p.empty ()) // Skip empty package names (see above for details).
+          s.next ("package", p);
+      }
+
+      s.next ("timestamp",
+              butl::to_string (ts,
+                               "%Y-%m-%dT%H:%M:%SZ",
+                               false /* special */,
+                               false /* local */));
+
+      if (!simulate.empty ())
+        s.next ("simulate", simulate);
+
+      // Serialize the User-Agent HTTP header and the client IP address.
+      //
+      optional<string> ip;
+      optional<string> ua;
+      for (const name_value& h: rq.headers ())
+      {
+        if (casecmp (h.name, ":Client-IP") == 0)
+          ip = h.value;
+        else if (casecmp (h.name, "User-Agent") == 0)
+          ua = h.value;
+      }
+
+      if (ip)
+        s.next ("client-ip", *ip);
+
+      if (ua)
+        s.next ("user-agent", *ua);
+
+      // Serialize the request parameters.
+      //
+      // Note that the serializer constraints the parameter names (can't start
+      // with '#', can't contain ':' and the whitespaces, etc.).
+      //
+      for (const name_value& nv: rps)
+      {
+        const string& n (nv.name);
+
+        if (n != "repository" &&
+            n != "_"          &&
+            n != "package"    &&
+            n != "simulate")
+          s.next (n, nv.value ? *nv.value : "");
+      }
+
+      s.next ("", ""); // End of manifest.
+      return true;
+    }
+    catch (const serialization& e)
+    {
+      respond_manifest (400, string ("invalid parameter: ") + e.what ());
+      return false;
+    }
+  };
+
+  // Serialize the CI request manifest to the submission directory.
+  //
+  path rqf (dd / "request.manifest");
+
+  try
+  {
+    ofdstream os (rqf);
+    bool r (rqm (os));
+    os.close ();
+
+    if (!r)
+      return true; // The client is already responded with the manifest.
+  }
+  catch (const io_error& e)
+  {
+    error << "unable to write to '" << rqf << "': " << e;
+    return respond_error ();
+  }
+
+  // Given that the submission data is now successfully persisted we are no
+  // longer in charge of removing it, except for the cases when the submission
+  // handler terminates with an error (see below for details).
+  //
+  ddr.cancel ();
+
+  // If the handler terminates with non-zero exit status or specifies 5XX
+  // (HTTP server error) submission result manifest status value, then we
+  // stash the submission data directory for troubleshooting. Otherwise, if
+  // it's the 4XX (HTTP client error) status value, then we remove the
+  // directory.
+  //
+  // Note that leaving the directory in place in case of a submission error
+  // would have prevent the user from re-submitting until we research the
+  // issue and manually remove the directory.
+  //
+  auto stash_submit_dir = [&dd, error] ()
+  {
+    if (dir_exists (dd))
+    try
+    {
+      mvdir (dd, dir_path (dd + ".fail"));
+    }
+    catch (const system_error& e)
+    {
+      // Not much we can do here. Let's just log the issue and bail out
+      // leaving the directory in place.
+      //
+      error << "unable to rename directory '" << dd << "': " << e;
+    }
+  };
+
+  // Run the submission handler, if specified, reading the result manifest
+  // from its stdout and caching it as a name/value pair list for later use
+  // (forwarding to the client, sending via email, etc.). Otherwise, create
+  // implied result manifest.
+  //
+  status_code sc;
+  vector<manifest_name_value> rvs;
+
+  if (options_->ci_handler_specified ())
+  {
+    using namespace external_handler;
+
+    optional<result_manifest> r (run (options_->ci_handler (),
+                                      options_->ci_handler_argument (),
+                                      dd,
+                                      options_->ci_handler_timeout (),
+                                      error,
+                                      warn,
+                                      verb_ ? &trace : nullptr));
+    if (!r)
+    {
+      stash_submit_dir ();
+      return respond_error (); // The diagnostics is already issued.
+    }
+
+    sc = r->status;
+    rvs = move (r->values);
+  }
+  else // Create the implied result manifest.
+  {
+    sc = 200;
+
+    auto add = [&rvs] (string n, string v)
+    {
+      manifest_name_value nv {move (n), move (v),
+                              0 /* name_line */,  0 /* name_column */,
+                              0 /* value_line */, 0 /* value_column */};
+
+      rvs.emplace_back (move (nv));
+    };
+
+    add ("", "1");                           // Start of manifest.
+    add ("status", "200");
+    add ("message", "CI request is queued");
+    add ("reference", request_id);
+    add ("", "");                            // End of manifest.
+  }
+
+  assert (!rvs.empty ()); // Produced by the handler or is implied.
+
+  // Serialize the submission result manifest to a stream. On the
+  // serialization error log the error description and return false, on the
+  // stream error pass through the io_error exception, otherwise return true.
+  //
+  auto rsm = [&rvs, &error, &request_id] (ostream& os) -> bool
+  {
+    try
+    {
+      serializer s (os, "result");
+      for (const manifest_name_value& nv: rvs)
+        s.next (nv.name, nv.value);
+
+      return true;
+    }
+    catch (const serialization& e)
+    {
+      error << "ref " << request_id << ": unable to serialize handler's "
+            << "output: " << e;
+      return false;
+    }
+  };
+
+  // If the submission data directory still exists then perform an appropriate
+  // action on it, depending on the submission result status. Note that the
+  // handler could move or remove the directory.
+  //
+  if (dir_exists (dd))
+  {
+    // Remove the directory if the client error is detected.
+    //
+    if (sc >= 400 && sc < 500)
+      rmdir_r (dd);
+
+    // Otherwise, save the result manifest, into the directory. Also stash the
+    // directory for troubleshooting in case of the server error.
+    //
+    else
+    {
+      path rsf (dd / "result.manifest");
+
+      try
+      {
+        ofdstream os (rsf);
+
+        // Not being able to stash the result manifest is not a reason to
+        // claim the submission failed. The error is logged nevertheless.
+        //
+        rsm (os);
+
+        os.close ();
+      }
+      catch (const io_error& e)
+      {
+        // Not fatal (see above).
+        //
+        error << "unable to write to '" << rsf << "': " << e;
+      }
+
+      if (sc >= 500 && sc < 600)
+        stash_submit_dir ();
+    }
+  }
+
+  // Send email, if configured, and the CI request submission is not simulated.
+  //
+  // Note that we don't consider the email sending failure to be a submission
+  // failure as the submission data is successfully persisted and the handler
+  // is successfully executed, if configured. One can argue that email can be
+  // essential for the submission processing and missing it would result in
+  // the incomplete submission. In this case it's natural to assume that the
+  // web server error log is monitored and the email sending failure will be
+  // noticed.
+  //
+  if (options_->ci_email_specified () && simulate.empty ())
+  try
+  {
+    // Redirect the diagnostics to the web server error log.
+    //
+    sendmail sm ([&trace, this] (const char* args[], size_t n)
+                 {
+                   l2 ([&]{trace << process_args {args, n};});
+                 },
+                 2 /* stderr */,
+                 options_->email (),
+                 "CI request submission (" + request_id + ")",
+                 {options_->ci_email ()});
+
+    // Write the submission request manifest.
+    //
+    bool r (rqm (sm.out));
+    assert (r); // The serialization succeeded once, so can't fail now.
+
+    // Write the submission result manifest.
+    //
+    sm.out << "\n\n";
+
+    rsm (sm.out); // We don't care about the result (see above).
+
+    sm.out.close ();
+
+    if (!sm.wait ())
+      error << "sendmail " << *sm.exit;
+  }
+  // Handle process_error and io_error (both derive from system_error).
+  //
+  catch (const system_error& e)
+  {
+    error << "sendmail error: " << e;
+  }
+
+  if (!rsm (rs.content (sc, "text/manifest;charset=utf-8")))
+    return respond_error (); // The error description is already logged.
+
+  return true;
+}
diff --git a/mod/mod-ci.hxx b/mod/mod-ci.hxx
new file mode 100644
index 0000000..f9e89ff
--- /dev/null
+++ b/mod/mod-ci.hxx
@@ -0,0 +1,45 @@
+// file      : mod/mod-ci.hxx -*- C++ -*-
+// copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+// license   : MIT; see accompanying LICENSE file
+
+#ifndef MOD_MOD_CI_HXX
+#define MOD_MOD_CI_HXX
+
+#include <web/xhtml-fragment.hxx>
+
+#include <libbrep/types.hxx>
+#include <libbrep/utility.hxx>
+
+#include <mod/module.hxx>
+#include <mod/options.hxx>
+
+namespace brep
+{
+  class ci: public handler
+  {
+  public:
+    ci () = default;
+
+    // Create a shallow copy (handling instance) if initialized and a deep
+    // copy (context exemplar) otherwise.
+    //
+    explicit
+    ci (const ci&);
+
+    virtual bool
+    handle (request&, response&);
+
+    virtual const cli::options&
+    cli_options () const {return options::ci::description ();}
+
+  private:
+    virtual void
+    init (cli::scanner&);
+
+  private:
+    shared_ptr<options::ci> options_;
+    shared_ptr<web::xhtml::fragment> form_;
+  };
+}
+
+#endif // MOD_MOD_CI_HXX
diff --git a/mod/mod-repository-root.cxx b/mod/mod-repository-root.cxx
index 27901d7..3b0ab1f 100644
--- a/mod/mod-repository-root.cxx
+++ b/mod/mod-repository-root.cxx
@@ -12,6 +12,8 @@
 
 #include <mod/module.hxx>
 #include <mod/options.hxx>
+
+#include <mod/mod-ci.hxx>
 #include <mod/mod-submit.hxx>
 #include <mod/mod-builds.hxx>
 #include <mod/mod-build-log.hxx>
@@ -112,7 +114,8 @@ namespace brep
         build_force_ (make_shared<build_force> ()),
         build_log_ (make_shared<build_log> ()),
         builds_ (make_shared<builds> ()),
-        submit_ (make_shared<submit> ())
+        submit_ (make_shared<submit> ()),
+        ci_ (make_shared<ci> ())
   {
   }
 
@@ -164,6 +167,10 @@ namespace brep
           r.initialized_
           ? r.submit_
           : make_shared<submit> (*r.submit_)),
+        ci_ (
+          r.initialized_
+          ? r.ci_
+          : make_shared<ci> (*r.ci_)),
         options_ (
           r.initialized_
           ? r.options_
@@ -188,6 +195,7 @@ namespace brep
     append (r, build_log_->options ());
     append (r, builds_->options ());
     append (r, submit_->options ());
+    append (r, ci_->options ());
     return r;
   }
 
@@ -231,6 +239,7 @@ namespace brep
     sub_init (*build_log_, "build_log");
     sub_init (*builds_, "builds");
     sub_init (*submit_, "submit");
+    sub_init (*ci_, "ci");
 
     // Parse own configuration options.
     //
@@ -371,6 +380,13 @@ namespace brep
 
           return handle ("submit", true);
         }
+        else if (fn == "ci")
+        {
+          if (handler_ == nullptr)
+            handler_.reset (new ci (*ci_));
+
+          return handle ("ci", true);
+        }
       }
 
       if (handler_ == nullptr)
diff --git a/mod/mod-repository-root.hxx b/mod/mod-repository-root.hxx
index 74691ea..9a71849 100644
--- a/mod/mod-repository-root.hxx
+++ b/mod/mod-repository-root.hxx
@@ -23,6 +23,7 @@ namespace brep
   class build_log;
   class builds;
   class submit;
+  class ci;
 
   class repository_root: public handler
   {
@@ -67,6 +68,8 @@ namespace brep
     shared_ptr<build_log> build_log_;
     shared_ptr<builds> builds_;
     shared_ptr<submit> submit_;
+    shared_ptr<ci> ci_;
+
     shared_ptr<options::repository_root> options_;
 
     // Sub-handler the request is dispatched to. Initially is NULL. It is set
diff --git a/mod/mod-submit.cxx b/mod/mod-submit.cxx
index 61eeaf6..470bd45 100644
--- a/mod/mod-submit.cxx
+++ b/mod/mod-submit.cxx
@@ -4,19 +4,9 @@
 
 #include <mod/mod-submit.hxx>
 
-#include <sys/time.h>   // timeval
-#include <sys/select.h>
-
-#include <ratio>        // ratio_greater_equal
-#include <chrono>
-#include <cstdlib>      // strtoul()
-#include <istream>
-#include <sstream>
-#include <type_traits>  // static_assert
-#include <system_error> // error_code, generic_category()
+#include <ostream>
 
 #include <libbutl/sha256.mxx>
-#include <libbutl/process.mxx>
 #include <libbutl/sendmail.mxx>
 #include <libbutl/fdstream.mxx>
 #include <libbutl/timestamp.mxx>
@@ -30,6 +20,7 @@
 
 #include <mod/page.hxx>
 #include <mod/options.hxx>
+#include <mod/external-handler.hxx>
 
 using namespace std;
 using namespace butl;
@@ -113,8 +104,6 @@ handle (request& rq, response& rs)
 {
   using namespace xhtml;
 
-  using parser        = manifest_parser;
-  using parsing       = manifest_parsing;
   using serializer    = manifest_serializer;
   using serialization = manifest_serialization;
 
@@ -285,8 +274,8 @@ handle (request& rq, response& rs)
       return respond_manifest (400, "invalid parameter " + nv.name);
   }
 
-  // Note that from now on the result manifest will contain the reference
-  // value.
+  // Note that from now on the result manifest we respond with will contain
+  // the reference value.
   //
   ref = string (sha256sum, 0, 12);
 
@@ -299,7 +288,7 @@ handle (request& rq, response& rs)
   if (dir_exists (dd) || simulate == "duplicate-archive")
     return respond_manifest (422, "duplicate submission");
 
-  // Create the temporary submission directory.
+  // Create the temporary submission data directory.
   //
   dir_path td;
 
@@ -573,310 +562,34 @@ handle (request& rq, response& rs)
   // (forwarding to the client, sending via email, etc.). Otherwise, create
   // implied result manifest.
   //
-  status_code sc (200);
+  status_code sc;
   vector<manifest_name_value> rvs;
 
   if (options_->submit_handler_specified ())
   {
-    // For the sake of the documentation we will call the handler's normal
-    // exit with 0 code "successful termination".
-    //
-    // To make sure the handler process execution doesn't exceed the specified
-    // timeout we set the non-blocking mode for the process stdout-reading
-    // stream, try to read from it with the 10 milliseconds timeout and check
-    // the process execution time between the reads. We then kill the process
-    // if the execution time is exceeded.
-    //
-    using namespace chrono;
-
-    using time_point = system_clock::time_point;
-    using duration   = system_clock::duration;
-
-    // Make sure that the system clock has at least milliseconds resolution.
-    //
-    static_assert(
-      ratio_greater_equal<milliseconds::period, duration::period>::value,
-      "The system clock resolution is too low");
-
-    optional<milliseconds> timeout;
-
-    if (options_->submit_handler_timeout_specified ())
-      timeout = milliseconds (options_->submit_handler_timeout () * 1000);
-
-    const path& handler (options_->submit_handler ());
-
-    // Note that due to the non-blocking mode we cannot just pass the stream
-    // to the manifest parser constructor. So we buffer the data in the string
-    // stream and then parse that.
-    //
-    stringstream ss;
-
-    for (;;) // Breakout loop.
-    try
-    {
-      fdpipe pipe (fdopen_pipe ()); // Can throw io_error.
-
-      // Redirect the diagnostics to the web server error log.
-      //
-      process pr (
-        process_start_callback (print_args,
-                                0     /* stdin  */,
-                                pipe  /* stdout */,
-                                2     /* stderr */,
-                                handler,
-                                options_->submit_handler_argument (),
-                                dd));
-      pipe.out.close ();
-
-      auto kill = [&pr, &warn, &handler, &ref] ()
-      {
-        // We may still end up well (see below), thus this is a warning.
-        //
-        warn << "ref " << ref << ": process " << handler
-             << " execution timeout expired";
-
-        pr.kill ();
-      };
-
-      try
-      {
-        ifdstream is (move (pipe.in), fdstream_mode::non_blocking);
-
-        const size_t nbuf (8192);
-        char buf[nbuf];
-
-        while (is.is_open ())
-        {
-          time_point start;
-          milliseconds wd (10); // Max time to wait for the data portion.
-
-          if (timeout)
-          {
-            start = system_clock::now ();
-
-            if (*timeout < wd)
-              wd = *timeout;
-          }
-
-          timeval tm {wd.count () / 1000        /* seconds */,
-                      wd.count () % 1000 * 1000 /* microseconds */};
-
-          fd_set rd;
-          FD_ZERO (&rd);
-          FD_SET  (is.fd (), &rd);
-
-          int r (select (is.fd () + 1, &rd, nullptr, nullptr, &tm));
-
-          if (r == -1)
-          {
-            // Don't fail if the select() call was interrupted by the signal.
-            //
-            if (errno != EINTR)
-              throw_system_ios_failure (errno, "select failed");
-          }
-          else if (r != 0) // Is data available?
-          {
-            assert (FD_ISSET (is.fd (), &rd));
-
-            // The only leagal way to read from non-blocking ifdstream.
-            //
-            streamsize n (is.readsome (buf, nbuf));
-
-            // Close the stream (and bail out) if the end of the data is
-            // reached. Otherwise cache the read data.
-            //
-            if (is.eof ())
-              is.close ();
-            else
-            {
-              // The data must be available.
-              //
-              // Note that we could keep reading until the readsome() call
-              // returns 0. However, this way we could potentially exceed the
-              // timeout significantly for some broken handler that floods us
-              // with data. So instead, we will be checking the process
-              // execution time after every data chunk read.
-              //
-              assert (n != 0);
-
-              ss.write (buf, n);
-            }
-          }
-          else // Timeout occured.
-          {
-            // Normally, we don't expect timeout to occur on the pipe read
-            // operation if the process has terminated successfully, as all its
-            // output must already be buffered (including eof). However, there
-            // can be some still running handler's child that has inherited
-            // the parent's stdout. In this case we assume that we have read
-            // all the handler's output, close the stream, log the warning and
-            // bail out.
-            //
-            if (pr.exit)
-            {
-              // We keep reading only upon successful handler termination.
-              //
-              assert (*pr.exit);
-
-              is.close ();
-
-              warn << "ref " << ref << ": process " << handler
-                   << " stdout is not closed after termination (possibly "
-                   << "handler's child still running)";
-            }
-          }
-
-          if (timeout)
-          {
-            time_point now (system_clock::now ());
-
-            // Assume we have waited the full amount if the time adjustment is
-            // detected.
-            //
-            duration d (now > start ? now - start : wd);
-
-            // If the timeout is not fully exhausted, then decrement it and
-            // try to read some more data from the handler' stdout. Otherwise,
-            // kill the process, if not done yet.
-            //
-            // Note that it may happen that we are killing an already
-            // terminated process, in which case kill() just sets the process
-            // exit information. On the other hand it's guaranteed that the
-            // process is terminated after the kill() call, and so the pipe is
-            // presumably closed on the write end (see above for details).
-            // Thus, if the process terminated successfully, we will continue
-            // reading until eof is reached or read timeout occurred. Yes, it
-            // may happen that we end up with a successful submission even
-            // with the kill.
-            //
-            if (*timeout > d)
-              *timeout -= duration_cast<milliseconds> (d);
-            else if (!pr.exit)
-            {
-              kill ();
-
-              assert (pr.exit);
-
-              // Close the stream (and bail out) if the process hasn't
-              // terminate successfully.
-              //
-              if (!*pr.exit)
-                is.close ();
-
-              *timeout = milliseconds::zero ();
-            }
-          }
-        }
-
-        assert (!is.is_open ());
-
-        if (!timeout)
-          pr.wait ();
-
-        // If the process is not terminated yet, then wait for its termination
-        // for the remaining time. Kill it if the timeout has been exceeded
-        // and the process still hasn't terminate.
-        //
-        else if (!pr.exit && !pr.timed_wait (*timeout))
-          kill ();
-
-        assert (pr.exit); // The process must finally be terminated.
-
-        if (*pr.exit)
-          break; // Get out of the breakout loop.
-
-        error << "ref " << ref << ": process " << handler << " " << *pr.exit;
-
-        // Fall through.
-      }
-      catch (const io_error& e)
-      {
-        if (pr.wait ())
-          error << "ref " << ref << ": unable to read handler's output: " << e;
+    using namespace external_handler;
 
-        // Fall through.
-      }
-
-      stash_submit_dir ();
-      return respond_error ();
-    }
-    // Handle process_error and io_error (both derive from system_error).
-    //
-    catch (const system_error& e)
+    optional<result_manifest> r (run (options_->submit_handler (),
+                                      options_->submit_handler_argument (),
+                                      dd,
+                                      options_->submit_handler_timeout (),
+                                      error,
+                                      warn,
+                                      verb_ ? &trace : nullptr));
+    if (!r)
     {
-      error << "unable to execute '" << handler << "': " << e;
-
       stash_submit_dir ();
-      return respond_error ();
-    }
-
-    try
-    {
-      // Parse and verify the manifest. Obtain the HTTP status code (must go
-      // first) and cache it for the subsequent response to the client.
-      //
-      parser p (ss, "handler");
-      manifest_name_value nv (p.next ());
-
-      auto bad_value ([&p, &nv] (const string& d) {
-          throw parsing (p.name (), nv.value_line, nv.value_column, d);});
-
-      if (nv.empty ())
-        bad_value ("empty manifest");
-
-      const string& n (nv.name);
-      const string& v (nv.value);
-
-      // The format version pair is verified by the parser.
-      //
-      assert (n.empty () && v == "1");
-
-      // Cache the format version pair.
-      //
-      rvs.push_back (move (nv));
-
-      // Get and verify the HTTP status.
-      //
-      nv = p.next ();
-      if (n != "status")
-        bad_value ("no status specified");
-
-      char* e (nullptr);
-      unsigned long c (strtoul (v.c_str (), &e, 10)); // Can't throw.
-
-      assert (e != nullptr);
-
-      if (!(*e == '\0' && c >= 100 && c < 600))
-        bad_value ("invalid HTTP status '" + v + "'");
-
-      // Cache the HTTP status.
-      //
-      sc = static_cast<status_code> (c);
-      rvs.push_back (move (nv));
-
-      // Cache the remaining name/value pairs.
-      //
-      for (nv = p.next (); !nv.empty (); nv = p.next ())
-        rvs.push_back (move (nv));
-
-      // Cache end of manifest.
-      //
-      rvs.push_back (move (nv));
+      return respond_error (); // The diagnostics is already issued.
     }
-    catch (const parsing& e)
-    {
-      error << "ref " << ref << ": unable to parse handler's output: " << e;
 
-      // It appears the handler had misbehaved, so let's stash the submission
-      // directory for troubleshooting.
-      //
-      stash_submit_dir ();
 
-      return respond_error ();
-    }
+    sc = r->status;
+    rvs = move (r->values);
   }
-  else // Create implied result manifest.
+  else // Create the implied result manifest.
   {
+    sc = 200;
+
     auto add = [&rvs] (string n, string v)
     {
       manifest_name_value nv {move (n), move (v),
diff --git a/mod/options.cli b/mod/options.cli
index f7a9387..046173c 100644
--- a/mod/options.cli
+++ b/mod/options.cli
@@ -729,7 +729,10 @@ namespace brep
     {
       // Package repository URL.
       //
-      bpkg::repository_url repository;
+      // Note that the ci parameter is renamed to '_' by the root handler (see
+      // the request_proxy class for details).
+      //
+      bpkg::repository_url repository | _;
 
       // Package names/versions.
       //
diff --git a/tests/ci/buildfile b/tests/ci/buildfile
new file mode 100644
index 0000000..b1c3146
--- /dev/null
+++ b/tests/ci/buildfile
@@ -0,0 +1,16 @@
+# file      : tests/ci/buildfile
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+define common: file
+common{*}: extension = test
+
+dir = ../../brep/handler/ci/
+
+include $dir
+
+commons = data
+
+./: test{* -{$commons}} common{$commons} $dir/exe{brep-ci-dir}
+
+test{ci-dir}@./: test = $out_base/$dir/brep-ci-dir
diff --git a/tests/ci/ci-dir.test b/tests/ci/ci-dir.test
new file mode 100644
index 0000000..5b9e8c8
--- /dev/null
+++ b/tests/ci/ci-dir.test
@@ -0,0 +1,77 @@
+# file      : tests/ci/ci-dir.test
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+.include data.test
+
+: args
+{
+  : none
+  :
+  $* 2>>~%EOE% != 0
+  %\[.+\] \[brep:error\] \[ref \] \[brep-ci-dir\]: usage: .+brep-ci-dir <dir>%
+  EOE
+
+  : not-exist
+  :
+  $* $~/dir 2>>~%EOE% != 0
+    %\[.+\] \[brep:error\] \[ref dir\] \[brep-ci-dir\]: '.+dir' does not exist or is not a directory%
+    EOE
+}
+
+: success
+:
+{
+  test.arguments += $data_dir
+
+  : simulate
+  :
+  {
+    $clone_root_data;
+
+    echo "simulate: success" >+$data_dir/request.manifest;
+
+    $* >>"EOO";
+      : 1
+      status: 200
+      message: CI request is queued
+      reference: $request_id
+      EOO
+
+    test -d $data_dir != 0
+  }
+
+  : for-real
+  :
+  {
+    $clone_root_data_clean;
+
+    $* >>"EOO"
+      : 1
+      status: 200
+      message: CI request is queued
+      reference: $request_id
+      EOO
+  }
+}
+
+: failure
+:
+{
+  test.arguments += $data_dir
+
+  : bad-simulate
+  :
+  {
+    $clone_root_data_clean;
+
+    echo "simulate: fly" >+$data_dir/request.manifest;
+
+    $* >>"EOO"
+      : 1
+      status: 400
+      message: unrecognized simulation outcome 'fly'
+      reference: $request_id
+      EOO
+  }
+}
diff --git a/tests/ci/data.test b/tests/ci/data.test
new file mode 100644
index 0000000..c30a691
--- /dev/null
+++ b/tests/ci/data.test
@@ -0,0 +1,39 @@
+# file      : tests/ci/data.test
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+# Pre-created CI request submission data directory that will be copied by
+# subsequent tests and scope setup commands. The common approach will be that
+# group scopes copy and modify the parent scope submission directory as
+# required by the nested tests and scopes. Tests will also clone the parent
+# scope submission data directory to optionally modify it, use and cleanup at
+# the end. Note that configuration can not be shared between multiple
+# submission handler processes. Also we need to make sure that submission data
+# directories are not cloned while being used by submission handler scripts.
+#
+request_id = 4cfa00ec-8459-4f4f-9ef0-8883ddcc4f5b
+data_dir = $request_id/
+
++mkdir $data_dir
+
++cat <<"EOI" >=$data_dir/request.manifest
+  : 1
+  id: $request_id
+  repository: https://example.com/hello.git#master
+  package: foo
+  package: bar/1.0
+  timestamp: 2018-08-24T18:08:01Z
+  EOI
+
+root_data_dir = $~/$data_dir
+
+# The most commonly used submission data directory cloning command that copies
+# it from the parent scope working directory.
+#
+clone_data       = cp --no-cleanup -r ../$data_dir ./
+clone_data_clean = cp --no-cleanup -r ../$data_dir ./ &$data_dir/***
+
+# Clones the original submission data directory.
+#
+clone_root_data       = cp --no-cleanup -r $root_data_dir ./
+clone_root_data_clean = cp --no-cleanup -r $root_data_dir ./ &$data_dir/***
diff --git a/tests/submit/buildfile b/tests/submit/buildfile
index 46e38ad..32bb44e 100644
--- a/tests/submit/buildfile
+++ b/tests/submit/buildfile
@@ -7,6 +7,8 @@ common{*}: extension = test
 
 dir = ../../brep/handler/submit/
 
+include $dir
+
 commons = data
 
 ./: test{* -{$commons}} common{$commons} {*/ -test/}{**} \
diff --git a/tests/submit/submit-dir.test b/tests/submit/submit-dir.test
index 7fa7341..055449a 100644
--- a/tests/submit/submit-dir.test
+++ b/tests/submit/submit-dir.test
@@ -22,14 +22,14 @@
 : success
 :
 {
-  test.arguments += $checksum
+  test.arguments += $data_dir
 
   : simulate
   :
   {
     $clone_root_data;
 
-    echo "simulate: success" >+$checksum/request.manifest;
+    echo "simulate: success" >+$data_dir/request.manifest;
 
     $* >>"EOO";
       : 1
@@ -38,7 +38,7 @@
       reference: $checksum
       EOO
 
-    test -d $checksum != 0
+    test -d $data_dir != 0
   }
 
   : for-real
@@ -58,14 +58,14 @@
 : failure
 :
 {
-  test.arguments += $checksum
+  test.arguments += $data_dir
 
   : bad-archive
   :
   {
     $clone_root_data_clean;
 
-    echo "junk" >=$checksum/libhello-0.1.0.tar.gz;
+    echo "junk" >=$data_dir/libhello-0.1.0.tar.gz;
 
     $* >>"EOO"
       : 1
@@ -80,7 +80,7 @@
   {
     $clone_root_data_clean;
 
-    echo "simulate: fly" >+$checksum/request.manifest;
+    echo "simulate: fly" >+$data_dir/request.manifest;
 
     $* >>"EOO"
       : 1
diff --git a/www/ci-body.css b/www/ci-body.css
new file mode 100644
index 0000000..1403dbd
--- /dev/null
+++ b/www/ci-body.css
@@ -0,0 +1,22 @@
+/*
+ * CI request submission form (based on proplist and form-table)
+ */
+#ci
+{
+  margin-top: .8em;
+  margin-bottom: .8em;
+
+  padding-top: .4em;
+  padding-bottom: .4em;
+}
+
+#ci th
+{
+  width: 7.5em;
+}
+
+#ci input, #submit-padding
+{
+  width: 100%;
+  margin:0;
+}
diff --git a/www/ci.css b/www/ci.css
new file mode 100644
index 0000000..1ec2d9c
--- /dev/null
+++ b/www/ci.css
@@ -0,0 +1,3 @@
+@import url(common.css);
+@import url(brep-common.css);
+@import url(ci-body.css);
diff --git a/www/ci.scss b/www/ci.scss
new file mode 100644
index 0000000..395938e
--- /dev/null
+++ b/www/ci.scss
@@ -0,0 +1,3 @@
+@import "common";
+@import "brep-common";
+@import "ci-body";
diff --git a/www/ci.xhtml b/www/ci.xhtml
new file mode 100644
index 0000000..185f08b
--- /dev/null
+++ b/www/ci.xhtml
@@ -0,0 +1,26 @@
+<!-- Note that in HTML5 the boolean attribute absence represents false value,
+     true otherwise. If it is present then the value must be empty or
+     case-insensitively match the attribute's name. -->
+
+<form method="post">
+  <table id="ci" class="proplist">
+    <tbody>
+      <tr>
+        <th>repository</th>
+        <td><input type="text" name="repository" required=""/></td>
+      </tr>
+      <tr>
+        <th>package</th>
+        <td><input type="text" name="package"/></td>
+      </tr>
+    </tbody>
+  </table>
+  <table class="form-table">
+    <tbody>
+      <tr>
+        <td id="submit-padding"/>
+        <td><input type="submit" value="Submit"/></td>
+      </tr>
+    </tbody>
+  </table>
+</form>