Always serialize reference submit result manifest value if available and restructure handlers dir

14 files changed, 157 insertions, 110 deletions

diff --git a/brep/handler/.gitignore b/brep/handler/.gitignore
new file mode 100644
index 0000000..e3cf716
--- /dev/null
+++ b/brep/handler/.gitignore
@@ -0,0 +1 @@
+handler.bash
diff --git a/brep/handler/buildfile b/brep/handler/buildfile
new file mode 100644
index 0000000..5223b5c
--- /dev/null
+++ b/brep/handler/buildfile
@@ -0,0 +1,10 @@
+# file      : brep/handler/buildfile
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+import mods  = libbutl.bash%bash{manifest-parser}
+import mods += libbutl.bash%bash{manifest-serializer}
+
+./: bash{handler} submit/
+
+bash{handler}: in{handler} $mods
diff --git a/brep/submit/submit.bash.in b/brep/handler/handler.bash.in
index babf081..89e7e21 100644
--- a/brep/submit/submit.bash.in
+++ b/brep/handler/handler.bash.in
@@ -1,13 +1,13 @@
-# file      : brep/submit/submit.bash.in
+# file      : brep/handler/handler.bash.in
 # copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
-# Utility functions useful for implementing submission handlers.
+# Utility functions useful for implementing request handlers.
 
-if [ "$brep_submit" ]; then
+if [ "$brep_handler" ]; then
   return 0
 else
-  brep_submit=true
+  brep_handler=true
 fi
 
 @import libbutl/manifest-parser@
@@ -33,7 +33,11 @@ fi
 info_self="$(basename $0)"
 
 if [ "$#" -gt 0 ]; then
-  info_ref="$(basename "${!#/}")" # Last argument is the submission directory.
+  # Last argument is the request data directory which leaf component normally
+  # identifies the posted entity. A handler may overwrite this value if that's
+  # not the case.
+  #
+  info_ref="$(basename "${!#/}")"
 fi
 
 function info () # <severity> <text>
@@ -145,53 +149,3 @@ function manifest_serialize () # <name> <value>
 #  trace "$1: $2"
   printf "%s:%s\0" "$1" "$2" >&"$manifest_serializer_ifd"
 }
-
-# Serialize the submission result manifest to stdout and exit the (sub-)shell
-# with the zero status.
-#
-function exit_with_manifest () # <status> <message> [<reference>]
-{
-  trace_func "$@"
-
-  local sts="$1"
-  local msg="$2"
-  local ref="$3"
-
-  manifest_serializer_start
-
-  manifest_serialize ""        "1"    # Start of manifest.
-  manifest_serialize "status"  "$sts"
-  manifest_serialize "message" "$msg"
-
-  if [ -n "$ref" ]; then
-    if [ "$sts" != "200" ]; then
-      error "reference for code $sts"
-    fi
-
-    manifest_serialize "reference" "$ref"
-  elif [ "$sts" == "200" ]; then
-    error "no reference for code $sts"
-  fi
-
-  manifest_serializer_finish
-  run exit 0
-}
-
-# Verify archive is a valid package and extract its manifest into
-# <manifest> file.
-#
-function extract_package_manifest () # <archive> <manifest>
-{
-  local arc="$1"
-  local man="$2"
-
-  if ! run_silent bpkg pkg-verify --manifest "$arc" >"$man"; then
-    # Perform the sanity check to make sure that bpkg is runnable.
-    #
-    if ! run bpkg --version >/dev/null; then
-      error "unable to run bpkg"
-    fi
-
-    exit_with_manifest 400 "archive is not a valid package (run bpkg pkg-verify for details)"
-  fi
-}
diff --git a/brep/submit/.gitignore b/brep/handler/submit/.gitignore
index ef91424..ef91424 100644
--- a/brep/submit/.gitignore
+++ b/brep/handler/submit/.gitignore
diff --git a/brep/handler/submit/buildfile b/brep/handler/submit/buildfile
new file mode 100644
index 0000000..b110a1d
--- /dev/null
+++ b/brep/handler/submit/buildfile
@@ -0,0 +1,13 @@
+# file      : brep/handler/submit/buildfile
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+./: exe{brep-submit-dir} exe{brep-submit-git}
+
+exe{brep-submit-dir}: in{submit-dir} bash{submit} ../bash{handler}
+
+exe{brep-submit-git}: in{submit-git} \
+                      bash{submit-git} bash{submit} ../bash{handler}
+
+bash{submit}:     in{submit} ../bash{handler}
+bash{submit-git}: in{submit-git} bash{submit} ../bash{handler}
diff --git a/brep/submit/submit-dir.in b/brep/handler/submit/submit-dir.in
index 4bcbe5f..bda0bf8 100644
--- a/brep/submit/submit-dir.in
+++ b/brep/handler/submit/submit-dir.in
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# file      : brep/submit/submit-dir.in
+# file      : brep/handler/submit/submit-dir.in
 # copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
@@ -18,7 +18,8 @@ verbose= #true
 trap "{ exit 1; }" ERR
 set -o errtrace # Trap ERR in functions.
 
-@import brep/submit/submit@
+@import brep/handler/handler@
+@import brep/handler/submit/submit@
 
 if [ "$#" != 1 ]; then
   error "$usage"
@@ -104,4 +105,4 @@ else
   trace "$name/$version submission is queued"
 fi
 
-exit_with_manifest 200 "$name/$version submission is queued" "$reference"
+exit_with_manifest 200 "$name/$version submission is queued"
diff --git a/brep/submit/submit-git.bash.in b/brep/handler/submit/submit-git.bash.in
index 06d9b7c..2fd26b0 100644
--- a/brep/submit/submit-git.bash.in
+++ b/brep/handler/submit/submit-git.bash.in
@@ -1,16 +1,17 @@
-# file      : brep/submit/submit-git.bash.in
+# file      : brep/handler/submit/submit-git.bash.in
 # copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
 # Utility functions for the submit-git handler.
 
-if [ "$brep_submit_git" ]; then
+if [ "$brep_handler_submit_git" ]; then
   return 0
 else
-  brep_submit_git=true
+  brep_handler_submit_git=true
 fi
 
-@import brep/submit/submit@
+@import brep/handler/handler@
+@import brep/handler/submit/submit@
 
 # If the section is mapped to a directory in the repository configuration then
 # return this directory path and empty string otherwise.
diff --git a/brep/submit/submit-git.in b/brep/handler/submit/submit-git.in
index 0c90309..47f9a1a 100644
--- a/brep/submit/submit-git.in
+++ b/brep/handler/submit/submit-git.in
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# file      : brep/submit/submit-git.in
+# file      : brep/handler/submit/submit-git.in
 # copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
 # license   : MIT; see accompanying LICENSE file
 
@@ -182,8 +182,9 @@ ref_lock_timeout=30
 trap "{ exit 1; }" ERR
 set -o errtrace # Trap ERR in functions.
 
-@import brep/submit/submit@
-@import brep/submit/submit-git@
+@import brep/handler/handler@
+@import brep/handler/submit/submit@
+@import brep/handler/submit/submit-git@
 
 # Parse the command line options.
 #
@@ -656,4 +657,4 @@ else
   trace "$name/$version submission is queued"
 fi
 
-exit_with_manifest 200 "$name/$version submission is queued" "$reference"
+exit_with_manifest 200 "$name/$version submission is queued"
diff --git a/brep/handler/submit/submit.bash.in b/brep/handler/submit/submit.bash.in
new file mode 100644
index 0000000..38a4e06
--- /dev/null
+++ b/brep/handler/submit/submit.bash.in
@@ -0,0 +1,60 @@
+# file      : brep/handler/submit/submit.bash.in
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license   : MIT; see accompanying LICENSE file
+
+# Utility functions useful for implementing package submission handlers.
+
+if [ "$brep_handler_submit" ]; then
+  return 0
+else
+  brep_handler_submit=true
+fi
+
+@import brep/handler/handler@
+
+# Serialize the package submission result manifest to stdout and exit the
+# (sub-)shell with the zero status.
+#
+reference= # Should be assigned later by the handler, when becomes available.
+
+function exit_with_manifest () # <status> <message>
+{
+  trace_func "$@"
+
+  local sts="$1"
+  local msg="$2"
+
+  manifest_serializer_start
+
+  manifest_serialize ""        "1"    # Start of manifest.
+  manifest_serialize "status"  "$sts"
+  manifest_serialize "message" "$msg"
+
+  if [ -n "$reference" ]; then
+    manifest_serialize "reference" "$reference"
+  elif [ "$sts" == "200" ]; then
+    error "no reference for code $sts"
+  fi
+
+  manifest_serializer_finish
+  run exit 0
+}
+
+# Verify archive is a valid package and extract its manifest into
+# <manifest> file.
+#
+function extract_package_manifest () # <archive> <manifest>
+{
+  local arc="$1"
+  local man="$2"
+
+  if ! run_silent bpkg pkg-verify --manifest "$arc" >"$man"; then
+    # Perform the sanity check to make sure that bpkg is runnable.
+    #
+    if ! run bpkg --version >/dev/null; then
+      error "unable to run bpkg"
+    fi
+
+    exit_with_manifest 400 "archive is not a valid package (run bpkg pkg-verify for details)"
+  fi
+}
diff --git a/brep/submit/buildfile b/brep/submit/buildfile
deleted file mode 100644
index 50f9615..0000000
--- a/brep/submit/buildfile
+++ /dev/null
@@ -1,14 +0,0 @@
-# file      : brep/submit/buildfile
-# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
-# license   : MIT; see accompanying LICENSE file
-
-import mods  = libbutl.bash%bash{manifest-parser}
-import mods += libbutl.bash%bash{manifest-serializer}
-
-./: exe{brep-submit-dir} exe{brep-submit-git}
-
-exe{brep-submit-dir}: in{submit-dir} bash{submit}
-exe{brep-submit-git}: in{submit-git} bash{submit-git} bash{submit}
-
-bash{submit}:     in{submit} $mods
-bash{submit-git}: in{submit-git} bash{submit}
diff --git a/mod/mod-submit.cxx b/mod/mod-submit.cxx
index da698bc..61eeaf6 100644
--- a/mod/mod-submit.cxx
+++ b/mod/mod-submit.cxx
@@ -140,9 +140,9 @@ handle (request& rq, response& rs)
   //
   // return respond_error (); // Request is handled with an error.
   //
-  auto respond_manifest = [&rs] (status_code status,
-                                 const string& message,
-                                 const char* ref = nullptr) -> bool
+  string ref; // Will be set later.
+  auto respond_manifest = [&rs, &ref] (status_code status,
+                                       const string& message) -> bool
   {
     serializer s (rs.content (status, "text/manifest;charset=utf-8"),
                   "response");
@@ -151,7 +151,7 @@ handle (request& rq, response& rs)
     s.next ("status", to_string (status));
     s.next ("message", message);
 
-    if (ref != nullptr)
+    if (!ref.empty ())
       s.next ("reference", ref);
 
     s.next ("", ""); // End of manifest.
@@ -285,12 +285,16 @@ handle (request& rq, response& rs)
       return respond_manifest (400, "invalid parameter " + nv.name);
   }
 
+  // Note that from now on the result manifest will contain the reference
+  // value.
+  //
+  ref = string (sha256sum, 0, 12);
+
   // Check for a duplicate submission.
   //
   // Respond with the unprocessable entity (422) code if a duplicate is found.
   //
-  string ac (sha256sum, 0, 12);
-  dir_path dd (options_->submit_data () / dir_path (ac));
+  dir_path dd (options_->submit_data () / dir_path (ref));
 
   if (dir_exists (dd) || simulate == "duplicate-archive")
     return respond_manifest (422, "duplicate submission");
@@ -306,7 +310,7 @@ handle (request& rq, response& rs)
     // using the abbreviated checksum can be helpful for troubleshooting.
     //
     td = dir_path (options_->submit_temp () /
-                   dir_path (path::traits::temp_name (ac)));
+                   dir_path (path::traits::temp_name (ref)));
 
     // It's highly unlikely but still possible that the temporary directory
     // already exists. This can only happen due to the unclean web server
@@ -624,11 +628,11 @@ handle (request& rq, response& rs)
                                 dd));
       pipe.out.close ();
 
-      auto kill = [&pr, &warn, &handler, &ac] ()
+      auto kill = [&pr, &warn, &handler, &ref] ()
       {
         // We may still end up well (see below), thus this is a warning.
         //
-        warn << "ref " << ac << ": process " << handler
+        warn << "ref " << ref << ": process " << handler
              << " execution timeout expired";
 
         pr.kill ();
@@ -716,7 +720,7 @@ handle (request& rq, response& rs)
 
               is.close ();
 
-              warn << "ref " << ac << ": process " << handler
+              warn << "ref " << ref << ": process " << handler
                    << " stdout is not closed after termination (possibly "
                    << "handler's child still running)";
             }
@@ -781,14 +785,14 @@ handle (request& rq, response& rs)
         if (*pr.exit)
           break; // Get out of the breakout loop.
 
-        error << "ref " << ac << ": process " << handler << " " << *pr.exit;
+        error << "ref " << ref << ": process " << handler << " " << *pr.exit;
 
         // Fall through.
       }
       catch (const io_error& e)
       {
         if (pr.wait ())
-          error << "ref " << ac << ": unable to read handler's output: " << e;
+          error << "ref " << ref << ": unable to read handler's output: " << e;
 
         // Fall through.
       }
@@ -861,7 +865,7 @@ handle (request& rq, response& rs)
     }
     catch (const parsing& e)
     {
-      error << "ref " << ac << ": unable to parse handler's output: " << e;
+      error << "ref " << ref << ": unable to parse handler's output: " << e;
 
       // It appears the handler had misbehaved, so let's stash the submission
       // directory for troubleshooting.
@@ -885,7 +889,7 @@ handle (request& rq, response& rs)
     add ("", "1");                           // Start of manifest.
     add ("status", "200");
     add ("message", "submission is queued");
-    add ("reference", ac);
+    add ("reference", ref);
     add ("", "");                            // End of manifest.
   }
 
@@ -895,7 +899,7 @@ handle (request& rq, response& rs)
   // serialization error log the error description and return false, on the
   // stream error pass through the io_error exception, otherwise return true.
   //
-  auto rsm = [&rvs, &error, &ac] (ostream& os) -> bool
+  auto rsm = [&rvs, &error, &ref] (ostream& os) -> bool
   {
     try
     {
@@ -907,7 +911,7 @@ handle (request& rq, response& rs)
     }
     catch (const serialization& e)
     {
-      error << "ref " << ac << ": unable to serialize handler's output: " << e;
+      error << "ref " << ref << ": unable to serialize handler's output: " << e;
       return false;
     }
   };
@@ -971,7 +975,7 @@ handle (request& rq, response& rs)
     sendmail sm (print_args,
                  2 /* stderr */,
                  options_->email (),
-                 "new package submission " + a.string () + " (" + ac + ")",
+                 "new package submission " + a.string () + " (" + ref + ")",
                  {options_->submit_email ()});
 
     // Write the submission request manifest.
diff --git a/tests/submit/buildfile b/tests/submit/buildfile
index 6606153..46e38ad 100644
--- a/tests/submit/buildfile
+++ b/tests/submit/buildfile
@@ -5,7 +5,7 @@
 define common: file
 common{*}: extension = test
 
-dir = ../../brep/submit/
+dir = ../../brep/handler/submit/
 
 commons = data
 
diff --git a/tests/submit/submit-dir.test b/tests/submit/submit-dir.test
index 97f7edd..7fa7341 100644
--- a/tests/submit/submit-dir.test
+++ b/tests/submit/submit-dir.test
@@ -67,10 +67,11 @@
 
     echo "junk" >=$checksum/libhello-0.1.0.tar.gz;
 
-    $* >>EOO
+    $* >>"EOO"
       : 1
       status: 400
-      message: archive is not a valid package (run bpkg pkg-verify for details)
+      message: archive is not a valid package \(run bpkg pkg-verify for details\)
+      reference: $checksum
       EOO
   }
 
@@ -85,6 +86,7 @@
       : 1
       status: 400
       message: unrecognized simulation outcome 'fly'
+      reference: $checksum
       EOO
   }
 }
diff --git a/tests/submit/submit-git.test b/tests/submit/submit-git.test
index a64e9a1..5bfa4d4 100644
--- a/tests/submit/submit-git.test
+++ b/tests/submit/submit-git.test
@@ -466,6 +466,7 @@ pkg_ctl="$prj_ctl/hello.git"
       : 1
       status: 400
       message: author-name manifest value expected
+      reference: $checksum
       EOO
   }
 
@@ -482,6 +483,7 @@ pkg_ctl="$prj_ctl/hello.git"
       : 1
       status: 400
       message: author-email manifest value expected
+      reference: $checksum
       EOO
   }
 
@@ -498,6 +500,7 @@ pkg_ctl="$prj_ctl/hello.git"
       : 1
       status: 400
       message: author-name manifest value expected
+      reference: $checksum
       EOO
   }
 
@@ -519,10 +522,11 @@ pkg_ctl="$prj_ctl/hello.git"
     $g -C ref commit -m 'Add libhello-0.1.0.tar.gz';
     $g -C ref push;
 
-    $* "$root_tgt_url" $~/ref $data_dir >>EOO
+    $* "$root_tgt_url" $~/ref $data_dir >>"EOO"
       : 1
       status: 422
       message: duplicate submission
+      reference: $checksum
       EOO
   }
 
@@ -559,10 +563,11 @@ pkg_ctl="$prj_ctl/hello.git"
     $g -C ref commit -m 'Add ownership info';
     $g -C ref push;
 
-    $* "$root_tgt_url" $~/ref $data_dir >>EOO
+    $* "$root_tgt_url" $~/ref $data_dir >>"EOO"
       : 1
       status: 401
       message: package owner authentication failed
+      reference: $checksum
       EOO
   }
 
@@ -590,10 +595,11 @@ pkg_ctl="$prj_ctl/hello.git"
     $g -C ref commit -m 'Add project ownership info';
     $g -C ref push;
 
-    $* "$root_tgt_url" $~/ref $data_dir >>EOO
+    $* "$root_tgt_url" $~/ref $data_dir >>"EOO"
       : 1
       status: 401
       message: project owner authentication failed
+      reference: $checksum
       EOO
   }
 
@@ -630,10 +636,11 @@ pkg_ctl="$prj_ctl/hello.git"
     $g -C ref commit -m 'Add ownership info';
     $g -C ref push;
 
-    $* "$root_tgt_url" $~/ref $data_dir >>EOO
+    $* "$root_tgt_url" $~/ref $data_dir >>"EOO"
       : 1
       status: 401
       message: package owner authentication failed
+      reference: $checksum
       EOO
   }
 
@@ -655,10 +662,11 @@ pkg_ctl="$prj_ctl/hello.git"
     $g -C tgt commit -m 'Add libhello-0.1.0.tar.gz';
     $g -C tgt push;
 
-    $* "file:///$~/tgt.git" $data_dir >>EOO
+    $* "file:///$~/tgt.git" $data_dir >>"EOO"
       : 1
       status: 422
       message: duplicate submission
+      reference: $checksum
       EOO
   }
 
@@ -694,10 +702,11 @@ pkg_ctl="$prj_ctl/hello.git"
     $g -C tgt commit -m 'Add ownership info';
     $g -C tgt push;
 
-    $* "file:///$~/tgt.git" $data_dir >>EOO
+    $* "file:///$~/tgt.git" $data_dir >>"EOO"
       : 1
       status: 401
       message: package owner authentication failed
+      reference: $checksum
       EOO
   }
 
@@ -724,10 +733,11 @@ pkg_ctl="$prj_ctl/hello.git"
     $g -C tgt commit -am 'Disable ownership';
     $g -C tgt push;
 
-    $* "file:///$~/tgt.git" $root_ref_dir $data_dir >>EOO
+    $* "file:///$~/tgt.git" $root_ref_dir $data_dir >>"EOO"
       : 1
       status: 401
       message: project owner authentication failed
+      reference: $checksum
       EOO
   }
 
@@ -771,10 +781,11 @@ pkg_ctl="$prj_ctl/hello.git"
     $g -C tgt commit -am 'Disable ownership';
     $g -C tgt push;
 
-    $* "file:///$~/tgt.git" ref $data_dir >>EOO
+    $* "file:///$~/tgt.git" ref $data_dir >>"EOO"
       : 1
       status: 401
       message: package owner authentication failed
+      reference: $checksum
       EOO
   }
 
@@ -811,10 +822,11 @@ pkg_ctl="$prj_ctl/hello.git"
     $g -C tgt commit -m 'Add ownership info';
     $g -C tgt push;
 
-    $* "file:///$~/tgt.git" $data_dir >>EOO
+    $* "file:///$~/tgt.git" $data_dir >>"EOO"
       : 1
       status: 401
       message: package owner authentication failed
+      reference: $checksum
       EOO
   }
 
@@ -832,10 +844,11 @@ pkg_ctl="$prj_ctl/hello.git"
     $clone_root_tgt;
     $g clone tgt.git &tgt/***;
 
-    $* "file:///$~/tgt.git" $data_dir >>EOO
+    $* "file:///$~/tgt.git" $data_dir >>"EOO"
       : 1
       status: 401
       message: package publishing authorization failed
+      reference: $checksum
       EOO
   }
 
@@ -851,6 +864,7 @@ pkg_ctl="$prj_ctl/hello.git"
       : 1
       status: 400
       message: unrecognized section 'delta'
+      reference: $checksum
       EOO
   }
 }