aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--INSTALL3
-rw-r--r--brep/submit/.gitignore4
-rw-r--r--brep/submit/buildfile9
-rw-r--r--brep/submit/submit-dir.in57
-rw-r--r--brep/submit/submit-git.bash.in331
-rw-r--r--brep/submit/submit-git.in498
-rw-r--r--brep/submit/submit.bash.in149
-rw-r--r--doc/manual.cli29
-rw-r--r--etc/brep-module.conf7
-rw-r--r--libbrep/common.hxx4
-rw-r--r--mod/mod-submit.cxx429
-rw-r--r--mod/options.cli8
-rw-r--r--tests/submit/0f6b1460b3ec/libhello-0.1.0.tar.gzbin0 -> 3033 bytes
-rw-r--r--tests/submit/0f6b1460b3ec/package.manifest10
-rw-r--r--tests/submit/0f6b1460b3ec/request.manifest10
-rw-r--r--tests/submit/0f6b1460b3ec/result.manifest4
-rw-r--r--tests/submit/README30
-rw-r--r--tests/submit/buildfile16
-rw-r--r--tests/submit/data.test35
-rw-r--r--tests/submit/hello.tar.gzbin0 -> 102400 bytes
-rw-r--r--tests/submit/submit-dir.test90
-rw-r--r--tests/submit/submit-git.test765
22 files changed, 2299 insertions, 189 deletions
diff --git a/INSTALL b/INSTALL
index b978c7d..8c679ca 100644
--- a/INSTALL
+++ b/INSTALL
@@ -240,8 +240,7 @@ example:
$ cp install/share/brep/www/submit.xhtml config/
$ edit config/submit.xhtml # Add custom form fields, adjust CSS style, etc.
-For an example of the submission handler see the brep/submit/submit.in bash
-script.
+For sample submission handler implementations see brep/submit/.
Here we assume you have setup an appropriate Apache2 virtual server. Open the
corresponding Apache2 .conf file and add the following inside VirtualHost (you
diff --git a/brep/submit/.gitignore b/brep/submit/.gitignore
index 8df3374..ef91424 100644
--- a/brep/submit/.gitignore
+++ b/brep/submit/.gitignore
@@ -1,3 +1,5 @@
+submit.bash
+submit-git.bash
+
brep-submit-dir
brep-submit-git
-
diff --git a/brep/submit/buildfile b/brep/submit/buildfile
index e3711f8..50f9615 100644
--- a/brep/submit/buildfile
+++ b/brep/submit/buildfile
@@ -5,5 +5,10 @@
import mods = libbutl.bash%bash{manifest-parser}
import mods += libbutl.bash%bash{manifest-serializer}
-exe{brep-submit-dir}: in{submit-dir} bash{submit} $mods
-bash{submit}: in{submit} $mods # @@ Currently doesn't depend on manifest-parser.
+./: exe{brep-submit-dir} exe{brep-submit-git}
+
+exe{brep-submit-dir}: in{submit-dir} bash{submit}
+exe{brep-submit-git}: in{submit-git} bash{submit-git} bash{submit}
+
+bash{submit}: in{submit} $mods
+bash{submit-git}: in{submit-git} bash{submit}
diff --git a/brep/submit/submit-dir.in b/brep/submit/submit-dir.in
index 31ae85d..4bcbe5f 100644
--- a/brep/submit/submit-dir.in
+++ b/brep/submit/submit-dir.in
@@ -13,70 +13,69 @@
#
usage="usage: $0 <dir>"
-verbose=true
+verbose= #true
trap "{ exit 1; }" ERR
set -o errtrace # Trap ERR in functions.
-@import libbutl/manifest-parser@
-@import libbutl/manifest-serializer@
-
@import brep/submit/submit@
-# Submission data directory (last argument).
+if [ "$#" != 1 ]; then
+ error "$usage"
+fi
+
+# Submission data directory (last and the only argument).
#
-dir="${!#/}"
+data_dir="${!#/}"
-if [ -z "$dir" ]; then
+if [ -z "$data_dir" ]; then
error "$usage"
fi
-if [ ! -d "$dir" ]; then
- error "'$dir' does not exist or is not a directory"
+if [ ! -d "$data_dir" ]; then
+ error "'$data_dir' does not exist or is not a directory"
fi
+reference="$(basename "$data_dir")"
+
# Parse the submission request manifest and obtain the archive path as well
# as the simulate value.
#
-trace "parsing $dir/request.manifest"
-butl_manifest_parser_start "$dir/request.manifest"
+manifest_parser_start "$data_dir/request.manifest"
archive=
simulate=
-while IFS=: read -ru "$butl_manifest_parser_ofd" -d '' n v; do
+while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
case "$n" in
- archive) archive="$v" ;;
+ archive) archive="$v" ;;
simulate) simulate="$v" ;;
esac
done
-butl_manifest_parser_finish
+manifest_parser_finish
if [ -z "$archive" ]; then
error "archive manifest value expected"
fi
if [ -n "$simulate" -a "$simulate" != "success" ]; then
- trace "unrecognized simulation outcome '$simulate'"
- result_manifest 400 "unrecognized simulation outcome"
- exit 0
+ exit_with_manifest 400 "unrecognized simulation outcome '$simulate'"
fi
-manifest="$dir/package.manifest"
-
-extract_package_manifest "$dir/$archive" "$manifest"
+m="$data_dir/package.manifest"
+extract_package_manifest "$data_dir/$archive" "$m"
-# Parse the package manifest and obtain the package name and version.
+# Parse the package manifest and obtain the package name, version, and
+# project.
#
-trace "parsing $manifest"
-butl_manifest_parser_start "$manifest"
+manifest_parser_start "$m"
name=
version=
project=
-while IFS=: read -ru "$butl_manifest_parser_ofd" -d '' n v; do
+while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
case "$n" in
name) name="$v" ;;
version) version="$v" ;;
@@ -84,14 +83,14 @@ while IFS=: read -ru "$butl_manifest_parser_ofd" -d '' n v; do
esac
done
-butl_manifest_parser_finish
+manifest_parser_finish
if [ -z "$name" ]; then
- error "name manifest values expected"
+ error "name manifest value expected"
fi
if [ -z "$version" ]; then
- error "version manifest values expected"
+ error "version manifest value expected"
fi
if [ -z "$project" ]; then
@@ -99,10 +98,10 @@ if [ -z "$project" ]; then
fi
if [ -n "$simulate" ]; then
- rm -r "$dir"
+ rm -r "$data_dir"
trace "$name/$version submission is simulated"
else
trace "$name/$version submission is queued"
fi
-result_manifest 200 "$name/$version submission is queued" "$reference"
+exit_with_manifest 200 "$name/$version submission is queued" "$reference"
diff --git a/brep/submit/submit-git.bash.in b/brep/submit/submit-git.bash.in
new file mode 100644
index 0000000..8d5cc84
--- /dev/null
+++ b/brep/submit/submit-git.bash.in
@@ -0,0 +1,331 @@
+# file : brep/submit/submit-git.bash.in
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+# Utility functions for the submit-git handler.
+
+if [ "$brep_submit_git" ]; then
+ return 0
+else
+ brep_submit_git=true
+fi
+
+@import brep/submit/submit@
+
+# If the section is mapped to a directory in the repository configuration then
+# return this directory path and empty string otherwise.
+#
+function section_dir () # <section> <repo-dir>
+{
+ trace_func "$@"
+
+ local sec="$1"
+ local rep="$2"
+
+ local owners # Unused but is declared to avoid polluting the global space.
+ local -A sections
+
+ run source "$rep/submit.config.bash"
+
+ local r="${sections[$sec]}"
+ if [ -z "$r" ]; then
+ r="${sections['*']}"
+ fi
+
+ echo "$r"
+}
+
+# If the owners directory is set in the repository configuration then return
+# this directory path prefixed with the repository directory path and the
+# empty string otherwise.
+#
+function owners_dir () # <repo-dir>
+{
+ local rep="$1"
+
+ local owners
+ local -A sections # Is declared to avoid polluting the global space.
+
+ run source "$rep/submit.config.bash"
+
+ local r=
+ if [ -n "$owners" ]; then
+ r="$rep/$owners"
+ fi
+
+ echo "$r"
+}
+
+# Check if a repository already contains the package. Respond with the
+# 'duplicate submission' result manifest and exit if that's the case.
+#
+function check_package_duplicate () # <name> <version> <repo-dir>
+{
+ trace_func "$@"
+
+ local nam="$1"
+ local ver="$2"
+ local rep="$3"
+
+ local owners # Unused but is declared to avoid polluting the global space.
+ local -A sections
+
+ run source "$rep/submit.config.bash"
+
+ # Check for duplicate package in all sections. Use <name>-<version>.*
+ # without .tar.gz in case we want to support more archive types later.
+ #
+ local s
+ for s in "${!sections[@]}"; do
+ local d="$rep/${sections[$s]}"
+
+ if [ -d "$d" ]; then
+ local f
+ f="$(run find "$d" -name "$nam-$ver.*")"
+
+ if [ -n "$f" ]; then
+ trace "found: $f"
+ exit_with_manifest 422 "duplicate submission"
+ fi
+ fi
+ done
+}
+
+# Serialize the project or package owner manifest (they have the same set of
+# values) to the specified manifest file.
+#
+function create_owner_manifest () # <name> <email> <control> <file>
+{
+ trace_func "$@"
+
+ local nam="$1"
+ local eml="$2"
+ local ctl="$3"
+ local man="$4"
+
+ if [ -f "$man" ]; then
+ error "'$man' already exists"
+ fi
+
+ manifest_serializer_start "$man"
+
+ manifest_serialize "" "1" # Start of manifest.
+ manifest_serialize "name" "$nam"
+ manifest_serialize "email" "$eml"
+ manifest_serialize "control" "$ctl"
+
+ manifest_serializer_finish
+}
+
+# Strip the query part and the leaf path component from the repository URL.
+#
+function repository_base () # <repo-url>
+{
+ # First, strip the URL query part, then component.
+ #
+ sed -n \
+-e 's%^\([^?]*\).*$%\1%' \
+-e 's%^\(.*\)/[^/]\{1,\}/\{0,1\}$%\1%p' \
+<<<"$1"
+}
+
+# Authenticate the project name owner. Make sure that the control manifest
+# value is specified unless authentication is disabled.
+#
+# Possible return values:
+#
+# - 'project' if the project belongs to the submitter
+# - 'unknown' if the project name is not yet known
+# - 'disabled' if the owners directory is not configured
+# - <manifest> result manifest describing the authentication error
+#
+# Note that the authentication error result always starts with ':'.
+#
+function auth_project () # <project> <control> <repo-dir>
+{
+ trace_func "$@"
+
+ local prj="$1"
+ local ctl="${2%/}"
+ local rep="$3"
+
+ local d
+ d="$(owners_dir "$rep")"
+
+ if [ -z "$d" ]; then
+ echo "disabled"
+ return
+ fi
+
+ if [ -z "$ctl" ]; then
+ exit_with_manifest 400 "control manifest value expected"
+ fi
+
+ local r="unknown"
+ local m="$d/$prj/project-owner.manifest"
+
+ # If the project owner manifest exists then parse it and try to authenticate
+ # the submitter as the project owner.
+ #
+ if [ -f "$m" ]; then
+ ctl="$(repository_base "$ctl")"
+
+ # Parse the project owner manifest.
+ #
+ manifest_parser_start "$m"
+
+ local n v
+ while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
+ if [ "$n" == "control" -a "${v%/}" == "$ctl" ]; then
+ r="project"
+ break
+ fi
+ done
+
+ manifest_parser_finish
+
+ if [ "$r" != "project" ]; then
+ exit_with_manifest 401 "project owner authentication failed"
+ fi
+ fi
+
+ echo "$r"
+}
+
+# Authenticate the package name owner. Make sure that the control manifest
+# value is specified unless authentication is disabled. It is assumed that the
+# project ownership is already authenticated (possibly by another repository).
+#
+# Possible return values:
+#
+# - 'package' if the package belongs to the submitter
+# - 'unknown' if the package name is not taken in the project
+# - 'disabled' if the owners directory is not configured
+# - <manifest> result manifest describing the authentication error
+#
+# Note that the authentication error result always starts with ':'.
+#
+function auth_package () # <project> <package> <control> <repo-dir>
+{
+ trace_func "$@"
+
+ local prj="$1"
+ local pkg="$2"
+ local ctl="${3%/}"
+ local rep="$4"
+
+ local d
+ d="$(owners_dir "$rep")"
+
+ if [ -z "$d" ]; then
+ echo "disabled"
+ return
+ fi
+
+ if [ -z "$ctl" ]; then
+ exit_with_manifest 400 "control manifest value expected"
+ fi
+
+ local r="unknown"
+ local m="$d/$prj/$pkg/package-owner.manifest"
+
+ # If the package owner manifest exists then parse it and try to authenticate
+ # the submitter as the package owner.
+ #
+ if [ -f "$m" ]; then
+
+ # Parse the package owner manifest.
+ #
+ manifest_parser_start "$m"
+
+ local n v
+ while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
+ if [ "$n" == "control" -a "${v%/}" == "$ctl" ]; then
+ r="package"
+ break
+ fi
+ done
+
+ manifest_parser_finish
+
+ if [ "$r" != "package" ]; then
+ exit_with_manifest 401 "package owner authentication failed"
+ fi
+ fi
+
+ echo "$r"
+}
+
+# Check that the package name is unknown to the repository. Owners directory
+# is expected to be configured.
+#
+function auth_package_unknown () # <package> <repo-dir>
+{
+ trace_func "$@"
+
+ local pkg="$1"
+ local rep="$2"
+
+ local d
+ d="$(owners_dir "$rep")"
+
+ # Sanity check that the owners directory configured for the repository.
+ #
+ if [ -z "$d" ]; then
+ error "no owners directory configured for '$rep'"
+ fi
+
+ # While configured, the owners directory may not yet exist.
+ #
+ if [ -d "$d" ]; then
+ local f
+ f="$(run find "$d" -path "$d/*/$pkg/package-owner.manifest")"
+
+ if [ -n "$f" ]; then
+ trace "found: $f"
+ exit_with_manifest 401 "package owner authentication failed"
+ fi
+ fi
+}
+
+# Return lower-case URL scheme or empty string if the argument doesn't look
+# like a URL.
+#
+function url_scheme () # <url>
+{
+ sed -n -e 's%^\(.*\)://.*$%\L\1%p' <<<"$1"
+}
+
+# Checks that the repository properly responds to the probing request before
+# the timeout (in seconds). Noop for protocols other than HTTP(S).
+#
+function check_connectivity () # <repo-url> <timeout>
+{
+ trace_func "$@"
+
+ local url="$1"
+ local tmo="$2"
+
+ local s
+ s="$(url_scheme "$url")"
+
+ if [ "$s" == "http" -o "$s" == "https" ]; then
+ local u q
+
+ u="$(sed -n -e 's%^\([^?]*\).*$%\1%p' <<<"$url")" # Strips query part.
+ q="$(sed -n -e 's%^[^?]*\(.*\)$%\1%p' <<<"$url")" # Query part.
+
+ if [ -z "$q" ]; then
+ u="$u/info/refs?service=git-upload-pack"
+ else
+ u="$u/info/refs$q&service=git-upload-pack"
+ fi
+
+ # This function is called on repositories other than ours (e.g., control)
+ # so we don't want a failure to be logged.
+ #
+ if ! run_silent curl -S -s --max-time "$tmo" "$u" >/dev/null; then
+ exit_with_manifest 503 "submission service temporarily unavailable"
+ fi
+ fi
+}
diff --git a/brep/submit/submit-git.in b/brep/submit/submit-git.in
index fbb69de..c27edab 100644
--- a/brep/submit/submit-git.in
+++ b/brep/submit/submit-git.in
@@ -14,8 +14,8 @@
#
# The handler also implements the project/package name ownership verification
# by performing the submitter authentication/authorization based on the
-# control repository mechanism describe in bdep-publish(1). This functionality
-# is optional.
+# control repository mechanism described in bdep-publish(1). This
+# functionality is optional.
#
# The handler can operate with a single git repository, called "target", or
# with two git repositories, in which case the first is the target and the
@@ -64,20 +64,20 @@
# following structure:
#
# <owners>/
-# ├── <project1>/
-# │ ├── <package1>/
-# │ │ └── package-owner.manifest
-# │ ├── <package2>/
-# │ │ └── package-owner.manifest
-# │ ├── ...
-# │ └── project-owner.manifest
-# ├── <project2>/
-# │ └── ...
-# └──...
+# |-- <project1>/
+# | |-- <package1>/
+# | | `-- package-owner.manifest
+# | |-- <package2>/
+# | | `-- package-owner.manifest
+# | |-- ...
+# | `-- project-owner.manifest
+# |-- <project2>/
+# | `-- ...
+# `-- ...
#
# If the submitted project name is not yet known, then the handler script
# creates a new project subdirectory and saves project-owner.manifest. The
-# project owner manifest contain the following values in the specified order:
+# project owner manifest contains the following values in the specified order:
#
# name: <project-name>
# email: <submitter-email>
@@ -95,7 +95,7 @@
#
# Similarly, if the submitted package name is not yet known, then the handler
# script creates a new package subdirectory and saves package-owner.manifest.
-# The package owner manifest contain the following values in the specified
+# The package owner manifest contains the following values in the specified
# order:
#
# name: <package-name>
@@ -131,42 +131,464 @@
# add, commit, and push to reference and then remove, commit, and push to
# target.
#
-# On the handler side, before adding a package or new ownership for a
-# project/package name, the script re-checks the reference repository for
-# updated information.
+# On the handler side, the script acts in the opposite order cloning the
+# target prior pulling the reference in order not to get into the situation
+# where it misses the ownership info that is not in the reference yet but no
+# longer in the target. Note that if some move happens after the cloning,
+# then the script will be unable to push the target modification and will
+# re-try the whole authentication procedure from scratch.
+#
+# - Filesystem entries that exist or are created in the data directory:
+#
+# <pkg>-<ver>.tar.gz saved by brep (could be other archives in the future)
+# request.manifest created by brep
+# package.manifest extracted by the handler
+# target/ cloned by the handler
+# control/ cloned by the handler
+# result.manifest saved by brep
#
usage="usage: $0 <tgt-repo> [<ref-repo>] <dir>"
+# Diagnostics.
+#
+verbose= #true
+
+# Git network operations timeout (seconds).
+#
+# Note that we don't cover protocols other than HTTP(S) since for them git
+# doesn't support any timeouts (though we may be able to cobble something
+# up for SSH).
+#
+git_timeout=10
+
+# The reference repository lock timeout (seconds).
+#
+ref_lock_timeout=30
+
trap "{ exit 1; }" ERR
set -o errtrace # Trap ERR in functions.
-# Implementation notes:
+@import brep/submit/submit@
+@import brep/submit/submit-git@
+
+if [ "$#" -lt 2 -o "$#" -gt 3 ]; then
+ error "$usage"
+fi
+
+# Target repository URL.
#
-# - Check for duplicate package archive in all the sections. Before auth. Use
-# <name>-<version>.* instead of .tar.gz in case we support other formats
-# later.
+tgt_repo="$1"
+shift
+
+if [ -z "$tgt_repo" ]; then
+ error "$usage"
+fi
+
+# Reference repository directory.
#
-# - Push permission for target repo (add www-data to scm group)?
+# Note that the last argument is always the submission data directory.
#
-# - Network errors/timeouts on git pull for ref repo? What is the error (try
-# again)? I think also let's not assume target repo is local.
+ref_repo=
-# Workflow:
-#
-# 0. The same steps as submit-dir.
+if [ "$#" -gt 1 ]; then
+ ref_repo="$1"
+ shift
+
+ if [ -z "$ref_repo" ]; then
+ error "$usage"
+ fi
+
+ if [ ! -d "$ref_repo" ]; then
+ error "'$ref_repo' does not exist or is not a directory"
+ fi
+fi
+
+# Submission data directory.
#
-# 1. If ref-repo specified, lock, pull, and check:
-# - duplicate
-# - auth (read-only)
+data_dir="$1"
+shift
+
+if [ -z "$data_dir" ]; then
+ error "$usage"
+fi
+
+if [ ! -d "$data_dir" ]; then
+ error "'$data_dir' does not exist or is not a directory"
+fi
+
+reference="$(basename "$data_dir")"
+
+# Git verbosity options.
+#
+# Note that not all git commands support the -q/-v options. Also note that
+# these variable expansions should not be quoted.
+#
+if [ "$verbose" ]; then
+ gqo=
+ gvo="-v"
+else
+ gqo="-q"
+ gvo=
+fi
+
+# Git doesn't support the connection timeout option. The options we use are
+# just an approximation of the former, that, in particular, don't cover the
+# connection establishing. To work around this problem, before running a git
+# command that assumes the remote repository communication we manually check
+# connectivity with the remote repository.
+#
+git_http_timeout=("-c" "http.lowSpeedLimit=1" \
+ "-c" "http.lowSpeedTime=$git_timeout")
+
+# Parse the submission request manifest and obtain the required values.
#
-# 2. Clone tgt-repo, check:
-# - duplicate
-# - auth (read-write)
-# ? if fully auth'd by ref-repo, should we skip it here?
+manifest_parser_start "$data_dir/request.manifest"
+
+archive=
+sha256sum=
+section=
+email=
+control=
+simulate=
+
+while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
+ case "$n" in
+ archive) archive="$v" ;;
+ sha256sum) sha256sum="$v" ;;
+ section) section="$v" ;;
+ email) email="$v" ;;
+ control) control="${v%/}" ;;
+ simulate) simulate="$v" ;;
+ esac
+done
+
+manifest_parser_finish
+
+if [ -z "$archive" ]; then
+ error "archive manifest value expected"
+fi
+
+if [ -z "$sha256sum" ]; then
+ error "sha256sum manifest value expected"
+fi
+
+if [ -n "$simulate" -a "$simulate" != "success" ]; then
+ exit_with_manifest 400 "unrecognized simulation outcome '$simulate'"
+fi
+
+# Note: checking for section, email, and control later.
+
+m="$data_dir/package.manifest"
+extract_package_manifest "$data_dir/$archive" "$m"
+
+# Parse the package manifest and obtain the package name and version.
#
-# 3. Clone control branch and authorize.
+manifest_parser_start "$m"
+
+name=
+version=
+project=
+
+while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do
+ case "$n" in
+ name) name="$v" ;;
+ version) version="$v" ;;
+ project) project="$v" ;;
+ esac
+done
+
+manifest_parser_finish
+
+if [ -z "$name" ]; then
+ error "name manifest value expected"
+fi
+
+if [ -z "$version" ]; then
+ error "version manifest value expected"
+fi
+
+if [ -z "$project" ]; then
+ project="$name"
+fi
+
+function git_add () # <repo-dir> <path>...
+{
+ local d="$1"
+ shift
+
+ run git -C "$d" add $gvo "$@" >&2
+}
+
+function git_clone () # <repo-url> <repo-dir> <extra-options>...
+{
+ local url="$1"
+ shift
+
+ local dir="$1"
+ shift
+
+ check_connectivity "$url" "$git_timeout"
+ run git "${git_http_timeout[@]}" clone $gqo $gvo "$@" "$url" "$dir" >&2
+}
+
+# Dor now we make 10 re-tries to add the package and push to target. Push can
+# fail due to the target-to-reference information move race (see the above
+# notes for details) or because concurrent submissions. We may want to make it
+# configurable in the future.
#
-# 4. Copy archive, commit and push. If push fails, remove clone and
-# restart from 1 (yes, from ref-repo).
-# - put submission manifest into commit message for record?
+pkg_added=
+
+for i in {1..11}; do
+
+ # Clone the target repository.
+ #
+ tgt_dir="$data_dir/target"
+ git_clone "$tgt_repo" "$tgt_dir" --single-branch --depth 1
+
+ check_package_duplicate "$name" "$version" "$tgt_dir"
+
+ # Check for duplicates and try to authenticate the package ownership using
+ # information in the reference repository, if specified.
+ #
+ if [ -n "$ref_repo" ]; then
+
+ remote_url=$(git -C "$ref_repo" config --get remote.origin.url)
+
+ # Open the reading file descriptor and lock the repository. Fail if unable
+ # to lock before timeout.
+ #
+ l="$ref_repo/submit.config.bash"
+ trace "+ exec {fd}<$l"
+ exec {fd}<"$l"
+
+ if ! run flock -w "$ref_lock_timeout" "$fd"; then
+ exit_with_manifest 503 "submission service temporarily unavailable"
+ fi
+
+ # Pull the reference repository.
+ #
+ check_connectivity "$remote_url" "$git_timeout"
+ run git "${git_http_timeout[@]}" -C "$ref_repo" pull $gqo $gvo >&2
+
+ # Check the package duplicate.
+ #
+ check_package_duplicate "$name" "$version" "$ref_repo"
+
+ # Authenticate the project ownership.
+ #
+ auth="$(auth_project "$project" "$control" "$ref_repo")"
+
+ # Try to authenticate the package ownership if the project ownership was
+ # authenticated successfully.
+ #
+ if [ "$auth" == "project" ]; then
+ a="$(auth_package "$project" "$name" "$control" "$ref_repo")"
+
+ # If the package is unknown to this project, we will try to authenticate
+ # the package name with the target repository later and so we keep the
+ # 'project' auth state.
+ #
+ if [ "$a" != "unknown" ]; then
+ auth="$a"
+ fi
+ fi
+
+ trace "reference auth: $auth"
+
+ if [ "${auth:0:1}" == ":" ]; then # Authentication error?
+ echo "$auth"
+ exit 0
+ fi
+
+ # If the package is not present in the specified project then we need to
+ # make sure it is also not present in any other project.
+ #
+ if [ "$auth" == "project" -o "$auth" == "unknown" ]; then
+ auth_package_unknown "$name" "$ref_repo"
+ fi
+
+ trace "+ exec {fd}<&-"
+ exec {fd}<&- # Close the file descriptor and unlock the repository.
+ else
+ auth="disabled"
+ fi
+
+ ref_auth="$auth"
+
+ # Now try to authenticate the package ownership using information in the
+ # target repository unless already authenticated with reference.
+ #
+ if [ "$auth" != "package" ]; then
+
+ # Don't authenticate the project ownership if this is already done with
+ # the reference repository.
+ #
+ if [ "$auth" != "project" ]; then
+ auth="$(auth_project "$project" "$control" "$tgt_dir")"
+ fi
+
+ # Try to authenticate the package ownership if the project ownership was
+ # authenticated successfully.
+ #
+ if [ "$auth" == "project" ]; then
+ auth="$(auth_package "$project" "$name" "$control" "$tgt_dir")"
+ fi
+
+ trace "target auth: $auth"
+
+ if [ "${auth:0:1}" == ":" ]; then # Authentication error?
+ echo "$auth"
+ exit 0
+ fi
+ fi
+
+ trace "resulting auth: $auth"
+
+ # Sanity check the auth variable value.
+ #
+ case "$auth" in
+ package) ;;
+ unknown) ;;
+ disabled) ;;
+
+ *) error "unexpected resulting auth '$auth'";;
+ esac
+
+ # Establish ownership of the package name unless already done.
+ #
+ if [ "$auth" == "unknown" ]; then
+
+ # Check that the necessary request manifest values are specified.
+ #
+ if [ -z "$email" ]; then
+ exit_with_manifest 400 "email manifest value expected"
+ fi
+
+ # Check that the package doesn't belong yet to some other project.
+ #
+ auth_package_unknown "$name" "$tgt_dir"
+
+ # First the project name.
+ #
+ # Note that owners_dir() shouldn't return an empty string at this stage.
+ #
+ d="$(owners_dir "$tgt_dir")/$project"
+
+ # Establish ownership of the project name unless already done. Note that
+ # it can only be owned by the submitter at this stage.
+ #
+ prj_man="$d/project-owner.manifest"
+
+ if [ ! -f "$prj_man" ]; then
+ run mkdir -p "$d" # Also creates the owners directory if not exist.
+
+ ctl="$(repository_base "$control")"
+ create_owner_manifest "$project" "$email" "$ctl" "$prj_man"
+
+ # Add the project owners manifest file to git repository using the path
+ # relative to the repository directory.
+ #
+ git_add "$tgt_dir" "${prj_man#$tgt_dir/}"
+ fi
+
+ # Now the package name.
+ #
+ d="$d/$name"
+ run mkdir "$d"
+
+ pkg_man="$d/package-owner.manifest"
+ create_owner_manifest "$name" "$email" "$control" "$pkg_man"
+
+ # Add the package owners manifest file using path relative to the
+ # repository directory.
+ #
+ git_add "$tgt_dir" "${pkg_man#$tgt_dir/}"
+
+ auth="package"
+ fi
+
+ # Respond with the 'unauthorized' manifest if we failed to authenticate the
+ # submitter as the package owner, unless both the reference and target
+ # repositories have the ownership authentication disabled. In the latter
+ # case no authorization is required.
+ #
+ if [ "$auth" != "disabled" -o "$ref_auth" != "disabled" ]; then
+
+ # Respond with the 'unauthorized' manifest if not the package owner.
+ #
+ if [ "$auth" != "package" ]; then
+ if [ "$auth" == "project" -o "$ref_auth" == "project" ]; then
+ exit_with_manifest 401 "package owner authentication failed"
+ else
+ exit_with_manifest 401 "project owner authentication failed"
+ fi
+ fi
+
+ # Authorize the submission.
+ #
+ ctl_dir="$data_dir/control"
+
+ git_clone "$control" "$ctl_dir" --single-branch --depth 1 \
+ --branch "build2-control"
+
+ if [ ! -f "$ctl_dir/submit/${sha256sum:0:16}" ]; then
+ exit_with_manifest 401 "package publishing authorization failed"
+ fi
+ fi
+
+ # Add the package archive to the target repository.
+ #
+ s="$(section_dir "$section" "$tgt_dir")"
+
+ if [ -z "$s" ]; then
+ exit_with_manifest 400 "unrecognized section '$section'"
+ fi
+
+ d="$tgt_dir/$s/$project"
+ run mkdir -p "$d" # Create all the parent directories as well.
+
+ # We copy the archive rather than move it since we may need it for a re-try.
+ #
+ a="$d/$archive"
+ run cp "$data_dir/$archive" "$a"
+
+ git_add "$tgt_dir" "${a#$tgt_dir/}"
+
+ run git -C "$tgt_dir" commit $gqo $gvo -F - <<EOF >&2
+Add $name/$version to $s/$project
+
+$(cat "$data_dir/request.manifest")
+EOF
+
+ check_connectivity "$tgt_repo" "$git_timeout"
+
+ # Try to push the target modifications, unless simulating. If this succeeds
+ # then we are done. Otherwise, drop the target directory and re-try the
+ # whole authentication/authorization procedure, unless we are out of
+ # attempts.
+ #
+ if [ -z "$simulate" ]; then
+ if run_silent git "${git_http_timeout[@]}" -C "$tgt_dir" push >&2; then
+ pkg_added=true
+ break
+ else
+ run rm -r -f "$tgt_dir" "$ctl_dir"
+ fi
+ fi
+done
+
+if [ ! "$pkg_added" ]; then
+ exit_with_manifest 503 "submission service temporarily unavailable"
+fi
+
+# Remove the no longer needed submission data directory.
#
+run rm -r -f "$data_dir"
+
+if [ -n "$simulate" ]; then
+ trace "$name/$version submission is simulated"
+else
+ trace "$name/$version submission is queued"
+fi
+
+exit_with_manifest 200 "$name/$version submission is queued" "$reference"
diff --git a/brep/submit/submit.bash.in b/brep/submit/submit.bash.in
index 8315315..babf081 100644
--- a/brep/submit/submit.bash.in
+++ b/brep/submit/submit.bash.in
@@ -2,22 +2,24 @@
# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
# license : MIT; see accompanying LICENSE file
+# Utility functions useful for implementing submission handlers.
+
if [ "$brep_submit" ]; then
return 0
else
brep_submit=true
fi
+@import libbutl/manifest-parser@
@import libbutl/manifest-serializer@
# Diagnostics.
#
-
# We expect the user to set the verbose variable either to true or empty
# (false).
#
if [ ! -v verbose ]; then
- echo "variable 'verbose' is not set" 2>&1
+ echo "error: variable 'verbose' is not set" >&2
exit 1
fi
@@ -26,10 +28,13 @@ fi
# [Mon Jul 23 17:48:46.945079 2018] [brep:error] [pid 123:tid 456] [brep::submit::init]: error description
#
# We will use the (almost) same format for our diagnostics (redirected to the
-# Apache's error_log) so it can easily be attributed to the brep module.
+# Apache's error_log), so it can easily be attributed to the brep module.
#
info_self="$(basename $0)"
-info_ref="$(basename "${!#/}")" # Last argument is the submission directory.
+
+if [ "$#" -gt 0 ]; then
+ info_ref="$(basename "${!#/}")" # Last argument is the submission directory.
+fi
function info () # <severity> <text>
{
@@ -47,52 +52,146 @@ function info () # <severity> <text>
}
function error () { info "error" "$*"; exit 1; }
+
function trace () { if [ "$verbose" ]; then info "info" "$*"; fi }
+# Trace a command line, quoting empty arguments as well as those that contain
+# spaces.
+#
+function trace_cmd () # <cmd> <arg>...
+{
+ if [ "$verbose" ]; then
+ local s="+"
+ while [ $# -gt 0 ]; do
+ if [ -z "$1" -o -z "${1##* *}" ]; then
+ s="$s '$1'"
+ else
+ s="$s $1"
+ fi
+
+ shift
+ done
+
+ info "info" "$s"
+ fi
+}
+
+# Trace the current function name and arguments.
+#
+function trace_func () # <args>...
+{
+ trace_cmd "${FUNCNAME[1]}" "$@"
+}
+
+# Trace and run a command.
+#
+function run () # <cmd> <arg>...
+{
+ trace_cmd "$@"
+ "$@"
+}
+
+# Same as above but also redirect the command stderr to /dev/null, unless
+# running in the verbose mode.
+#
+# Note that we don't redirect stdout, so it can still be captured.
+#
+function run_silent () # <cmd> <arg>...
+{
+ trace_cmd "$@"
+
+ if [ "$verbose" ]; then
+ "$@"
+ else
+ "$@" 2>/dev/null
+ fi
+}
+
+# Wrap libbutl manifest parsing/serializing functions to shorten names and to
+# add tracing.
+#
+function manifest_parser_start () # [<file>]
+{
+ trace_func "$@"
+ butl_manifest_parser_start "$@"
+
+ manifest_parser_ofd="$butl_manifest_parser_ofd"
+}
+
+function manifest_parser_finish ()
+{
+ trace_func
+ butl_manifest_parser_finish
+}
+
+function manifest_serializer_start () # [<file>]
+{
+ trace_func "$@"
+ butl_manifest_serializer_start "$@"
+
+ manifest_serializer_ifd="$butl_manifest_serializer_ifd"
+}
+
+function manifest_serializer_finish ()
+{
+ trace_func
+ butl_manifest_serializer_finish
+}
+
# Serialize one manifest name/value pair.
#
-function serialize () # <name> <value>
+function manifest_serialize () # <name> <value>
{
- printf "%s:%s\0" "$1" "$2" >&"$butl_manifest_serializer_ifd"
+# trace "$1: $2"
+ printf "%s:%s\0" "$1" "$2" >&"$manifest_serializer_ifd"
}
-# Serialize the submission result manifest to stdout.
+# Serialize the submission result manifest to stdout and exit the (sub-)shell
+# with the zero status.
#
-function result_manifest () # <status> <message> [<reference>]
+function exit_with_manifest () # <status> <message> [<reference>]
{
+ trace_func "$@"
+
local sts="$1"
local msg="$2"
local ref="$3"
- trace "serializing result manifest"
- butl_manifest_serializer_start
+ manifest_serializer_start
- serialize "" "1" # Start of manifest.
- serialize "status" "$sts"
- serialize "message" "$msg"
+ manifest_serialize "" "1" # Start of manifest.
+ manifest_serialize "status" "$sts"
+ manifest_serialize "message" "$msg"
if [ -n "$ref" ]; then
- serialize "reference" "$ref"
+ if [ "$sts" != "200" ]; then
+ error "reference for code $sts"
+ fi
+
+ manifest_serialize "reference" "$ref"
+ elif [ "$sts" == "200" ]; then
+ error "no reference for code $sts"
fi
- butl_manifest_serializer_finish
+ manifest_serializer_finish
+ run exit 0
}
-# Verify the archive is a valid bpkg package and extract its manifest file.
+# Verify archive is a valid package and extract its manifest into
+# <manifest> file.
#
function extract_package_manifest () # <archive> <manifest>
{
local arc="$1"
local man="$2"
- # Should we remove the submission directory with an invalid package?
- # Probably it's better to leave it for potential investigation. Note that we
- # can always grep for such directories based on the result.manifest file
- # they contain.
- #
- if ! bpkg pkg-verify --manifest "$arc" >"$man" 2>/dev/null; then
- trace "$arc is not a valid package"
- result_manifest 400 "archive is not a valid package (run bpkg pkg-verify for details)"
- exit 0
+ if ! run_silent bpkg pkg-verify --manifest "$arc" >"$man"; then
+ # Perform the sanity check to make sure that bpkg is runnable.
+ #
+ if ! run bpkg --version >/dev/null; then
+ error "unable to run bpkg"
+ fi
+
+ exit_with_manifest 400 "archive is not a valid package (run bpkg pkg-verify for details)"
fi
}
diff --git a/doc/manual.cli b/doc/manual.cli
index 0a798bb..eba170d 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -75,10 +75,7 @@ temporary subdirectory next to the archive.|
Move/rename the temporary submission subdirectory to \c{submit-data} as an
atomic operation using the 12-character abbreviated checksum as its new
-name. If such a directory already exist, then this is a duplicate submission.
-
-Note also that once the directory is successfully moved, it is never removed
-by \c{brep}, even in case of a subsequent error.|
+name. If such a directory already exist, then this is a duplicate submission.|
\li|Invoke the submission handler program.
@@ -90,11 +87,27 @@ The handler program is expected to write the submission result manifest to
\c{stdout} and terminate with the zero exit status. A non-zero exit status is
treated as an internal error. The handler program's \c{stderr} is logged.
+Note that the handler program should report temporary server errors (service
+overload, network connectivity loss, etc.) via the submission result manifest
+status values in the [500-599] range (HTTP server error) rather than via a
+non-zero exit status.
+
The handler program assumes ownership of the submission directory and can
-move/remove it (for example, in case of an invalid submission). If after the
-handler program terminates the submission directory still exists, the
-submission result manifest is saved as \c{result.manifest} into this
-directory, next to the request manifest and archive.
+move/remove it. If after the handler program terminates the submission
+directory still exists, then it is handled by \c{brep} depending on the
+handler process exit status and the submission result manifest status value.
+If the process has terminated abnormally or with a non-zero exit status or the
+result manifest status is in the [500-599] range (HTTP server error), then the
+directory is saved for troubleshooting by appending a numeric extension to its
+name. Otherwise, if the status is in the [400-499] range (HTTP client error),
+then the directory is removed. If the directory is left in place by the
+handler or is saved for troubleshooting, then the submission result manifest
+is saved as \c{result.manifest} into this directory, next to the request
+manifest and archive.
+
+If \c{submit-handler-timeout} is configured and the handler program does not
+exit in the alloted time, then it is killed and its termination is treated as
+abnormal.
If the handler program is not specified, then the following submission result
manifest is implied (note that it is not saved):
diff --git a/etc/brep-module.conf b/etc/brep-module.conf
index 0612969..e5770fb 100644
--- a/etc/brep-module.conf
+++ b/etc/brep-module.conf
@@ -258,6 +258,13 @@ menu About=?about
# submit-handler-argument
+# The handler program timeout in seconds. If specified and the handler does
+# not exit in the alloted time, then it is killed and its termination is
+# treated as abnormal.
+#
+# submit-handler-timeout 60
+
+
# Trace verbosity. Disabled by default.
#
# verbosity 0
diff --git a/libbrep/common.hxx b/libbrep/common.hxx
index df838a7..5b172ec 100644
--- a/libbrep/common.hxx
+++ b/libbrep/common.hxx
@@ -104,8 +104,8 @@ namespace brep
#pragma db map type(dir_path) as(string) \
to((?).string ()) from(brep::dir_path (?))
- // Ensure that timestamp can be represented in nonoseconds without loss of
- // accuracy, so the following ODB mapping is adequate.
+ // Make sure that timestamp can be represented in nonoseconds without loss
+ // of accuracy, so the following ODB mapping is adequate.
//
static_assert(
std::ratio_greater_equal<timestamp::period,
diff --git a/mod/mod-submit.cxx b/mod/mod-submit.cxx
index accd56d..c68b773 100644
--- a/mod/mod-submit.cxx
+++ b/mod/mod-submit.cxx
@@ -4,8 +4,16 @@
#include <mod/mod-submit.hxx>
-#include <cstdlib> // strtoul()
+#include <sys/time.h> // timeval
+#include <sys/select.h>
+
+#include <ratio> // ratio_greater_equal
+#include <chrono>
+#include <cstdlib> // strtoul()
#include <istream>
+#include <sstream>
+#include <type_traits> // static_assert
+#include <system_error> // error_code, generic_category()
#include <libbutl/sha256.mxx>
#include <libbutl/process.mxx>
@@ -501,10 +509,56 @@ handle (request& rq, response& rs)
}
// Given that the submission data is now successfully persisted we are no
- // longer in charge of removing it, even in case of a subsequent error.
+ // longer in charge of removing it, except for the cases when the submission
+ // handler terminates with an error (see below for details).
//
tdr.cancel ();
+ // If the handler terminates with non-zero exit status or specifies 5XX
+ // (HTTP server error) submission result manifest status value, then we
+ // stash the submission data directory for troubleshooting. Otherwise, if
+ // it's the 4XX (HTTP client error) status value, then we remove the
+ // directory.
+ //
+ // Note that leaving the directory in place in case of a submission error
+ // would have prevent the user from re-submitting until we research the
+ // issue and manually remove the directory.
+ //
+ auto stash_submit_dir = [&dd, error] ()
+ {
+ try
+ {
+ if (!dir_exists (dd))
+ return;
+
+ for (size_t n (1); true; ++n) // Eventually we should find the free one.
+ {
+ string ext ('.' + to_string (n));
+ dir_path d (dd + ext);
+
+ if (!dir_exists (d))
+ try
+ {
+ mvdir (dd, d);
+ break;
+ }
+ catch (const system_error& e)
+ {
+ int ec (e.code ().value ());
+ if (ec != ENOTEMPTY && ec != EEXIST) // Note: there can be a race.
+ throw;
+ }
+ }
+ }
+ catch (const system_error& e)
+ {
+ // Not much we can do here. Let's just log the issue and bail out
+ // leaving the directory in place.
+ //
+ error << "unable to rename directory '" << dd << "': " << e;
+ }
+ };
+
auto print_args = [&trace, this] (const char* args[], size_t n)
{
l2 ([&]{trace << process_args {args, n};});
@@ -518,13 +572,44 @@ handle (request& rq, response& rs)
// containing at least the status value. Thus, an empty cache indicates that
// the handler is not configured.
//
- status_code sc;
+ status_code sc (200);
vector<manifest_name_value> rvs;
if (options_->submit_handler_specified ())
{
+ // For the sake of the documentation we will call the handler's normal
+ // exit with 0 code "successful termination".
+ //
+ // To make sure the handler process execution doesn't exceed the specified
+ // timeout we set the non-blocking mode for the process stdout-reading
+ // stream, try to read from it with the 10 milliseconds timeout and check
+ // the process execution time between the reads. We then kill the process
+ // if the execution time is exceeded.
+ //
+ using namespace chrono;
+
+ using time_point = system_clock::time_point;
+ using duration = system_clock::duration;
+
+ // Make sure that the system clock has at least milliseconds resolution.
+ //
+ static_assert(
+ ratio_greater_equal<milliseconds::period, duration::period>::value,
+ "The system clock resolution is too low");
+
+ optional<milliseconds> timeout;
+
+ if (options_->submit_handler_timeout_specified ())
+ timeout = milliseconds (options_->submit_handler_timeout () * 1000);
+
const path& handler (options_->submit_handler ());
+ // Note that due to the non-blocking mode we cannot just pass the stream
+ // to the manifest parser constructor. So we buffer the data in the string
+ // stream and then parse that.
+ //
+ stringstream ss;
+
for (;;) // Breakout loop.
try
{
@@ -542,86 +627,177 @@ handle (request& rq, response& rs)
dd));
pipe.out.close ();
- try
+ auto kill = [&pr, &warn, &handler, &ac] ()
{
- ifdstream is (move (pipe.in));
-
- // Parse and verify the manifest. Obtain the HTTP status code (must go
- // first) and cache it for the subsequent responding to the client.
- //
- parser p (is, "handler");
- manifest_name_value nv (p.next ());
-
- auto bad_value ([&p, &nv] (const string& d) {
- throw parsing (p.name (), nv.value_line, nv.value_column, d);});
-
- if (nv.empty ())
- bad_value ("empty manifest");
-
- const string& n (nv.name);
- const string& v (nv.value);
-
- // The format version pair is verified by the parser.
- //
- assert (n.empty () && v == "1");
-
- // Cache the format version pair.
- //
- rvs.push_back (move (nv));
-
- // Get and verify the HTTP status.
- //
- nv = p.next ();
- if (n != "status")
- bad_value ("no status specified");
-
- char* e (nullptr);
- unsigned long c (strtoul (v.c_str (), &e, 10)); // Can't throw.
-
- assert (e != nullptr);
-
- if (!(*e == '\0' && c >= 100 && c < 600))
- bad_value ("invalid http status '" + v + "'");
-
- // Cache the HTTP status.
+ // We may still end up well (see below), thus this is a warning.
//
- sc = static_cast<status_code> (c);
- rvs.push_back (move (nv));
+ warn << "ref " << ac << ": process " << handler
+ << " execution timeout expired";
- // Cache the remaining name/value pairs.
- //
- for (nv = p.next (); !nv.empty (); nv = p.next ())
- rvs.push_back (move (nv));
+ pr.kill ();
+ };
- // Cache end of manifest.
+ try
+ {
+ ifdstream is (move (pipe.in), fdstream_mode::non_blocking);
+
+ const size_t nbuf (8192);
+ char buf[nbuf];
+
+ while (is.is_open ())
+ {
+ time_point start;
+ milliseconds wd (10); // Max time to wait for the data portion.
+
+ if (timeout)
+ {
+ start = system_clock::now ();
+
+ if (*timeout < wd)
+ wd = *timeout;
+ }
+
+ timeval tm {wd.count () / 1000 /* seconds */,
+ wd.count () % 1000 * 1000 /* microseconds */};
+
+ fd_set rd;
+ FD_ZERO (&rd);
+ FD_SET (is.fd (), &rd);
+
+ int r (select (is.fd () + 1, &rd, nullptr, nullptr, &tm));
+
+ if (r == -1)
+ {
+ // Don't fail if the select() call was interrupted by the signal.
+ //
+ if (errno != EINTR)
+ throw io_error ("select failed",
+ error_code (errno, generic_category ()));
+ }
+ else if (r != 0) // Is data available?
+ {
+ assert (FD_ISSET (is.fd (), &rd));
+
+ // The only leagal way to read from non-blocking ifdstream.
+ //
+ streamsize n (is.readsome (buf, nbuf));
+
+ // Close the stream (and bail out) if the end of the data is
+ // reached. Otherwise cache the read data.
+ //
+ if (is.eof ())
+ is.close ();
+ else
+ {
+ // The data must be available.
+ //
+ // Note that we could keep reading until the readsome() call
+ // returns 0. However, this way we could potentially exceed the
+ // timeout significantly for some broken handler that floods us
+ // with data. So instead, we will be checking the process
+ // execution time after every data chunk read.
+ //
+ assert (n != 0);
+
+ ss.write (buf, n);
+ }
+ }
+ else // Timeout occured.
+ {
+ // Normally, we don't expect timeout to occur on the pipe read
+ // operation if the process has terminated successfully, as all its
+ // output must already be buffered (including eof). However, there
+ // can be some still running handler's child that has inherited
+ // the parent's stdout. In this case we assume that we have read
+ // all the handler's output, close the stream, log the warning and
+ // bail out.
+ //
+ if (pr.exit)
+ {
+ // We keep reading only upon successful handler termination.
+ //
+ assert (*pr.exit);
+
+ is.close ();
+
+ warn << "ref " << ac << ": process " << handler
+ << " stdout is not closed after termination (possibly "
+ << "handler's child still running)";
+ }
+ }
+
+ if (timeout)
+ {
+ time_point now (system_clock::now ());
+
+ // Assume we have waited the full amount if the time adjustment is
+ // detected.
+ //
+ duration d (now > start ? now - start : wd);
+
+ // If the timeout is not fully exhausted, then decrement it and
+ // try to read some more data from the handler' stdout. Otherwise,
+ // kill the process, if not done yet.
+ //
+ // Note that it may happen that we are killing an already
+ // terminated process, in which case kill() just sets the process
+ // exit information. On the other hand it's guaranteed that the
+ // process is terminated after the kill() call, and so the pipe is
+ // presumably closed on the write end (see above for details).
+ // Thus, if the process terminated successfully, we will continue
+ // reading until eof is reached or read timeout occurred. Yes, it
+ // may happen that we end up with a successful submission even
+ // with the kill.
+ //
+ if (*timeout > d)
+ *timeout -= duration_cast<milliseconds> (d);
+ else if (!pr.exit)
+ {
+ kill ();
+
+ assert (pr.exit);
+
+ // Close the stream (and bail out) if the process hasn't
+ // terminate successfully.
+ //
+ if (!*pr.exit)
+ is.close ();
+
+ *timeout = milliseconds::zero ();
+ }
+ }
+ }
+
+ assert (!is.is_open ());
+
+ if (!timeout)
+ pr.wait ();
+
+ // If the process is not terminated yet, then wait for its termination
+ // for the remaining time. Kill it if the timeout has been exceeded
+ // and the process still hasn't terminate.
//
- rvs.push_back (move (nv));
+ else if (!pr.exit && !pr.timed_wait (*timeout))
+ kill ();
- is.close ();
+ assert (pr.exit); // The process must finally be terminated.
- if (pr.wait ())
+ if (*pr.exit)
break; // Get out of the breakout loop.
- assert (pr.exit);
- error << "process " << handler << " " << *pr.exit;
-
- // Fall through.
- }
- catch (const parsing& e)
- {
- if (pr.wait ())
- error << "unable to parse handler's output: " << e;
+ error << "ref " << ac << ": process " << handler << " " << *pr.exit;
// Fall through.
}
catch (const io_error& e)
{
if (pr.wait ())
- error << "unable to read handler's output: " << e;
+ error << "ref " << ac << ": unable to read handler's output: " << e;
// Fall through.
}
+ stash_submit_dir ();
return respond_error ();
}
// Handle process_error and io_error (both derive from system_error).
@@ -629,6 +805,73 @@ handle (request& rq, response& rs)
catch (const system_error& e)
{
error << "unable to execute '" << handler << "': " << e;
+
+ stash_submit_dir ();
+ return respond_error ();
+ }
+
+ try
+ {
+ // Parse and verify the manifest. Obtain the HTTP status code (must go
+ // first) and cache it for the subsequent response to the client.
+ //
+ parser p (ss, "handler");
+ manifest_name_value nv (p.next ());
+
+ auto bad_value ([&p, &nv] (const string& d) {
+ throw parsing (p.name (), nv.value_line, nv.value_column, d);});
+
+ if (nv.empty ())
+ bad_value ("empty manifest");
+
+ const string& n (nv.name);
+ const string& v (nv.value);
+
+ // The format version pair is verified by the parser.
+ //
+ assert (n.empty () && v == "1");
+
+ // Cache the format version pair.
+ //
+ rvs.push_back (move (nv));
+
+ // Get and verify the HTTP status.
+ //
+ nv = p.next ();
+ if (n != "status")
+ bad_value ("no status specified");
+
+ char* e (nullptr);
+ unsigned long c (strtoul (v.c_str (), &e, 10)); // Can't throw.
+
+ assert (e != nullptr);
+
+ if (!(*e == '\0' && c >= 100 && c < 600))
+ bad_value ("invalid HTTP status '" + v + "'");
+
+ // Cache the HTTP status.
+ //
+ sc = static_cast<status_code> (c);
+ rvs.push_back (move (nv));
+
+ // Cache the remaining name/value pairs.
+ //
+ for (nv = p.next (); !nv.empty (); nv = p.next ())
+ rvs.push_back (move (nv));
+
+ // Cache end of manifest.
+ //
+ rvs.push_back (move (nv));
+ }
+ catch (const parsing& e)
+ {
+ error << "ref " << ac << ": unable to parse handler's output: " << e;
+
+ // It appears the handler had misbehaved, so let's stash the submission
+ // directory for troubleshooting.
+ //
+ stash_submit_dir ();
+
return respond_error ();
}
}
@@ -637,7 +880,7 @@ handle (request& rq, response& rs)
// serialization error log the error description and return false, on the
// stream error pass through the io_error exception, otherwise return true.
//
- auto rsm = [&rvs, &error] (ostream& os) -> bool
+ auto rsm = [&rvs, &error, &ac] (ostream& os) -> bool
{
assert (!rvs.empty ());
@@ -651,30 +894,52 @@ handle (request& rq, response& rs)
}
catch (const serialization& e)
{
- error << "unable to serialize handler's output: " << e;
+ error << "ref " << ac << ": unable to serialize handler's output: " << e;
return false;
}
};
- // Save the result manifest, if generated, into the submission directory
- // if it still exists (note that the handler could move or remove it).
+ // If the submission data directory still exists then perform an appropriate
+ // action on it, depending on the submission result status. Note that the
+ // handler could move or remove the directory.
//
- path rsf (dd / "result.manifest");
-
- if (!rvs.empty () && dir_exists (dd))
- try
+ if (dir_exists (dd))
{
- ofdstream os (rsf);
- bool r (rsm (os));
- os.close ();
+ // Remove the directory if the client error is detected.
+ //
+ if (sc >= 400 && sc < 500)
+ rmdir_r (dd);
- if (!r)
- return respond_error (); // The error description is already logged.
- }
- catch (const io_error& e)
- {
- error << "unable to write to '" << rsf << "': " << e;
- return respond_error ();
+ // Otherwise, save the result manifest, if generated, into the directory.
+ // Also stash the directory for troubleshooting in case of the server
+ // error.
+ //
+ else
+ {
+ path rsf (dd / "result.manifest");
+
+ if (!rvs.empty ())
+ try
+ {
+ ofdstream os (rsf);
+
+ // Not being able to stash the result manifest is not a reason to
+ // claim the submission failed. The error is logged nevertheless.
+ //
+ rsm (os);
+
+ os.close ();
+ }
+ catch (const io_error& e)
+ {
+ // Not fatal (see above).
+ //
+ error << "unable to write to '" << rsf << "': " << e;
+ }
+
+ if (sc >= 500 && sc < 600)
+ stash_submit_dir ();
+ }
}
// Send email, if configured, and the submission is not simulated.
diff --git a/mod/options.cli b/mod/options.cli
index 992ffc4..59aceb6 100644
--- a/mod/options.cli
+++ b/mod/options.cli
@@ -458,6 +458,14 @@ namespace brep
(see \cb{submit-handler} for details). Repeat this option to specify
multiple arguments."
}
+
+ size_t submit-handler-timeout
+ {
+ "<seconds>",
+ "The handler program timeout in seconds. If specified and the handler
+ does not exit in the alloted time, then it is killed and its
+ termination is treated as abnormal."
+ }
};
class repository_root: handler
diff --git a/tests/submit/0f6b1460b3ec/libhello-0.1.0.tar.gz b/tests/submit/0f6b1460b3ec/libhello-0.1.0.tar.gz
new file mode 100644
index 0000000..604a536
--- /dev/null
+++ b/tests/submit/0f6b1460b3ec/libhello-0.1.0.tar.gz
Binary files differ
diff --git a/tests/submit/0f6b1460b3ec/package.manifest b/tests/submit/0f6b1460b3ec/package.manifest
new file mode 100644
index 0000000..6fc36e7
--- /dev/null
+++ b/tests/submit/0f6b1460b3ec/package.manifest
@@ -0,0 +1,10 @@
+: 1
+name: libhello
+version: 0.1.0
+project: hello
+summary: hello library
+license: TODO
+url: https://example.org/hello
+email: user@example.org
+depends: * build2 >= 0.8.0-
+depends: * bpkg >= 0.8.0-
diff --git a/tests/submit/0f6b1460b3ec/request.manifest b/tests/submit/0f6b1460b3ec/request.manifest
new file mode 100644
index 0000000..1173210
--- /dev/null
+++ b/tests/submit/0f6b1460b3ec/request.manifest
@@ -0,0 +1,10 @@
+: 1
+archive: libhello-0.1.0.tar.gz
+sha256sum: 0f6b1460b3ec479499ae26841322af7a8a4312fcc5a6e890f3dbba91a83b38cc
+timestamp: 2018-08-20T04:57:05Z
+client-ip: fe80::56e1:adff:fe83:82f5
+user-agent: bdep/0.8.0-a.0.20180815130917 (GNU/Linux; +https://build2.org)\
+ libbpkg/0.8.0-a.0.0f50af28d1cf libbutl/0.8.0-a.0.3e0db12932d5 curl
+section: alpha
+email: user@example.org
+control: http://example.org/hello.git
diff --git a/tests/submit/0f6b1460b3ec/result.manifest b/tests/submit/0f6b1460b3ec/result.manifest
new file mode 100644
index 0000000..93a3555
--- /dev/null
+++ b/tests/submit/0f6b1460b3ec/result.manifest
@@ -0,0 +1,4 @@
+: 1
+status: 200
+message: libhello/0.1.0 submission is queued
+reference: 0f6b1460b3ec
diff --git a/tests/submit/README b/tests/submit/README
new file mode 100644
index 0000000..7286c83
--- /dev/null
+++ b/tests/submit/README
@@ -0,0 +1,30 @@
+Prepare the test data with the following instructions.
+
+In an empty directory run:
+
+$ bdep new -t empty -C @cfg hello
+$ BDEP_EMAIL=user@example.org bdep new --package -t lib libhello -d hello
+$ bdep init -d hello/libhello
+
+Edit hello/libhello/manifest setting version to 0.1.0.
+
+$ mkdir hello.git
+$ git -C hello.git/ init --bare
+
+$ git -C hello remote add origin "$(pwd)/hello.git"
+$ git -C hello add '*'
+$ git -C hello commit -m "Create"
+$ git -C hello push --set-upstream origin master
+
+tar cf hello.tar.gz hello.git/
+
+Move the archive into brep/tests/submit/ directory.
+
+Locally run brep server configured to use submit-dir handler.
+
+$ bdep publish --control http://example.org/hello.git \
+ --email user@example.org --repository http://localhost/pkg --yes \
+ -d hello
+
+Replace the submission data directory in brep/tests/submit/ with the one
+produced with the above command.
diff --git a/tests/submit/buildfile b/tests/submit/buildfile
new file mode 100644
index 0000000..6606153
--- /dev/null
+++ b/tests/submit/buildfile
@@ -0,0 +1,16 @@
+# file : tests/submit/buildfile
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+define common: file
+common{*}: extension = test
+
+dir = ../../brep/submit/
+
+commons = data
+
+./: test{* -{$commons}} common{$commons} {*/ -test/}{**} \
+ $dir/exe{brep-submit-dir} $dir/exe{brep-submit-git}
+
+test{submit-dir}@./: test = $out_base/$dir/brep-submit-dir
+test{submit-git}@./: test = $out_base/$dir/brep-submit-git
diff --git a/tests/submit/data.test b/tests/submit/data.test
new file mode 100644
index 0000000..938d6b8
--- /dev/null
+++ b/tests/submit/data.test
@@ -0,0 +1,35 @@
+# file : tests/submit/data.test
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+# Pre-created submission data directory that will be copied by subsequent
+# tests and scope setup commands. The common approach will be that group
+# scopes copy and modify the parent scope submission directory as required by
+# the nested tests and scopes. Tests will also clone the parent scope
+# submission data directory to optionally modify it, use and cleanup at the
+# end. Note that configuration can not be shared between multiple submission
+# handler processes. Also we need to make sure that submission data
+# directories are not cloned while being used by submission handler scripts.
+#
+data_dir = $regex.replace($path_search('*/request.manifest', $src_base), \
+ '(.*)/.*', \
+ '\1')
+
+checksum = "$data_dir"
+
+# Copy the original submission data directory to the root scope.
+#
++cp -r $src_base/$data_dir ./
+
+root_data_dir = $~/$data_dir
+
+# The most commonly used submission data directory cloning command that copies
+# it from the parent scope working directory.
+#
+clone_data = cp --no-cleanup -r ../$data_dir ./
+clone_data_clean = cp --no-cleanup -r ../$data_dir ./ &$data_dir/***
+
+# Clones the original submission data directory.
+#
+clone_root_data = cp --no-cleanup -r $root_data_dir ./
+clone_root_data_clean = cp --no-cleanup -r $root_data_dir ./ &$data_dir/***
diff --git a/tests/submit/hello.tar.gz b/tests/submit/hello.tar.gz
new file mode 100644
index 0000000..67baca7
--- /dev/null
+++ b/tests/submit/hello.tar.gz
Binary files differ
diff --git a/tests/submit/submit-dir.test b/tests/submit/submit-dir.test
new file mode 100644
index 0000000..97f7edd
--- /dev/null
+++ b/tests/submit/submit-dir.test
@@ -0,0 +1,90 @@
+# file : tests/submit/submit-dir.test
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+.include data.test
+
+: args
+{
+ : none
+ :
+ $* 2>>~%EOE% != 0
+ %\[.+\] \[brep:error\] \[ref \] \[brep-submit-dir\]: usage: .+brep-submit-dir <dir>%
+ EOE
+
+ : not-exist
+ :
+ $* $~/dir 2>>~%EOE% != 0
+ %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-dir\]: '.+dir' does not exist or is not a directory%
+ EOE
+}
+
+: success
+:
+{
+ test.arguments += $checksum
+
+ : simulate
+ :
+ {
+ $clone_root_data;
+
+ echo "simulate: success" >+$checksum/request.manifest;
+
+ $* >>"EOO";
+ : 1
+ status: 200
+ message: libhello/0.1.0 submission is queued
+ reference: $checksum
+ EOO
+
+ test -d $checksum != 0
+ }
+
+ : for-real
+ :
+ {
+ $clone_root_data_clean;
+
+ $* >>"EOO"
+ : 1
+ status: 200
+ message: libhello/0.1.0 submission is queued
+ reference: $checksum
+ EOO
+ }
+}
+
+: failure
+:
+{
+ test.arguments += $checksum
+
+ : bad-archive
+ :
+ {
+ $clone_root_data_clean;
+
+ echo "junk" >=$checksum/libhello-0.1.0.tar.gz;
+
+ $* >>EOO
+ : 1
+ status: 400
+ message: archive is not a valid package (run bpkg pkg-verify for details)
+ EOO
+ }
+
+ : bad-simulate
+ :
+ {
+ $clone_root_data_clean;
+
+ echo "simulate: fly" >+$checksum/request.manifest;
+
+ $* >>"EOO"
+ : 1
+ status: 400
+ message: unrecognized simulation outcome 'fly'
+ EOO
+ }
+}
diff --git a/tests/submit/submit-git.test b/tests/submit/submit-git.test
new file mode 100644
index 0000000..cf499b9
--- /dev/null
+++ b/tests/submit/submit-git.test
@@ -0,0 +1,765 @@
+# file : tests/submit/submit-git.test
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+.include data.test
+
+# Prior to running testscript with -v for debugging purposes assign true to
+# the verbosity variable in the brep/submit/submit-git.in handler script and
+# uncomment the following line.
+#
+# test.redirects += 2>!
+
+g = git 2>! >&2
+
+# Create and clone the reference repository.
+#
+root_ref = $~/ref.git
+root_ref_dir = $~/ref
+
+clone_root_ref = cp --no-cleanup -r $root_ref ./ &ref.git/***
+
++mkdir --no-cleanup $root_ref
++$g -C $root_ref init --bare &ref.git/***
+
++$g clone $root_ref $root_ref_dir &ref/***
+
++cat <<EOI >=$root_ref_dir/submit.config.bash
+ sections[alpha]=1/alpha
+ sections[beta]=1/beta
+ sections[stable]=1/testing
+
+ owners=owners
+ EOI
+
++$g -C $root_ref_dir add '*'
++$g -C $root_ref_dir commit -m 'Add submit.config.bash'
++$g -C $root_ref_dir push
+
+# Create the target repository.
+#
+root_tgt = $~/tgt.git
+root_tgt_url = "file:///$~/tgt.git"
+
++cp -r $root_ref $root_tgt
+
+clone_root_tgt = cp --no-cleanup -r $root_tgt ./ &tgt.git/***
+
+# Extract the package repository.
+#
++tar -C $~ -xf $src_base/hello.tar.gz &hello.git/***
+
+# Adjust the request manifest control value to point to the package repository.
+#
+prj_ctl="file://$~"
+pkg_ctl="$prj_ctl/hello.git"
+
++sed -i -e "s%^\(control:\) .+\$%\\1 $pkg_ctl%" $data_dir/request.manifest
+
+: args
+{
+ : none
+ :
+ $* 2>>~%EOE% != 0
+ %\[.+\] \[brep:error\] \[ref \] \[brep-submit-git\]: usage: .+brep-submit-git <tgt-repo> \[<ref-repo>\] <dir>%
+ EOE
+
+ : dir-only
+ :
+ $* $~/dir 2>>~%EOE% != 0
+ %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-git\]: usage: .+brep-submit-git <tgt-repo> \[<ref-repo>\] <dir>%
+ EOE
+
+ : ref-not-exist
+ :
+ $* "$root_tgt_url" ref $~/dir 2>>~%EOE% != 0
+ %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-git\]: 'ref' does not exist or is not a directory%
+ EOE
+}
+
+: success
+:
+{
+ : ref-unknown-tgt-aquire-prj-pkg
+ :
+ : Test that on the first package submission the project and package names
+ : ownership is successfully aquired. Authentication is enabled on both the
+ : reference and target repos.
+ :
+ {
+ $clone_root_data;
+
+ $clone_root_tgt;
+ tgt_url = "file:///$~/tgt.git";
+
+ $* "$tgt_url" $root_ref_dir $data_dir >>"EOO";
+ : 1
+ status: 200
+ message: libhello/0.1.0 submission is queued
+ reference: $checksum
+ EOO
+
+ # Check that the ownership information and the package are commited into
+ # the target repository.
+ #
+ $g clone "$tgt_url" &tgt/***;
+
+ # Note that some manifest values may well wrap over several lines.
+ #
+ cat tgt/owners/hello/project-owner.manifest >>~%EOO%;
+ : 1
+ name: hello
+ email: user@example.org
+ %control: file:///.+%
+ %.*
+ EOO
+
+ cat tgt/owners/hello/libhello/package-owner.manifest >>~%EOO%;
+ : 1
+ name: libhello
+ email: user@example.org
+ %control: file:///.+%
+ %.*
+ EOO
+
+ test -f tgt/1/alpha/hello/libhello-0.1.0.tar.gz;
+
+ git -C tgt log -1 >>~%EOO%
+ %commit .+%
+ %Author: .+%
+ %Date: .+%
+
+ Add libhello/0.1.0 to 1/alpha/hello
+ % %
+ : 1
+ archive: libhello-0.1.0.tar.gz
+ % sha256sum: .{64}%
+ % timestamp: ....-..-..T..:..:..Z%
+ % client-ip: .+%
+ % user-agent: bdep/.+%
+ %.
+ section: alpha
+ email: user@example.org
+ % control: file:///.+%
+ %.*
+ EOO
+ }
+ : ref-disabled-tgt-aquire-prj-pkg
+ :
+ : Test that on the first package submit the project and package names
+ : ownership is successfully aquired. Authentication is disabled for the
+ : reference repo.
+ :
+ {
+ $clone_root_data;
+
+ $clone_root_ref;
+ $g clone ref.git &ref/***;
+
+ cat <<EOI >=ref/submit.config.bash;
+ sections[alpha]=1/alpha
+ sections[beta]=1/beta
+ sections[stable]=1/testing
+
+ # owners=owners
+ EOI
+
+ $g -C ref commit -am 'Disable ownership';
+ $g -C ref push;
+
+ $clone_root_tgt;
+
+ $* "file:///$~/tgt.git" ref $data_dir >>"EOO"
+ : 1
+ status: 200
+ message: libhello/0.1.0 submission is queued
+ reference: $checksum
+ EOO
+ }
+
+ : ref-absent-tgt-aquire-prj-pkg
+ :
+ : Test that on the first package submit the project and package names
+ : ownership is successfully aquired. Reference repo is absent.
+ :
+ {
+ $clone_root_data;
+ $clone_root_tgt;
+
+ $* "file:///$~/tgt.git" $data_dir >>"EOO"
+ : 1
+ status: 200
+ message: libhello/0.1.0 submission is queued
+ reference: $checksum
+ EOO
+ }
+
+ : ref-unknown-tgt-auth-prj-pkg
+ :
+ : Test that the project and package ownership is authenticated by the target
+ : repository.
+ :
+ {
+ $clone_root_data;
+
+ $clone_root_tgt;
+ $g clone tgt.git &tgt/***;
+
+ mkdir -p tgt/owners/hello/libhello;
+
+ cat <<"EOI" >=tgt/owners/hello/project-owner.manifest;
+ : 1
+ name: hello
+ email: user@example.org
+ control: $prj_ctl/
+ EOI
+
+ cat <<"EOI" >=tgt/owners/hello/libhello/package-owner.manifest;
+ : 1
+ name: libhello
+ email: user@example.org
+ control: $pkg_ctl
+ EOI
+
+ $g -C tgt add owners;
+ $g -C tgt commit -m 'Add ownership info';
+ $g -C tgt push;
+
+ $* "file:///$~/tgt.git" $root_ref_dir $data_dir >>"EOO"
+ : 1
+ status: 200
+ message: libhello/0.1.0 submission is queued
+ reference: $checksum
+ EOO
+ }
+
+ : ref-auth-prj-tgt-auth-pkg
+ :
+ : Test that the project ownersip is authenticated by the reference
+ : repository and the package ownersip is authenticated by the target
+ : repository.
+ :
+ {
+ $clone_root_data;
+
+ $clone_root_ref;
+ $g clone ref.git &ref/***;
+
+ mkdir -p ref/owners/hello;
+
+ cat <<"EOI" >=ref/owners/hello/project-owner.manifest;
+ : 1
+ name: hello
+ email: user@example.org
+ control: $prj_ctl
+ EOI
+
+ $g -C ref add owners;
+ $g -C ref commit -m 'Add ownership info';
+ $g -C ref push;
+
+ $clone_root_tgt;
+ $g clone tgt.git &tgt/***;
+
+ mkdir -p tgt/owners/hello/libhello;
+
+ cat <<"EOI" >=tgt/owners/hello/libhello/package-owner.manifest;
+ : 1
+ name: libhello
+ email: user@example.org
+ control: $pkg_ctl
+ EOI
+
+ $g -C tgt add owners;
+ $g -C tgt commit -m 'Add ownership info';
+ $g -C tgt push;
+
+ $* "file:///$~/tgt.git" ref $data_dir >>"EOO"
+ : 1
+ status: 200
+ message: libhello/0.1.0 submission is queued
+ reference: $checksum
+ EOO
+ }
+
+ : ref-auth-prj-pkg
+ :
+ : Test that the project and package ownership is authenticated by the
+ : reference repository.
+ :
+ {
+ $clone_root_data;
+
+ $clone_root_ref;
+ $g clone ref.git &ref/***;
+
+ mkdir -p ref/owners/hello/libhello;
+
+ cat <<"EOI" >=ref/owners/hello/project-owner.manifest;
+ : 1
+ name: hello
+ email: user@example.org
+ control: $prj_ctl
+ EOI
+
+ cat <<"EOI" >=ref/owners/hello/libhello/package-owner.manifest;
+ : 1
+ name: libhello
+ email: user@example.org
+ control: $pkg_ctl
+ EOI
+
+ $g -C ref add owners;
+ $g -C ref commit -m 'Add ownership info';
+ $g -C ref push;
+
+ $clone_root_tgt;
+
+ $* "file:///$~/tgt.git" ref $data_dir >>"EOO"
+ : 1
+ status: 200
+ message: libhello/0.1.0 submission is queued
+ reference: $checksum
+ EOO
+ }
+
+ : ref-auth-prj-tgt-aquire-pkg
+ :
+ : Test that the project ownersip is authenticated by the reference
+ : repository and the package ownersip is aquired.
+ :
+ {
+ $clone_root_data;
+
+ $clone_root_ref;
+ $g clone ref.git &ref/***;
+
+ mkdir -p ref/owners/hello/libhello;
+
+ cat <<"EOI" >=ref/owners/hello/project-owner.manifest;
+ : 1
+ name: hello
+ email: user@example.org
+ control: $prj_ctl
+ EOI
+
+ $g -C ref add owners;
+ $g -C ref commit -m 'Add ownership info';
+ $g -C ref push;
+
+ $clone_root_tgt;
+
+ $* "file:///$~/tgt.git" ref $data_dir >>"EOO"
+ : 1
+ status: 200
+ message: libhello/0.1.0 submission is queued
+ reference: $checksum
+ EOO
+ }
+
+ : ref-absent-tgt-disabled
+ :
+ : Test the package ownership authentication when reference is unspecified and
+ : the target ownership handling is disabled.
+ :
+ {
+ $clone_root_data;
+
+ $clone_root_tgt;
+ $g clone tgt.git &tgt/***;
+
+ cat <<EOI >=tgt/submit.config.bash;
+ sections[alpha]=1/alpha
+ sections[beta]=1/beta
+ sections[stable]=1/testing
+
+ # owners=owners
+ EOI
+
+ $g -C tgt commit -am 'Disable ownership';
+ $g -C tgt push;
+
+ $* "file:///$~/tgt.git" $data_dir >>"EOO"
+ : 1
+ status: 200
+ message: libhello/0.1.0 submission is queued
+ reference: $checksum
+ EOO
+ }
+
+ : section-fallback
+ :
+ {
+ $clone_root_data;
+ sed -i -e "s%^\(section:\) .+\$%\\1 delta%" $data_dir/request.manifest;
+
+ $clone_root_tgt;
+ $g clone tgt.git &tgt/***;
+
+ cat <<EOI >=tgt/submit.config.bash;
+ sections[alpha]=1/alpha
+ sections[beta]=1/beta
+ sections[stable]=1/testing
+ sections['*']=1/junk
+
+ owners=owners
+ EOI
+
+ $g -C tgt commit -am "Add section name fallback";
+ $g -C tgt push;
+
+ $* "file:///$~/tgt.git" $root_ref_dir $data_dir >>"EOO"
+ : 1
+ status: 200
+ message: libhello/0.1.0 submission is queued
+ reference: $checksum
+ EOO
+ }
+}
+
+: failure
+:
+{
+ : ref-dup-pkg
+ :
+ : Test the duplicate submission due presence of the package archive in the
+ : reference repo.
+ :
+ {
+ $clone_root_data_clean;
+
+ $clone_root_ref;
+ $g clone ref.git &ref/***;
+
+ mkdir -p ref/1/alpha/hello;
+ cp $data_dir/libhello-0.1.0.tar.gz ref/1/alpha/hello/;
+
+ $g -C ref add 1/;
+ $g -C ref commit -m 'Add libhello-0.1.0.tar.gz';
+ $g -C ref push;
+
+ $* "$root_tgt_url" $~/ref $data_dir >>EOO
+ : 1
+ status: 422
+ message: duplicate submission
+ EOO
+ }
+
+ : ref-used-pkg
+ :
+ : Test the package ownership authentication failure using the reference
+ : repo. The package name is already used in other project.
+ :
+ {
+ $clone_root_data_clean;
+
+ $clone_root_ref;
+ $g clone ref.git &ref/***;
+
+ mkdir -p ref/owners/hi/libhello;
+
+ cat <<"EOI" >=ref/owners/hi/project-owner.manifest;
+ : 1
+ name: hi
+ email: user@example.org
+ control: $prj_ctl
+ EOI
+
+ cat <<"EOI" >=ref/owners/hi/libhello/package-owner.manifest;
+ : 1
+ name: libhello
+ email: user@example.org
+ control: $prj_ctl/foo
+ EOI
+
+ $g -C ref add owners;
+ $g -C ref commit -m 'Add ownership info';
+ $g -C ref push;
+
+ $* "$root_tgt_url" $~/ref $data_dir >>EOO
+ : 1
+ status: 401
+ message: package owner authentication failed
+ EOO
+ }
+
+ : ref-auth-prj
+ :
+ : Test the project ownership authentication failure using the reference
+ : repo.
+ :
+ {
+ $clone_root_data_clean;
+
+ $clone_root_ref;
+ $g clone ref.git &ref/***;
+
+ mkdir -p ref/owners/hello;
+ cat <<EOI >=ref/owners/hello/project-owner.manifest;
+ : 1
+ name: hello
+ email: user@example.org
+ control: https://example.com/foo
+ EOI
+
+ $g -C ref add owners/hello/project-owner.manifest;
+ $g -C ref commit -m 'Add project ownership info';
+ $g -C ref push;
+
+ $* "$root_tgt_url" $~/ref $data_dir >>EOO
+ : 1
+ status: 401
+ message: project owner authentication failed
+ EOO
+ }
+
+ : ref-auth-pkg
+ :
+ : Test the package ownership authentication failure using the reference
+ : repo.
+ :
+ {
+ $clone_root_data_clean;
+
+ $clone_root_ref;
+ $g clone ref.git &ref/***;
+
+ mkdir -p ref/owners/hello/libhello;
+
+ cat <<"EOI" >=ref/owners/hello/project-owner.manifest;
+ : 1
+ name: hello
+ email: user@example.org
+ control: $prj_ctl
+ EOI
+
+ cat <<"EOI" >=ref/owners/hello/libhello/package-owner.manifest;
+ : 1
+ name: libhello
+ email: user@example.org
+ control: $prj_ctl/foo
+ EOI
+
+ $g -C ref add owners;
+ $g -C ref commit -m 'Add ownership info';
+ $g -C ref push;
+
+ $* "$root_tgt_url" $~/ref $data_dir >>EOO
+ : 1
+ status: 401
+ message: package owner authentication failed
+ EOO
+ }
+
+ : ref-absent-tgt-dup-pkg
+ :
+ : Test the duplicate submission due presence of the package archive in the
+ : target repo.
+ :
+ {
+ $clone_root_data_clean;
+
+ $clone_root_tgt;
+ $g clone tgt.git &tgt/***;
+
+ mkdir -p tgt/1/alpha/hello;
+ cp $data_dir/libhello-0.1.0.tar.gz tgt/1/alpha/hello/;
+
+ $g -C tgt add 1/;
+ $g -C tgt commit -m 'Add libhello-0.1.0.tar.gz';
+ $g -C tgt push;
+
+ $* "file:///$~/tgt.git" $data_dir >>EOO
+ : 1
+ status: 422
+ message: duplicate submission
+ EOO
+ }
+
+ : ref-absent-tgt-auth-pkg
+ :
+ : Test the package ownership authentication failure using the target repo.
+ :
+ {
+ $clone_root_data_clean;
+
+ $clone_root_tgt;
+ $g clone tgt.git &tgt/***;
+
+ mkdir -p tgt/owners/hello/libhello;
+
+ cat <<"EOI" >=tgt/owners/hello/project-owner.manifest;
+ : 1
+ name: hello
+ email: user@example.org
+ control: $prj_ctl
+ EOI
+
+ cat <<"EOI" >=tgt/owners/hello/libhello/package-owner.manifest;
+ : 1
+ name: libhello
+ email: user@example.org
+ control: $prj_ctl/foo
+ EOI
+
+ $g -C tgt add owners;
+ $g -C tgt commit -m 'Add ownership info';
+ $g -C tgt push;
+
+ $* "file:///$~/tgt.git" $data_dir >>EOO
+ : 1
+ status: 401
+ message: package owner authentication failed
+ EOO
+ }
+
+ : ref-unknown-tgt-disabled
+ :
+ : Test the project ownership authentication failure when no project
+ : ownership information is present in the reference and the target ownership
+ : handling is disabled.
+ :
+ {
+ $clone_root_data_clean;
+
+ $clone_root_tgt;
+ $g clone tgt.git &tgt/***;
+
+ cat <<EOI >=tgt/submit.config.bash;
+ sections[alpha]=1/alpha
+ sections[beta]=1/beta
+ sections[stable]=1/testing
+
+ # owners=owners
+ EOI
+
+ $g -C tgt commit -am 'Disable ownership';
+ $g -C tgt push;
+
+ $* "file:///$~/tgt.git" $root_ref_dir $data_dir >>EOO
+ : 1
+ status: 401
+ message: project owner authentication failed
+ EOO
+ }
+
+ : ref-prj-tgt-disabled
+ :
+ : Test the project ownership authentication failure when no package
+ : ownership information is present in the reference and the target ownership
+ : handling is disabled.
+ :
+ {
+ $clone_root_data_clean;
+
+ $clone_root_ref;
+ $g clone ref.git &ref/***;
+
+ mkdir -p ref/owners/hello/libhello;
+
+ cat <<"EOI" >=ref/owners/hello/project-owner.manifest;
+ : 1
+ name: hello
+ email: user@example.org
+ control: $prj_ctl
+ EOI
+
+ $g -C ref add owners;
+ $g -C ref commit -m 'Add project ownership info';
+ $g -C ref push;
+
+ $clone_root_tgt;
+ $g clone tgt.git &tgt/***;
+
+ cat <<EOI >=tgt/submit.config.bash;
+ sections[alpha]=1/alpha
+ sections[beta]=1/beta
+ sections[stable]=1/testing
+
+ # owners=owners
+ EOI
+
+ $g -C tgt commit -am 'Disable ownership';
+ $g -C tgt push;
+
+ $* "file:///$~/tgt.git" ref $data_dir >>EOO
+ : 1
+ status: 401
+ message: package owner authentication failed
+ EOO
+ }
+
+ : ref-absent-tgt-used-pkg
+ :
+ : Test the package ownership authentication failure using the target repo.
+ : The package name is already used in other project.
+ :
+ {
+ $clone_root_data_clean;
+
+ $clone_root_tgt;
+ $g clone tgt.git &tgt/***;
+
+ mkdir -p tgt/owners/hi/libhello;
+
+ cat <<"EOI" >=tgt/owners/hi/project-owner.manifest;
+ : 1
+ name: hi
+ email: user@example.org
+ control: $prj_ctl
+ EOI
+
+ cat <<"EOI" >=tgt/owners/hi/libhello/package-owner.manifest;
+ : 1
+ name: libhello
+ email: user@example.org
+ control: $prj_ctl/foo
+ EOI
+
+ $g -C tgt add owners;
+ $g -C tgt commit -m 'Add ownership info';
+ $g -C tgt push;
+
+ $* "file:///$~/tgt.git" $data_dir >>EOO
+ : 1
+ status: 401
+ message: package owner authentication failed
+ EOO
+ }
+
+ : authorization
+ :
+ : Test the package submission authorization failure due to the archive
+ : abbreviated checksum mismatch.
+ :
+ {
+ $clone_root_data_clean;
+
+ sed -i -e "s%^\(sha256sum:\) .+\$%\\1 59941e842667%" \
+ $data_dir/request.manifest;
+
+ $clone_root_tgt;
+ $g clone tgt.git &tgt/***;
+
+ $* "file:///$~/tgt.git" $data_dir >>EOO
+ : 1
+ status: 401
+ message: package publishing authorization failed
+ EOO
+ }
+
+ : section-unknown
+ :
+ {
+ $clone_root_data_clean;
+ sed -i -e "s%^\(section:\) .+\$%\\1 delta%" $data_dir/request.manifest;
+
+ $clone_root_tgt;
+
+ $* "file:///$~/tgt.git" $root_ref_dir $data_dir >>"EOO"
+ : 1
+ status: 400
+ message: unrecognized section 'delta'
+ EOO
+ }
+}
generated by cgit v1.1 at 2024-11-10 19:30:42 +0000