diff options
Diffstat (limited to 'brep')
| -rw-r--r-- | brep/handler/buildfile | 4 | ||||
| -rw-r--r-- | brep/handler/ci/buildfile | 1 | ||||
| -rw-r--r-- | brep/handler/ci/ci-dir.in | 1 | ||||
| -rw-r--r-- | brep/handler/ci/ci-load.in | 96 | ||||
| -rw-r--r-- | brep/handler/ci/ci.bash.in | 1 | ||||
| -rw-r--r-- | brep/handler/handler.bash.in | 14 | ||||
| -rw-r--r-- | brep/handler/submit/.gitignore | 1 | ||||
| -rw-r--r-- | brep/handler/submit/buildfile | 5 | ||||
| -rw-r--r-- | brep/handler/submit/submit-dir.in | 10 | ||||
| -rw-r--r-- | brep/handler/submit/submit-git.bash.in | 102 | ||||
| -rw-r--r-- | brep/handler/submit/submit-git.in | 50 | ||||
| -rw-r--r-- | brep/handler/submit/submit-pub.in | 435 | ||||
| -rw-r--r-- | brep/handler/submit/submit.bash.in | 57 | ||||
| -rw-r--r-- | brep/handler/upload/.gitignore | 2 | ||||
| -rw-r--r-- | brep/handler/upload/buildfile | 13 | ||||
| -rw-r--r-- | brep/handler/upload/upload-bindist-clean.in | 224 | ||||
| -rw-r--r-- | brep/handler/upload/upload-bindist.in | 595 | ||||
| -rw-r--r-- | brep/handler/upload/upload.bash.in | 40 |
18 files changed, 1547 insertions, 104 deletions
diff --git a/brep/handler/buildfile b/brep/handler/buildfile index 3b9245b..cd11231 100644 --- a/brep/handler/buildfile +++ b/brep/handler/buildfile @@ -1,10 +1,10 @@ # file : brep/handler/buildfile -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file import mods = libbutl.bash%bash{manifest-parser} import mods += libbutl.bash%bash{manifest-serializer} +import mods += bpkg-util%bash{package-archive} -./: bash{handler} submit/ ci/ +./: bash{handler} submit/ ci/ upload/ bash{handler}: in{handler} $mods diff --git a/brep/handler/ci/buildfile b/brep/handler/ci/buildfile index 3ed6807..69234d6 100644 --- a/brep/handler/ci/buildfile +++ b/brep/handler/ci/buildfile @@ -1,5 +1,4 @@ # file : brep/handler/ci/buildfile -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file ./: exe{brep-ci-dir} exe{brep-ci-load} diff --git a/brep/handler/ci/ci-dir.in b/brep/handler/ci/ci-dir.in index 47387ea..58aa991 100644 --- a/brep/handler/ci/ci-dir.in +++ b/brep/handler/ci/ci-dir.in @@ -1,7 +1,6 @@ #!/usr/bin/env bash # file : brep/handler/ci/ci-dir.in -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file # Simple package CI request handler with directory storage. diff --git a/brep/handler/ci/ci-load.in b/brep/handler/ci/ci-load.in index 915f9a6..3f04ea8 100644 --- a/brep/handler/ci/ci-load.in +++ b/brep/handler/ci/ci-load.in @@ -1,7 +1,6 @@ #!/usr/bin/env bash # file : brep/handler/ci/ci-load.in -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file # Package CI request handler that loads the packages into the brep database. @@ -26,7 +25,10 @@ verbose= #true fetch_timeout=60 trap "{ exit 1; }" ERR -set -o errtrace # Trap ERR in functions. +set -o errtrace # Trap ERR in functions. +set -o pipefail # Fail if any pipeline command fails. +shopt -s lastpipe # Execute last pipeline command in the current shell. +shopt -s nullglob # Expand no-match globs to nothing rather than themselves. @import brep/handler/handler@ @import brep/handler/ci/ci@ @@ -34,7 +36,7 @@ set -o errtrace # Trap ERR in functions. # The handler's own options. # result_url= -while [ $# -gt 0 ]; do +while [[ "$#" -gt 0 ]]; do case $1 in --result-url) shift @@ -51,7 +53,7 @@ done # loader="$1" -if [ -z "$loader" ]; then +if [[ -z "$loader" ]]; then error "$usage" fi @@ -61,7 +63,7 @@ shift # options. # loader_options=() -while [ $# -gt 1 ]; do +while [[ "$#" -gt 1 ]]; do loader_options+=("$1") shift done @@ -70,11 +72,11 @@ done # data_dir="${1%/}" -if [ -z "$data_dir" ]; then +if [[ -z "$data_dir" ]]; then error "$usage" fi -if [ ! -d "$data_dir" ]; then +if [[ ! -d "$data_dir" ]]; then error "'$data_dir' does not exist or is not a directory" fi @@ -85,8 +87,9 @@ reference="$(basename "$data_dir")" # manifest_parser_start "$data_dir/request.manifest" -simulate= repository= +interactive= +simulate= # Package map. We first enter packages from the request manifest as keys and # setting the values to true. Then we go through the repository package list @@ -105,40 +108,58 @@ declare -A packages # spec= +# Third party service information which, if specified, needs to be associated +# with the being created tenant. +# +service_id= +service_type= +service_data= + while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do case "$n" in - simulate) simulate="$v" ;; - repository) repository="$v" ;; + repository) repository="$v" ;; + interactive) interactive="$v" ;; + simulate) simulate="$v" ;; package) packages["$v"]=true - if [ -n "$spec" ]; then + if [[ -n "$spec" ]]; then spec="$spec," fi spec="$spec$v" ;; + + service-id) service_id="$v" ;; + service-type) service_type="$v" ;; + service-data) service_data="$v" ;; esac done manifest_parser_finish -if [ -n "$spec" ]; then +if [[ -n "$spec" ]]; then spec="$spec@" fi spec="$spec$repository" -if [ -z "$repository" ]; then +if [[ -z "$repository" ]]; then error "repository manifest value expected" fi -if [ -n "$simulate" -a "$simulate" != "success" ]; then +if [[ -n "$simulate" && "$simulate" != "success" ]]; then exit_with_manifest 400 "unrecognized simulation outcome '$simulate'" fi +# Use the generated reference if the tenant service id is not specified. +# +if [[ -n "$service_type" && -z "$service_id" ]]; then + service_id="$reference" +fi + message_suffix= -if [ -n "$result_url" ]; then +if [[ -n "$result_url" ]]; then message_suffix=": $result_url/@$reference" # Append the tenant id. fi @@ -147,7 +168,7 @@ fi # Note that we can't assume a real repository URL is specified if simulating # so trying to query the repository info is not a good idea. # -if [ -n "$simulate" ]; then +if [[ -n "$simulate" ]]; then run rm -r "$data_dir" trace "CI request for '$spec' is simulated$message_suffix" @@ -189,9 +210,9 @@ manifest_values=() manifest_version= more=true -while [ "$more" ]; do +while [[ "$more" ]]; do - if [ -n "$manifest_version" ]; then + if [[ -n "$manifest_version" ]]; then manifest_names=("") manifest_values=("$manifest_version") fi @@ -214,35 +235,32 @@ while [ "$more" ]; do manifest_names+=("$n") manifest_values+=("$v") - done # Reduce the first manifest case. # - if [ ${#manifest_names[@]} -eq 0 ]; then + if [[ "${#manifest_names[@]}" -eq 0 ]]; then continue fi # Add or filter out the manifest, if present. # - if [ ${#packages[@]} -ne 0 ]; then - - if [[ -v packages["$name"] ]]; then + if [[ "${#packages[@]}" -ne 0 ]]; then + if [[ -v "packages[$name]" ]]; then packages["$name"]= packages["$name/$version"]= # Clear it either, as may also be present. - elif [[ -v packages["$name/$version"] ]]; then + elif [[ -v "packages[$name/$version]" ]]; then packages["$name/$version"]= else continue # Skip. fi - fi packages_manifest_names+=("${manifest_names[@]}") packages_manifest_values+=("${manifest_values[@]}") - if [ -z "$display_name" ]; then - if [ -n "$project" ]; then + if [[ -z "$display_name" ]]; then + if [[ -n "$project" ]]; then display_name="$project" else display_name="$name" @@ -256,7 +274,7 @@ manifest_parser_finish # the repository. # for p in "${!packages[@]}"; do - if [ "${packages[$p]}" ]; then + if [[ "${packages[$p]}" ]]; then exit_with_manifest 422 "unknown package $p" fi done @@ -264,7 +282,7 @@ done # Verify that the repository is not empty. Failed that, the repository display # name wouldn't be set. # -if [ -z "$display_name" ]; then +if [[ -z "$display_name" ]]; then exit_with_manifest 422 "no packages in repository" fi @@ -276,7 +294,7 @@ run mv "$cache_dir/packages.manifest" "$cache_dir/packages.manifest.orig" # manifest_serializer_start "$cache_dir/packages.manifest" -for ((i=0; i <= ${#packages_manifest_names[@]}; ++i)); do +for ((i=0; i != "${#packages_manifest_names[@]}"; ++i)); do manifest_serialize "${packages_manifest_names[$i]}" \ "${packages_manifest_values[$i]}" done @@ -290,7 +308,7 @@ run echo "$repository $display_name cache:cache" >"$loadtab" # Apply overrides, if uploaded. # -if [ -f "$data_dir/overrides.manifest" ]; then +if [[ -f "$data_dir/overrides.manifest" ]]; then loader_options+=(--overrides-file "$data_dir/overrides.manifest") fi @@ -299,6 +317,22 @@ fi # loader_options+=(--force --shallow --tenant "$reference") +# Build the packages interactively, if requested. +# +if [[ -n "$interactive" ]]; then + loader_options+=(--interactive "$interactive") +fi + +# Pass the tenant service information, if specified, to the loader. +# +if [[ -n "$service_id" ]]; then + loader_options+=(--service-id "$service_id" --service-type "$service_type") + + if [[ -n "$service_data" ]]; then + loader_options+=(--service-data "$service_data") + fi +fi + run "$loader" "${loader_options[@]}" "$loadtab" # Remove the no longer needed CI request data directory. diff --git a/brep/handler/ci/ci.bash.in b/brep/handler/ci/ci.bash.in index c188ab9..4ed5fab 100644 --- a/brep/handler/ci/ci.bash.in +++ b/brep/handler/ci/ci.bash.in @@ -1,5 +1,4 @@ # file : brep/handler/ci/ci.bash.in -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file # Utility functions useful for implementing CI request handlers. diff --git a/brep/handler/handler.bash.in b/brep/handler/handler.bash.in index 2e66afc..d9e7eaa 100644 --- a/brep/handler/handler.bash.in +++ b/brep/handler/handler.bash.in @@ -1,5 +1,4 @@ # file : brep/handler/handler.bash.in -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file # Utility functions useful for implementing request handlers. @@ -10,8 +9,11 @@ else brep_handler=true fi -@import libbutl/manifest-parser@ -@import libbutl/manifest-serializer@ +@import libbutl.bash/manifest-parser@ +@import libbutl.bash/manifest-serializer@ + +bpkg_util_bpkg=bpkg +@import bpkg-util/package-archive@ # Diagnostics. # @@ -52,7 +54,7 @@ function info () # <severity> <text> ts= fi - echo "[$ts] [brep:$severity] [ref $info_ref] [$info_self]: $*" 1>&2; + echo "[$ts] [brep:$severity] [ref $info_ref] [$info_self]: $*" 1>&2 } function error () { info "error" "$*"; exit 1; } @@ -149,3 +151,7 @@ function manifest_serialize () # <name> <value> # trace "$1: $2" printf "%s:%s\0" "$1" "$2" >&"$manifest_serializer_ifd" } + +function pkg_verify_archive () { bpkg_util_pkg_verify_archive "$@"; } +function pkg_find_archives () { bpkg_util_pkg_find_archives "$@"; } +function pkg_find_archive () { bpkg_util_pkg_find_archive "$@"; } diff --git a/brep/handler/submit/.gitignore b/brep/handler/submit/.gitignore index cbbd541..098bf75 100644 --- a/brep/handler/submit/.gitignore +++ b/brep/handler/submit/.gitignore @@ -1,2 +1,3 @@ brep-submit-dir brep-submit-git +brep-submit-pub diff --git a/brep/handler/submit/buildfile b/brep/handler/submit/buildfile index 7951a46..1747c64 100644 --- a/brep/handler/submit/buildfile +++ b/brep/handler/submit/buildfile @@ -1,8 +1,7 @@ # file : brep/handler/submit/buildfile -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -./: exe{brep-submit-dir} exe{brep-submit-git} +./: exe{brep-submit-dir} exe{brep-submit-git} exe{brep-submit-pub} include ../ @@ -11,5 +10,7 @@ exe{brep-submit-dir}: in{submit-dir} bash{submit} ../bash{handler} exe{brep-submit-git}: in{submit-git} \ bash{submit-git} bash{submit} ../bash{handler} +exe{brep-submit-pub}: in{submit-pub} bash{submit} ../bash{handler} + bash{submit}: in{submit} ../bash{handler} bash{submit-git}: in{submit-git} bash{submit} ../bash{handler} diff --git a/brep/handler/submit/submit-dir.in b/brep/handler/submit/submit-dir.in index ae0dcbd..b28ab38 100644 --- a/brep/handler/submit/submit-dir.in +++ b/brep/handler/submit/submit-dir.in @@ -1,7 +1,6 @@ #!/usr/bin/env bash # file : brep/handler/submit/submit-dir.in -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file # Simple package submission handler with directory storage. @@ -67,20 +66,17 @@ fi m="$data_dir/package.manifest" extract_package_manifest "$data_dir/$archive" "$m" -# Parse the package manifest and obtain the package name, version, and -# project. +# Parse the package manifest and obtain the package name and version. # manifest_parser_start "$m" name= version= -project= while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do case "$n" in name) name="$v" ;; version) version="$v" ;; - project) project="$v" ;; esac done @@ -94,10 +90,6 @@ if [ -z "$version" ]; then error "version manifest value expected" fi -if [ -z "$project" ]; then - project="$name" -fi - if [ -n "$simulate" ]; then run rm -r "$data_dir" trace "package submission is simulated: $name/$version" diff --git a/brep/handler/submit/submit-git.bash.in b/brep/handler/submit/submit-git.bash.in index 56cce33..cf7300d 100644 --- a/brep/handler/submit/submit-git.bash.in +++ b/brep/handler/submit/submit-git.bash.in @@ -1,5 +1,4 @@ # file : brep/handler/submit/submit-git.bash.in -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file # Utility functions for the submit-git handler. @@ -60,6 +59,10 @@ function owners_dir () # <repo-dir> # Check if a repository already contains the package. Respond with the # 'duplicate submission' result manifest and exit if that's the case. # +# Also check if the repository contains newer revision of this package +# version. Respond with the 'newer revision is present' result manifest and +# exit if that's the case. +# function check_package_duplicate () # <name> <version> <repo-dir> { trace_func "$@" @@ -73,22 +76,54 @@ function check_package_duplicate () # <name> <version> <repo-dir> run source "$rep/submit.config.bash" - # Check for duplicate package in all sections. Use <name>-<version>.* - # without .tar.gz in case we want to support more archive types later. + local rev + rev="$(version_revision "$ver")" + + # Check for duplicate package and its newer revisions in all sections. Use + # <name>-<version>.* without .tar.gz in case we want to support more archive + # types later. # local s for s in "${!sections[@]}"; do local d="$rep/${sections[$s]}" - if [ -d "$d" ]; then - local f - f="$(run find "$d" -name "$nam-$ver.*")" + # Check for duplicate. + # + local p + run pkg_find_archive "$nam-$ver.*" "$d" | readarray -t p + + if [ "${#p[@]}" -ne 0 ]; then + local n="${p[1]}" + local v="${p[2]}" - if [ -n "$f" ]; then - trace "found: $f" + trace "found: $n/$v in ${p[0]}" + + if [ "$n" == "$nam" ]; then exit_with_manifest 422 "duplicate submission" + else + exit_with_manifest 422 "submission conflicts with $n/$v" fi fi + + # Check for newer revision. + # + local arcs + run pkg_find_archives "$nam" "$ver*" "$d" | readarray -t arcs + + local f + for f in "${arcs[@]}"; do + local p + pkg_verify_archive "$f" | readarray -t p + + local v="${p[1]}" + + local rv + rv="$(version_revision "$v")" + + if [ "$rv" -gt "$rev" ]; then + exit_with_manifest 422 "newer revision $nam/$v is present" + fi + done done } @@ -164,6 +199,7 @@ function auth_project () # <project> <control> <repo-dir> local r="unknown" local m="$d/$prj/project-owner.manifest" + local info= # If the project owner manifest exists then parse it and try to authenticate # the submitter as the project owner. @@ -176,16 +212,31 @@ function auth_project () # <project> <control> <repo-dir> local n v while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do - if [[ "$n" == "control" && "$ctl" == "$v"* ]]; then - r="project" - break + if [[ "$n" == "control" ]]; then + if [[ "$ctl" == "$v"* ]]; then + r="project" + break + fi + + # If the control URLs don't match, then compare them case- + # insensitively, converting them to the lower case. If they match + # case-insensitively, then still fail the authentication but provide + # additional information in the manifest message value. + # + if [[ "${ctl,,}" == "${v,,}"* ]]; then + info=" + info: control repository URL differs only in character case + info: submitted URL: $ctl + info: project owner's URL: $v + info: consider using --control to specify exact URL" + fi fi done manifest_parser_finish if [ "$r" != "project" ]; then - exit_with_manifest 401 "project owner authentication failed" + exit_with_manifest 401 "project owner authentication failed$info" fi fi @@ -211,7 +262,8 @@ function auth_package () # <project> <package> <control> <repo-dir> local prj="$1" local pkg="$2" - local ctl="${3%.git}" # Strip the potential .git extension. + local ctl="${3%.git}" # For comparison strip the potential .git extension. + local ctl_orig="$3" # For diagnostics use the original URL. local rep="$4" local d @@ -228,6 +280,7 @@ function auth_package () # <project> <package> <control> <repo-dir> local r="unknown" local m="$d/$prj/$pkg/package-owner.manifest" + local info= # If the package owner manifest exists then parse it and try to authenticate # the submitter as the package owner. @@ -242,16 +295,31 @@ function auth_package () # <project> <package> <control> <repo-dir> # local n v while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do - if [ "$n" == "control" -a "${v%.git}" == "$ctl" ]; then - r="package" - break + if [ "$n" == "control" ]; then + local u="${v%.git}" + + if [ "$u" == "$ctl" ]; then + r="package" + break + fi + + # If the control URLs don't match, then compare them case- + # insensitively (see auth_project() for details). + # + if [ "${u,,}" == "${ctl,,}" ]; then + info=" + info: control repository URL differs only in character case + info: submitted URL: $ctl_orig + info: package owner's URL: $v + info: consider using --control to specify exact URL" + fi fi done manifest_parser_finish if [ "$r" != "package" ]; then - exit_with_manifest 401 "package owner authentication failed" + exit_with_manifest 401 "package owner authentication failed$info" fi fi diff --git a/brep/handler/submit/submit-git.in b/brep/handler/submit/submit-git.in index 8263efe..c882b84 100644 --- a/brep/handler/submit/submit-git.in +++ b/brep/handler/submit/submit-git.in @@ -1,7 +1,6 @@ #!/usr/bin/env bash # file : brep/handler/submit/submit-git.in -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file # Package submission handler with git repository storage. @@ -187,8 +186,10 @@ git_timeout=10 ref_lock_timeout=30 trap "{ exit 1; }" ERR -set -o errtrace # Trap ERR in functions. -set -o pipefail # Return the rightmost non-zero exit status in a pipeline. +set -o errtrace # Trap in functions and subshells. +set -o pipefail # Fail if any pipeline command fails. +shopt -s lastpipe # Execute last pipeline command in the current shell. +shopt -s nullglob # Expand no-match globs to nothing rather than themselves. @import brep/handler/handler@ @import brep/handler/submit/submit@ @@ -404,7 +405,7 @@ function git_add () # <repo-dir> <path>... local d="$1" shift - run git -C "$d" add $gvo "$@" >&2 + run git -C "$d" add --force $gvo "$@" >&2 } # For now we make 10 re-tries to add the package and push to target. Push can @@ -440,8 +441,12 @@ for i in {1..11}; do trace "+ exec {fd}<$l" exec {fd}<"$l" + # Note that on the locking failure we don't suggest the user to try again, + # since the client program may suggest to re-try later for all server + # errors (as bdep-publish(1) does). + # if ! run flock -w "$ref_lock_timeout" "$fd"; then - exit_with_manifest 503 "submission service temporarily unavailable" + exit_with_manifest 503 "submission service is busy" fi # Pull the reference repository. @@ -558,7 +563,7 @@ for i in {1..11}; do # prj_man="$d/project-owner.manifest" - if [ ! -f "$prj_man" ]; then + if [ ! -f "$prj_man" -a "$ref_auth" != "project" ]; then run mkdir -p "$d" # Also creates the owners directory if not exist. ctl="$(repository_base "$control")" @@ -575,7 +580,7 @@ for i in {1..11}; do # Now the package name. # d="$d/$name" - run mkdir "$d" + run mkdir -p "$d" # Also creates the project directory if not exist. pkg_man="$d/package-owner.manifest" @@ -636,34 +641,21 @@ for i in {1..11}; do exit_with_manifest 400 "unrecognized section '$section'" fi - # Strips the version revision part, if present. - # - v="$(sed -n -re 's%^(\+?[^+]+)(\+[0-9]+)?$%\1%p' <<<"$version")" - - # Make sure the section directory exists before we run find in it. - # - d="$tgt_dir/$s/$project" - run mkdir -p "$d" # Create all the parent directories as well. + run pkg_find_archives "$name" "$version*" "$tgt_dir/$s" | readarray -t arcs - # Go through the potentially matching archives (for example, for - # foo-1.2.3+2: foo-1.2.3.tar.gz, foo-1.2.3+1.tar.gz, foo-1.2.30.tar.gz, etc) - # and remove those that match exactly. - # - # Change CWD to the section directory to make sure that the found archive - # paths don't contain spaces. - # - fs=($(run cd "$tgt_dir/$s" && run find -name "$name-$v*")) - - for f in "${fs[@]}"; do - if [[ "$f" =~ ^\./[^/]+/"$name-$v"(\+[0-9]+)?\.[^/]+$ ]]; then - run git -C "$tgt_dir" rm $gqo "$s/$f" >&2 - fi + for f in "${arcs[@]}"; do + run git -C "$tgt_dir" rm $gqo "${f#$tgt_dir/}" >&2 done # Finally, add the package archive to the target repository. # - # We copy the archive rather than move it since we may need it for a re-try. + # Copy the archive rather than move it since we may need it for a re-try. + # Make sure the project directory exists before we copy the archive into it. + # Note that it was removed by git-rm if it became empty. # + d="$tgt_dir/$s/$project" + run mkdir -p "$d" # Create all the parent directories as well. + a="$d/$archive" run cp "$data_dir/$archive" "$a" diff --git a/brep/handler/submit/submit-pub.in b/brep/handler/submit/submit-pub.in new file mode 100644 index 0000000..42d478d --- /dev/null +++ b/brep/handler/submit/submit-pub.in @@ -0,0 +1,435 @@ +#!/usr/bin/env bash + +# file : brep/handler/submit/submit-pub.in +# license : MIT; see accompanying LICENSE file + +# Package submission handler with direct repository publishing. +# +# The overall idea behind this handler is to directly add the package to a +# private/trusted (unsigned) pkg repository with a simple structure (no +# sections). Upon successful execution of this handler no additional steps are +# required. +# +# Specifically, the handler performs the following steps: +# +# - Lock the repository directory for the duration of the package submission. +# +# - Check for the package duplicate. +# +# - Create the new repository as a hardlink-copy of the current one. +# +# - Remove any package revisions, if present. +# +# - Validate and add the package archive to the new repository (with project +# subdirectory). +# +# - Re-generate the new repository without signing. +# +# - Verify that the new repository is loadable into the brep package database. +# +# - Atomically switch the repository symlink to refer to the new repository. +# +# - Release the lock and remove the old repository. +# +# The repository argument (<repo>) should be an absolute path to a symbolic +# link to the pkg repository directory, with the archive and manifest files +# residing in its 1/ subdirectory. The base name of the <repo> path is used +# as a base for new repository directories. +# +# Unless the handler is called for testing, the loader program's absolute path +# and options should be specified so that the handler can verify that the +# package is loadable into the brep package database (this makes sure the +# package dependencies are resolvable, etc). +# +# Notes: +# +# - Filesystem entries that exist or are created in the data directory: +# +# <pkg>-<ver>.tar.gz saved by brep (could be other archives in the future) +# request.manifest created by brep +# package.manifest extracted by the handler +# loadtab created by the handler +# result.manifest saved by brep +# +# Options: +# +# --user <name> +# +# Re-execute itself under the specified user. +# +# Note that the repository can also be modified manually (e.g., to remove +# packages). This option is normally specified to make sure that all the +# repository filesystem entries belong to a single user, which, in +# particular, can simplify their permissions handling (avoid extra ACLs, +# etc). +# +# Note that if this option is specified, then current user (normally the +# user under which Apache2 is running) must be allowed to execute sudo +# without a password, which is only recommended in private/trusted +# environments. +# +# --result-url <url> +# +# Result URL base for the response. If specified, the handler appends the +# <package>/<version> to this value and includes the resulting URL in the +# response message. +# +usage="usage: $0 [<options>] [<loader-path> <loader-options>] <repo> <dir>" + +# Diagnostics. +# +verbose= #true + +# The repository lock timeout (seconds). +# +rep_lock_timeout=60 + +trap "{ exit 1; }" ERR +set -o errtrace # Trap in functions and subshells. +set -o pipefail # Fail if any pipeline command fails. +shopt -s lastpipe # Execute last pipeline command in the current shell. +shopt -s nullglob # Expand no-match globs to nothing rather than themselves. + +@import brep/handler/handler@ +@import brep/handler/submit/submit@ + +# Parse the command line options and, while at it, compose the arguments array +# for potential re-execution under a different user. +# +user= +result_url= + +scr_exe="$(realpath "${BASH_SOURCE[0]}")" +scr_dir="$(dirname "$scr_exe")" + +args=("$scr_exe") + +while [ "$#" -gt 0 ]; do + case $1 in + --user) + shift + user="$1" + shift + ;; + --result-url) + args+=("$1") + shift + result_url="${1%/}" + args+=("$1") + shift + ;; + *) + break; # The end of options is encountered. + ;; + esac +done + +loader_args=() # The loader path and options. + +# Assume all the remaining arguments except for the last two (repository +# symlink and data directory) as the loader program path and arguments. +# +while [ "$#" -gt 2 ]; do + loader_args+=("$1") + args+=("$1") + shift +done + +if [ "$#" -ne 2 ]; then + error "$usage" +fi + +# pkg repository symlink. +# +repo="${1%/}" +shift + +if [ -z "$repo" ]; then + error "$usage" +fi + +# Submission data directory. +# +data_dir="${1%/}" +shift + +if [ -z "$data_dir" ]; then + error "$usage" +fi + +# Re-execute itself under a different user, if requested. +# +if [ -n "$user" ]; then + args+=("$repo" "$data_dir") + + # Compose the arguments string to pass to the su program, quoting empty + # arguments as well as those that contain spaces. Note that here, for + # simplicity, we assume that the arguments may not contain '"'. + # + as= + for a in "${args[@]}"; do + if [ -z "$a" -o -z "${a##* *}" ]; then + a="\"$a\"" + fi + if [ -n "$as" ]; then + a=" $a" + fi + as="$as$a" + done + + run exec sudo --non-interactive su -l "$user" -c "$as" +fi + +# Check path presence (do it after user switch for permissions). +# +if [ ! -L "$repo" ]; then + error "'$repo' does not exist or is not a symlink" +fi + +if [ ! -d "$data_dir" ]; then + error "'$data_dir' does not exist or is not a directory" +fi + +reference="$(basename "$data_dir")" + +# Parse the submission request manifest and obtain the archive path as well as +# the simulate value. +# +manifest_parser_start "$data_dir/request.manifest" + +archive= +simulate= + +while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + case "$n" in + archive) archive="$v" ;; + simulate) simulate="$v" ;; + esac +done + +manifest_parser_finish + +if [ -z "$archive" ]; then + error "archive manifest value expected" +fi + +if [ -n "$simulate" -a "$simulate" != "success" ]; then + exit_with_manifest 400 "unrecognized simulation outcome '$simulate'" +fi + +m="$data_dir/package.manifest" +extract_package_manifest "$data_dir/$archive" "$m" + +# Parse the package manifest and obtain the package name, version, and +# project. +# +manifest_parser_start "$m" + +name= +version= +project= + +while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + case "$n" in + name) name="$v" ;; + version) version="$v" ;; + project) project="$v" ;; + esac +done + +manifest_parser_finish + +if [ -z "$name" ]; then + error "name manifest value expected" +fi + +if [ -z "$version" ]; then + error "version manifest value expected" +fi + +if [ -z "$project" ]; then + project="$name" +fi + +if [ -n "$result_url" ]; then + message_suffix=": $result_url/$name/$version" +else + message_suffix=": $name/$version" +fi + +revision="$(version_revision "$version")" + +# Open the reading file descriptor and lock the repository. Fail if unable to +# lock before timeout. +# +l="$repo.lock" +run touch "$l" +trace "+ exec {lfd}<$l" +exec {lfd}<"$l" + +# Note that on the locking failure we don't suggest the user to try again, +# since the client program may suggest to re-try later for all server errors +# (as bdep-publish(1) does). +# +if ! run flock -w "$rep_lock_timeout" "$lfd"; then + exit_with_manifest 503 "submission service is busy" +fi + +repo_old="$(realpath "$repo")" # Old repo path. +repo_name="$(basename "$repo")-$(date "+%Y%m%d-%H%M%S-%N")" # New repo name. +repo_new="$(dirname "$repo_old")/$repo_name" # New repo path. +repo_link="$repo_new.link" # New repo symlink. + +# On exit, remove the new repository symlink and directory, unless the link +# doesn't exist or the directory removal is canceled (for example, the new +# repository is made current). +# +function exit_trap () +{ + if [ -L "$repo_link" ]; then + run rm -r -f "$repo_link" + fi + + if [ -n "$repo_new" -a -d "$repo_new" ]; then + run rm -r -f "$repo_new" + fi +} + +trap exit_trap EXIT + +# Check for the package duplicate (in all projects). +# +# Use <name>-<version>.* without .tar.gz in case we want to support more +# archive types later. +# +run pkg_find_archive "$name-$version.*" "$repo_old/1" | readarray -t p + +if [ "${#p[@]}" -ne 0 ]; then + n="${p[1]}" + v="${p[2]}" + + trace "found: $n/$v in ${p[0]}" + + if [ "$n" == "$name" ]; then + exit_with_manifest 422 "duplicate submission" + else + exit_with_manifest 422 "submission conflicts with $n/$v" + fi +fi + +# Copy the current repository using hardlinks. +# +# -r (recursive) +# -t (preserve timestamps) +# -O (omit dir timestamps) +# --link-dest (hardlink files instead of copying) +# +# We also exclude the packages.manifest file that will be re-generated anyway. +# +run rsync -rtO --exclude 'packages.manifest' --link-dest="$repo_old" \ + "$repo_old/" "$repo_new" + +# Remove the package version revision archives that may exist in the +# repository. +# +# But first check if the repository contains newer revision of this package +# version. Respond with the 'newer revision is present' result manifest and +# exit if that's the case. +# +run pkg_find_archives "$name" "$version*" "$repo_new/1" | readarray -t arcs + +for f in "${arcs[@]}"; do + pkg_verify_archive "$f" | readarray -t p + + v="${p[1]}" + rv="$(version_revision "$v")" + + if [ "$rv" -gt "$revision" ]; then + exit_with_manifest 422 "newer revision $name/$v is present" + fi +done + +for f in "${arcs[@]}"; do + run rm "$f" +done + +# Copy the archive rather than moving it since we may need it for +# troubleshooting. Note: the data and repository directories can be on +# different filesystems and so hardlinking could fail. +# +run mkdir -p "$repo_new/1/$project" +run cp "$data_dir/$archive" "$repo_new/1/$project" + +# Create the new repository. +# +# Note that if bpkg-rep-create fails, we can't reliably distinguish if this is +# a user or internal error (broken package archive vs broken repository). +# Thus, we always treat is as a user error, providing the full error +# description in the response and assuming that the submitter can either fix +# the issue or report it to the repository maintainers. This again assumes +# private/trusted environment. +# +trace "+ bpkg rep-create '$repo_new/1' 2>&1" + +if ! e="$(bpkg rep-create "$repo_new/1" 2>&1)"; then + exit_with_manifest 400 "submitted archive is not a valid package +$e" +fi + +# If requested, verify that the new repository is loadable into the package +# database and, as in the above case, treat the potential error as a user +# error. +# +if [ "${#loader_args[@]}" -ne 0 ]; then + f="$data_dir/loadtab" + echo "http://testrepo/1 private cache:$repo_new/1" >"$f" + + trace "+ ${loader_args[@]} '$f' 2>&1" + + if ! e="$("${loader_args[@]}" "$f" 2>&1)"; then + + # Sanitize the error message, removing the confusing lines. + # + e="$(run sed -re '/testrepo/d' <<<"$e")" + exit_with_manifest 400 "unable to add package to repository +$e" + fi +fi + +# Finally, create the new repository symlink and replace the current symlink +# with it, unless we are simulating. +# +run ln -sf "$repo_name" "$repo_link" + +if [ -z "$simulate" ]; then + run mv -T "$repo_link" "$repo" # Switch the repository symlink atomically. + + # Now, when the repository link is switched, disable the new repository + # removal. + # + # Note that we still can respond with an error status. However, the + # remaining operations are all cleanups and thus unlikely to fail. + # + repo_new= +fi + +trace "+ exec {lfd}<&-" +exec {lfd}<&- # Close the file descriptor and unlock the repository. + +# Remove the old repository, unless we are simulating. +# +# Note that if simulating, we leave the new repository directory/symlink +# removal to the exit trap (see above). +# +if [ -z "$simulate" ]; then + run rm -r "$repo_old" + + what="published" +else + what="simulated" +fi + +run rm -r "$data_dir" + +trace "package is $what$message_suffix" +exit_with_manifest 200 "package is published$message_suffix" diff --git a/brep/handler/submit/submit.bash.in b/brep/handler/submit/submit.bash.in index d1d0634..7826809 100644 --- a/brep/handler/submit/submit.bash.in +++ b/brep/handler/submit/submit.bash.in @@ -1,5 +1,4 @@ # file : brep/handler/submit/submit.bash.in -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file # Utility functions useful for implementing package submission handlers. @@ -48,12 +47,29 @@ function extract_package_manifest () # <archive> <manifest> local arc="$1" local man="$2" - # Pass the --deep option to make sure that the *-file manifest values are - # resolvable, so rep-create will not fail due to this package down the road. - # Note that we also make sure that all the manifest values are known (see - # bpkg-pkg-verify for details). + # Pass the --deep option to make sure that the bootstrap buildfile is + # present and the *-file manifest values are resolvable, so rep-create will + # not fail due to this package down the road. Note that we also make sure + # that all the manifest values are known (see bpkg-pkg-verify for details). # - if ! run_silent bpkg pkg-verify --deep --manifest "$arc" >"$man"; then + local cmd=(bpkg pkg-verify --deep --manifest "$arc") + trace_cmd "${cmd[@]}" + + # Note that we used to just advise the user to run bpkg-pkg-verify locally + # for the details on the potential failure. That, however, may not always be + # helpful since the user can use a different version of the toolchain and so + # may observe a different behavior. Thus, we add the bpkg-pkg-verify error + # message to the response, turning it into an info. This way the user may + # potentially see the following bdep-publish diagnostics: + # + # error: package archive is not valid + # info: unable to satisfy constraint (build2 >= 0.17.0-) for package libhello-1.0.0.tar.gz + # info: available build2 version is 0.16.0 + # info: run bpkg pkg-verify for details + # info: reference: 308e155764c8 + # + local e + if ! e="$("${cmd[@]}" 2>&1 >"$man")"; then # Perform the sanity check to make sure that bpkg is runnable. # @@ -61,6 +77,33 @@ function extract_package_manifest () # <archive> <manifest> error "unable to run bpkg" fi - exit_with_manifest 400 "archive is not a valid package (run bpkg pkg-verify for details)" + # Note that bpkg-pkg-verify diagnostics may potentially contain the + # archive absolute path. Let's sanitize this diagnostics by stripping the + # archive directory path, if present. Also note that to use sed for that + # we first need to escape the special regex characters and slashes in the + # archive directory path (see sed's basic regular expressions for + # details). + # + local d="$(sed 's/[[\.*^$/]/\\&/g' <<<"$(dirname "$arc")/")" + + e="$(sed -e "s/$d//g" -e 's/^error:/ info:/' <<<"$e")" + e=$'package archive is not valid\n'"$e"$'\n info: run bpkg pkg-verify for details' + + exit_with_manifest 400 "$e" fi } + +# Extract the revision part from the package version. Return 0 if the version +# doesn't contain revision. +# +function version_revision () # version +{ + local r + r="$(sed -n -re 's%^(\+?[^+]+)(\+([0-9]+))?$%\3%p' <<<"$1")" + + if [ -z "$r" ]; then + r="0" + fi + + echo "$r" +} diff --git a/brep/handler/upload/.gitignore b/brep/handler/upload/.gitignore new file mode 100644 index 0000000..da4dc5a --- /dev/null +++ b/brep/handler/upload/.gitignore @@ -0,0 +1,2 @@ +brep-upload-bindist +brep-upload-bindist-clean diff --git a/brep/handler/upload/buildfile b/brep/handler/upload/buildfile new file mode 100644 index 0000000..ca52ddd --- /dev/null +++ b/brep/handler/upload/buildfile @@ -0,0 +1,13 @@ +# file : brep/handler/upload/buildfile +# license : MIT; see accompanying LICENSE file + +./: exe{brep-upload-bindist} exe{brep-upload-bindist-clean} + +include ../ + +exe{brep-upload-bindist}: in{upload-bindist} bash{upload} ../bash{handler} + +[rule_hint=bash] \ +exe{brep-upload-bindist-clean}: in{upload-bindist-clean} + +bash{upload}: in{upload} ../bash{handler} diff --git a/brep/handler/upload/upload-bindist-clean.in b/brep/handler/upload/upload-bindist-clean.in new file mode 100644 index 0000000..99914a7 --- /dev/null +++ b/brep/handler/upload/upload-bindist-clean.in @@ -0,0 +1,224 @@ +#!/usr/bin/env bash + +# file : brep/handler/upload/upload-bindist-clean.in +# license : MIT; see accompanying LICENSE file + +# Remove expired package configuration directories created by the +# upload-bindist handler. +# +# Specifically, perform the following steps: +# +# - Recursively scan the specified root directory and collect the package +# configuration directories with age older than the specified timeout (in +# minutes). Recognize the package configuration directories by matching the +# *-????-??-??T??:??:??Z* pattern and calculate their age based on the +# modification time of the packages.sha256 file they may contain. If +# packages.sha256 doesn't exist in the configuration directory, then +# consider it as still being prepared and skip. +# +# - Iterate over the expired package configuration directories and for each of +# them: +# +# - Lock the root directory. +# +# - Re-check the expiration criteria. +# +# - Remove the package configuration symlink if it refers to this directory. +# +# - Remove this directory. +# +# - Remove all the the parent directories of this directory which become +# empty, up to (but excluding) the root directory. +# +# - Unlock the root directory. +# +usage="usage: $0 <root> <timeout>" + +# Diagnostics. +# +verbose= #true + +# The root directory lock timeout (in seconds). +# +lock_timeout=60 + +trap "{ exit 1; }" ERR +set -o errtrace # Trap in functions and subshells. +set -o pipefail # Fail if any pipeline command fails. +shopt -s lastpipe # Execute last pipeline command in the current shell. +shopt -s nullglob # Expand no-match globs to nothing rather than themselves. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } +function trace () { if [ "$verbose" ]; then info "$*"; fi } + +# Trace a command line, quoting empty arguments as well as those that contain +# spaces. +# +function trace_cmd () # <cmd> <arg>... +{ + if [[ "$verbose" ]]; then + local s="+" + while [ $# -gt 0 ]; do + if [ -z "$1" -o -z "${1##* *}" ]; then + s="$s '$1'" + else + s="$s $1" + fi + + shift + done + + info "$s" + fi +} + +# Trace and run a command. +# +function run () # <cmd> <arg>... +{ + trace_cmd "$@" + "$@" +} + +if [[ "$#" -ne 2 ]]; then + error "$usage" +fi + +# Package configurations root directory. +# +root_dir="${1%/}" +shift + +if [[ -z "$root_dir" ]]; then + error "$usage" +fi + +if [[ ! -d "$root_dir" ]]; then + error "'$root_dir' does not exist or is not a directory" +fi + +# Package configuration directories timeout. +# +timeout="$1" +shift + +if [[ ! "$timeout" =~ ^[0-9]+$ ]]; then + error "$usage" +fi + +# Note that while the '%s' date format is not POSIX, it is supported on both +# Linux and FreeBSD. +# +expiration=$(($(date -u +"%s") - $timeout * 60)) + +# Collect the list of expired package configuration directories. +# +expired_dirs=() + +run find "$root_dir" -type d -name "*-????-??-??T??:??:??Z*" | while read d; do + f="$d/packages.sha256" + + # Note that while the -r date option is not POSIX, it is supported on both + # Linux and FreeBSD. + # + trace_cmd date -u -r "$f" +"%s" + if t="$(date -u -r "$f" +"%s" 2>/dev/null)" && (($t <= $expiration)); then + expired_dirs+=("$d") + fi +done + +if [[ "${#expired_dirs[@]}" -eq 0 ]]; then + exit 0 # Nothing to do. +fi + +# Make sure the root directory lock file exists. +# +lock="$root_dir/upload.lock" +run touch "$lock" + +# Remove the expired package configuration directories, symlinks which refer +# to them, and the parent directories which become empty. +# +for d in "${expired_dirs[@]}"; do + # Deduce the path of the potential package configuration symlink that may + # refer to this package configuration directory by stripping the + # -<timestamp>[-<number>] suffix. + # + l="$(sed -n -re 's/^(.+)-[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z(-[0-9]+)?$/\1/p' <<<"$d")" + if [[ -z "$l" ]]; then + error "invalid name '$d' for package configuration directory" + fi + + f="$d/packages.sha256" + + # Open the reading file descriptor and lock the root directory. Fail if + # unable to lock before timeout. + # + trace "+ exec {lfd}<$lock" + exec {lfd}<"$lock" + + if ! run flock -w "$lock_timeout" "$lfd"; then + error "unable to lock root directory" + fi + + # Now, as the lock is acquired, recheck the package configuration directory + # expiration criteria (see above) and, if it still holds, remove this + # directory, the package configuration symlink if it refers to it, and all + # the parent directories which become empty up to (but excluding) the root + # directory. + # + trace_cmd date -u -r "$f" +"%s" + if t="$(date -u -r "$f" +"%s" 2>/dev/null)" && (($t <= $expiration)); then + # Remove the package configuration symlink. + # + # Do this first to avoid dangling symlinks which may potentially be + # exposed by brep. + # + # Note that while the realpath utility is not POSIX, it is present on + # both Linux and FreeBSD. + # + if [[ -L "$l" ]]; then + p="$(realpath "$l")" + if [[ "$p" == "$d" ]]; then + run rm "$l" + fi + fi + + # Remove the package configuration directory. + # + # Note that this directory contains files copied from a subdirectory of + # upload-data. These files are normally owned by the Apache2 user/group + # and have rw-r--r-- permissions. This script is normally executed as the + # brep user/group and thus the uploads root directory and all its + # subdirectories must have read, write, and execute permissions granted to + # the brep user, for example, by using ACL (see INSTALL file for + # details). Since cp preserves the file permissions by default, these + # files effective permissions will normally be r-- (read-only) for this + # script. In this case rm pops up the 'remove write-protected regular + # file' prompt by default prior to removing these files. To suppress the + # prompt we will pass the -f option to rm. + # + run rm -rf "$d" + + # Remove the empty parent directories. + # + # Note that we iterate until the rmdir command fails, presumably because a + # directory is not empty. + # + d="$(dirname "$d")" + while [[ "$d" != "$root_dir" ]]; do + trace_cmd rmdir "$d" + if rmdir "$d" 2>/dev/null; then + d="$(dirname "$d")" + else + break + fi + done + fi + + # Close the file descriptor and unlock the root directory. + # + trace "+ exec {lfd}<&-" + exec {lfd}<&- +done diff --git a/brep/handler/upload/upload-bindist.in b/brep/handler/upload/upload-bindist.in new file mode 100644 index 0000000..05d0bcf --- /dev/null +++ b/brep/handler/upload/upload-bindist.in @@ -0,0 +1,595 @@ +#!/usr/bin/env bash + +# file : brep/handler/upload/upload-bindist.in +# license : MIT; see accompanying LICENSE file + +# Binary distribution packages upload handler which places the uploaded +# packages under the following filesystem hierarchy: +# +# <root>/[<tenant>/]<instance>/<os-release-name-id><os-release-version-id>/<project>/<package>/<version>/<package-config> +# +# The overall idea behind this handler is to create a uniquely named package +# configuration directory for each upload and maintain the package +# configuration symlink at the above path to refer to the directory of the +# latest upload. +# +# The root directory is passed as an argument (via upload-handler-argument). +# All the remaining directory components are retrieved from the respective +# manifest values of request.manifest created by brep and +# bindist-result.manifest contained in the uploaded archive. +# +# Note that the leaf component of the package configuration symlink path is +# sanitized, having the "bindist", <instance>, <os-release-name-id>, and +# <os-release-name-id><os-release-version-id> dash-separated sub-components +# removed. If the component becomes empty as a result of the sanitization, +# then the target CPU is assumed, if the package is not architecture- +# independent, and "noarch" otherwise. If the sanitized component is not +# empty, the package is not architecture-independent, and the resulting +# component doesn't containt the target CPU, then prepend it with the <cpu>- +# prefix. For example, the following symlink paths: +# +# .../archive/windows10/foo/libfoo/1.0.0/bindist-archive-windows10-release +# .../archive/windows10/foo/libfoo/1.0.0/bindist-archive-windows10 +# +# are reduced to: +# +# .../archive/windows10/foo/libfoo/1.0.0/x86_64-release +# .../archive/windows10/foo/libfoo/1.0.0/x86_64 +# +# To achieve this the handler performs the following steps (<dir> is passed as +# last argument by brep and is a subdirectory of upload-data): +# +# - Parse <dir>/request.manifest to retrieve the upload archive path, +# timestamp, and the values which are required to compose the package +# configuration symlink path. +# +# - Extract files from the upload archive. +# +# - Parse <dir>/<instance>/bindist-result.manifest to retrieve the values +# required to compose the package configuration symlink path and the package +# file paths. +# +# - Compose the package configuration symlink path. +# +# - Compose the package configuration directory path by appending the +# -<timestamp>[-<number>] suffix to the package configuration symlink path. +# +# - Create the package configuration directory. +# +# - Copy the uploaded package files into the package configuration directory. +# +# - Generate the packages.sha256 file in the package configuration directory, +# which lists the SHA256 checksums of the files contained in this directory. +# +# - Switch the package configuration symlink to refer to the newly created +# package configuration directory. +# +# - If the --keep-previous option is not specified, then remove the previous +# target of the package configuration symlink, if exists. +# +# Notes: +# +# - There could be a race both with upload-bindist-clean and other +# upload-bindist instances while creating the package version/configuration +# directories, querying the package configuration symlink target, switching +# the symlink, and removing the symlink's previous target. To avoid it, the +# root directory needs to be locked for the duration of these operations. +# This, however, needs to be done granularly to perform the time consuming +# operations (files copying, etc) while not holding the lock. +# +# - The brep module doesn't acquire the root directory lock. Thus, the package +# configuration symlink during its lifetime should always refer to a +# valid/complete package configuration directory. +# +# - Filesystem entries that exist or are created in the data directory: +# +# <archive> saved by brep +# request.manifest created by brep +# <instance>/* extracted by the handler (bindist-result.manifest, etc) +# result.manifest saved by brep +# +# Options: +# +# --keep-previous +# +# Don't remove the previous target of the package configuration symlink. +# +usage="usage: $0 [<options>] <root> <dir>" + +# Diagnostics. +# +verbose= #true + +# The root directory lock timeout (in seconds). +# +lock_timeout=60 + +# If the package configuration directory already exists (may happen due to the +# low timestamp resolution), then re-try creating the configuration directory +# by adding the -<number> suffix and incrementing it until the creation +# succeeds or the retries limit is reached. +# +create_dir_retries=99 + +trap "{ exit 1; }" ERR +set -o errtrace # Trap in functions and subshells. +set -o pipefail # Fail if any pipeline command fails. +shopt -s lastpipe # Execute last pipeline command in the current shell. +shopt -s nullglob # Expand no-match globs to nothing rather than themselves. + +@import brep/handler/handler@ +@import brep/handler/upload/upload@ + +# Parse the command line options. +# +keep_previous= + +while [[ "$#" -gt 0 ]]; do + case $1 in + --keep-previous) + shift + keep_previous=true + ;; + *) + break + ;; + esac +done + +if [[ "$#" -ne 2 ]]; then + error "$usage" +fi + +# Destination root directory. +# +root_dir="${1%/}" +shift + +if [[ -z "$root_dir" ]]; then + error "$usage" +fi + +if [[ ! -d "$root_dir" ]]; then + error "'$root_dir' does not exist or is not a directory" +fi + +# Upload data directory. +# +data_dir="${1%/}" +shift + +if [[ -z "$data_dir" ]]; then + error "$usage" +fi + +if [[ ! -d "$data_dir" ]]; then + error "'$data_dir' does not exist or is not a directory" +fi + +reference="$(basename "$data_dir")" # Upload request reference. + +# Parse the upload request manifest. +# +manifest_parser_start "$data_dir/request.manifest" + +archive= +instance= +timestamp= +name= +version= +project= +package_config= +target= +tenant= + +while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + case "$n" in + archive) archive="$v" ;; + instance) instance="$v" ;; + timestamp) timestamp="$v" ;; + name) name="$v" ;; + version) version="$v" ;; + project) project="$v" ;; + package-config) package_config="$v" ;; + target) target="$v" ;; + tenant) tenant="$v" ;; + esac +done + +manifest_parser_finish + +if [[ -z "$archive" ]]; then + error "archive manifest value expected" +fi + +if [[ -z "$instance" ]]; then + error "instance manifest value expected" +fi + +if [[ -z "$timestamp" ]]; then + error "timestamp manifest value expected" +fi + +if [[ -z "$name" ]]; then + error "name manifest value expected" +fi + +if [[ -z "$version" ]]; then + error "version manifest value expected" +fi + +if [[ -z "$project" ]]; then + error "project manifest value expected" +fi + +if [[ -z "$package_config" ]]; then + error "package-config manifest value expected" +fi + +if [[ -z "$target" ]]; then + error "target manifest value expected" +fi + +# Let's disallow the leading dot in the package-config manifest value since +# the latter serves as the package configuration symlink name and brep skips +# symlinks with the leading dots assuming them as hidden (see +# mod/mod-package-version-details.cxx for details). +# +if [[ "$package_config" == "."* ]]; then + exit_with_manifest 400 "package-config manifest value may not start with dot" +fi + +# Extract the CPU component from the target triplet and deduce the binary +# distribution-specific CPU representation which is normally used in the +# package file names. +# +cpu="$(sed -n -re 's/^([^-]+)-.+/\1/p' <<<"$target")" + +if [[ -z "$cpu" ]]; then + error "CPU expected in target triplet '$target'" +fi + +# Use CPU extracted from the target triplet as a distribution-specific +# representation, unless this is Debian or Fedora (see bpkg's +# system-package-manager-{fedora,debian}.cxx for details). +# +cpu_dist="$cpu" + +case $instance in + debian) + case $cpu in + x86_64) cpu_dist="amd64" ;; + aarch64) cpu_dist="arm64" ;; + i386 | i486 | i586 | i686) cpu_dist="i386" ;; + esac + ;; + fedora) + case $cpu in + i386 | i486 | i586 | i686) cpu_dist="i686" ;; + esac + ;; +esac + +# Unpack the archive. +# +run tar -xf "$data_dir/$archive" -C "$data_dir" + +# Parse the bindist result manifest list. +# +f="$data_dir/$instance/bindist-result.manifest" + +if [[ ! -f "$f" ]]; then + exit_with_manifest 400 "$instance/bindist-result.manifest not found" +fi + +manifest_parser_start "$f" + +# Parse the distribution manifest. +# +# Note that we need to skip the first manifest version value and parse until +# the next one is encountered, which introduces the first package file +# manifest. +# +os_release_name_id= +os_release_version_id= + +first=true +more= +while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + case "$n" in + "") if [[ "$first" ]]; then # Start of the first (distribution) manifest? + first= + else # Start of the second (package file) manifest. + more=true + break + fi + ;; + + os-release-name-id) os_release_name_id="$v" ;; + os-release-version-id) os_release_version_id="$v" ;; + esac +done + +if [[ -z "$os_release_name_id" ]]; then + exit_with_manifest 400 "os-release-name-id bindist result manifest value expected" +fi + +if [[ -z "$os_release_version_id" ]]; then + exit_with_manifest 400 "os-release-version-id bindist result manifest value expected" +fi + +if [[ ! "$more" ]]; then + exit_with_manifest 400 "no package file manifests in bindist result manifest list" +fi + +# Parse the package file manifest list and cache the file paths. +# +# While at it, detect if the package is architecture-specific or not by +# checking if any package file names contain the distribution-specific CPU +# representation (as a sub-string). +# +# Note that while we currently only need the package file paths, we can make +# use of their types and system names in the future. Thus, let's verify that +# all the required package file values are present and, while at it, cache +# them all in the parallel arrays. +# +package_file_paths=() +package_file_types=() +package_file_system_names=() + +arch_specific= + +# The outer loop iterates over package file manifests while the inner loop +# iterates over manifest values in each such manifest. +# +while [[ "$more" ]]; do + more= + type= + path= + system_name= + + while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + case "$n" in + "") # Start of the next package file manifest. + more=true + break + ;; + + package-file-path) path="$v" ;; + package-file-type) type="$v" ;; + package-file-system-name) system_name="$v" ;; + esac + done + + if [[ -z "$path" ]]; then + exit_with_manifest 400 "package-file-path bindist result manifest value expected" + fi + + if [[ -z "$type" ]]; then + exit_with_manifest 400 "package-file-type bindist result manifest value expected" + fi + + package_file_paths+=("$path") + package_file_types+=("$type") + package_file_system_names+=("$system_name") # Note: system name can be empty. + + if [[ "$path" == *"$cpu_dist"* ]]; then + arch_specific=true + fi +done + +manifest_parser_finish + +# Sanitize the package configuration name. +# +config= +for c in $(sed 's/-/ /g' <<<"$package_config"); do + if [[ "$c" != "bindist" && + "$c" != "$instance" && + "$c" != "$os_release_name_id" && + "$c" != "$os_release_name_id$os_release_version_id" ]]; then + if [[ -z "$config" ]]; then + config="$c" + else + config="$config-$c" + fi + fi +done + +# Reflect the architecture in the sanitized configuration name. +# +if [[ -z "$config" ]]; then + if [[ "$arch_specific" ]]; then + config="$cpu" + else + config="noarch" + fi +else + if [[ "$arch_specific" && ("$config" != *"$cpu"*) ]]; then + config="$cpu-$config" + fi +fi + +# Compose the package configuration symlink path. +# +config_link="$root_dir" + +if [[ -n "$tenant" ]]; then + config_link="$config_link/$tenant" +fi + +config_link="$config_link/$instance/$os_release_name_id$os_release_version_id" +config_link="$config_link/$project/$name/$version/$config" + +# Compose the package configuration directory path. +# +config_dir="$config_link-$timestamp" + +# Create the package configuration directory. +# +# Note that it is highly unlikely that multiple uploads for the same package +# configuration/distribution occur at the same time (with the seconds +# resolution) making the directory name not unique. If that still happens, +# lets retry for some reasonable number of times to create the directory, +# while adding the -<number> suffix to its path on each iteration. If +# that also fails, then we assume that there is some issue with the handler +# setup and fail, printing the cached mkdir diagnostics to stderr. +# +# Note that we need to prevent removing of the potentially empty package +# version directory by the upload-bindist-clean script before we create +# configuration directory. To achieve that, we lock the root directory for the +# duration of the package version/configuration directories creation. +# +# Open the reading file descriptor and lock the root directory. Fail if +# unable to lock before timeout. +# +lock="$root_dir/upload.lock" +run touch "$lock" +trace "+ exec {lfd}<$lock" +exec {lfd}<"$lock" + +if ! run flock -w "$lock_timeout" "$lfd"; then + exit_with_manifest 503 "upload service is busy" +fi + +# Create parent (doesn't fail if directory exists). +# +config_parent_dir="$(dirname "$config_dir")" +run mkdir -p "$config_parent_dir" + +created= + +trace_cmd mkdir "$config_dir" +if ! e="$(mkdir "$config_dir" 2>&1)"; then # Note: fails if directory exists. + for ((i=0; i != $create_dir_retries; ++i)); do + d="$config_dir-$i" + trace_cmd mkdir "$d" + if e="$(mkdir "$d" 2>&1)"; then + config_dir="$d" + created=true + break + fi + done +else + created=true +fi + +# Close the file descriptor and unlock the root directory. +# +trace "+ exec {lfd}<&-" +exec {lfd}<&- + +if [[ ! "$created" ]]; then + echo "$e" 1>&2 + error "unable to create package configuration directory" +fi + +# On exit, remove the newly created package configuration directory, unless +# its removal is canceled (for example, the symlink is switched to refer to +# it). Also remove the new symlink, if already created. +# +# Make sure we don't fail if the entries are already removed, for example, by +# the upload-bindist-clean script. +# +config_link_new= +function exit_trap () +{ + if [[ -n "$config_dir" && -d "$config_dir" ]]; then + if [[ -n "$config_link_new" && -L "$config_link_new" ]]; then + run rm -f "$config_link_new" + fi + run rm -rf "$config_dir" + fi +} + +trap exit_trap EXIT + +# Copy all the extracted package files to the package configuration directory. +# +for ((i=0; i != "${#package_file_paths[@]}"; ++i)); do + run cp "$data_dir/$instance/${package_file_paths[$i]}" "$config_dir" +done + +# Generate the packages.sha256 file. +# +# Note that since we don't hold the root directory lock at this time, we +# temporary "hide" the resulting file from the upload-bindist-clean script +# (which uses it for the upload age calculation) by adding the leading dot to +# its name. Not doing so we may potentially end up with upload-bindist-clean +# removing the half-cooked directory and so breaking the upload handling. +# +trace "+ (cd $config_dir && exec sha256sum -b ${package_file_paths[@]} >.packages.sha256)" +(cd "$config_dir" && exec sha256sum -b "${package_file_paths[@]}" >".packages.sha256") + +# Create the new package configuration "hidden" symlink. Construct its name by +# prepending the configuration directory name with a dot. +# +config_dir_name="$(basename "$config_dir")" +config_link_new="$config_parent_dir/.$config_dir_name" +run ln -s "$config_dir_name" "$config_link_new" + +# Switch the package configuration symlink atomically. But first, cache the +# previous package configuration symlink target if the --keep-previous option +# is not specified and "unhide" the packages.sha256 file. +# +# Note that to avoid a race with upload-bindist-clean and other upload-bindist +# instances, we need to perform all the mentioned operations as well as +# removing the previous package configuration directory while holding the root +# directory lock. + +# Lock the root directory. +# +trace "+ exec {lfd}<$lock" +exec {lfd}<"$lock" + +if ! run flock -w "$lock_timeout" "$lfd"; then + exit_with_manifest 503 "upload service is busy" +fi + +# Note that while the realpath utility is not POSIX, it is present on both +# Linux and FreeBSD. +# +config_dir_prev= +if [[ ! "$keep_previous" && -L "$config_link" ]]; then + config_dir_prev="$(realpath "$config_link")" +fi + +# "Unhide" the packages.sha256 file. +# +run mv "$config_dir/.packages.sha256" "$config_dir/packages.sha256" + +# Note that since brep doesn't acquire the root directory lock, we need to +# switch the symlink as the final step, when the package directory is fully +# prepared and can be exposed. +# +# @@ Also note that the -T option is Linux-specific. To add support for +# FreeBSD we need to use -h option there (but maybe -T also works, +# who knows). +# +run mv -T "$config_link_new" "$config_link" + +# Now, when the package configuration symlink is switched, disable removal of +# the newly created package configuration directory. +# +# Note that we still can respond with an error status. However, the remaining +# operations are all cleanups and thus unlikely to fail. +# +config_dir= + +# Remove the previous package configuration directory, if requested. +# +if [[ -n "$config_dir_prev" ]]; then + run rm -r "$config_dir_prev" +fi + +# Unlock the root directory. +# +trace "+ exec {lfd}<&-" +exec {lfd}<&- + +# Remove the no longer needed upload data directory. +# +run rm -r "$data_dir" + +trace "binary distribution packages are published" +exit_with_manifest 200 "binary distribution packages are published" diff --git a/brep/handler/upload/upload.bash.in b/brep/handler/upload/upload.bash.in new file mode 100644 index 0000000..9acead9 --- /dev/null +++ b/brep/handler/upload/upload.bash.in @@ -0,0 +1,40 @@ +# file : brep/handler/upload/upload.bash.in +# license : MIT; see accompanying LICENSE file + +# Utility functions useful for implementing upload handlers. + +if [ "$brep_handler_upload" ]; then + return 0 +else + brep_handler_upload=true +fi + +@import brep/handler/handler@ + +# Serialize the upload result manifest to stdout and exit the (sub-)shell with +# the zero status. +# +reference= # Should be assigned later by the handler, when becomes available. + +function exit_with_manifest () # <status> <message> +{ + trace_func "$@" + + local sts="$1" + local msg="$2" + + manifest_serializer_start + + manifest_serialize "" "1" # Start of manifest. + manifest_serialize "status" "$sts" + manifest_serialize "message" "$msg" + + if [ -n "$reference" ]; then + manifest_serialize "reference" "$reference" + elif [ "$sts" == "200" ]; then + error "no reference for code $sts" + fi + + manifest_serializer_finish + run exit 0 +} |