From b1888e516c9c9d750726318227bf69856ec91b8b Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Fri, 31 Aug 2018 21:41:53 +0300 Subject: Add ci-load --- brep/handler/ci/.gitignore | 1 + brep/handler/ci/buildfile | 5 +- brep/handler/ci/ci-dir.in | 21 +++- brep/handler/ci/ci-load.in | 186 +++++++++++++++++++++++++++++++++ brep/handler/ci/ci.bash.in | 24 +++++ brep/handler/submit/submit-dir.in | 2 +- brep/handler/submit/submit-git.bash.in | 11 +- brep/handler/submit/submit-git.in | 2 +- brep/handler/submit/submit.bash.in | 1 + 9 files changed, 237 insertions(+), 16 deletions(-) create mode 100644 brep/handler/ci/ci-load.in (limited to 'brep') diff --git a/brep/handler/ci/.gitignore b/brep/handler/ci/.gitignore index f31b542..8a7b2cf 100644 --- a/brep/handler/ci/.gitignore +++ b/brep/handler/ci/.gitignore @@ -1 +1,2 @@ brep-ci-dir +brep-ci-load diff --git a/brep/handler/ci/buildfile b/brep/handler/ci/buildfile index c45a79b..ad57e33 100644 --- a/brep/handler/ci/buildfile +++ b/brep/handler/ci/buildfile @@ -2,10 +2,11 @@ # copyright : Copyright (c) 2014-2018 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -./: exe{brep-ci-dir} +./: exe{brep-ci-dir} exe{brep-ci-load} include ../ -exe{brep-ci-dir}: in{ci-dir} bash{ci} ../bash{handler} +exe{brep-ci-dir}: in{ci-dir} bash{ci} ../bash{handler} +exe{brep-ci-load}: in{ci-load} bash{ci} ../bash{handler} bash{ci}: in{ci} ../bash{handler} diff --git a/brep/handler/ci/ci-dir.in b/brep/handler/ci/ci-dir.in index 6a4f0af..2915b25 100644 --- a/brep/handler/ci/ci-dir.in +++ b/brep/handler/ci/ci-dir.in @@ -6,13 +6,18 @@ # Simple package CI request handler with directory storage. # -# Keep the CI request directory unless simulating. Write the CI result -# manifest to stdout. +# Dump the repositories.manifest and packages.manifest files into the CI +# request data directory as a sanity check. Keep the directory unless +# simulating. Write the CI result manifest to stdout. # usage="usage: $0 " verbose= #true +# Repository information fetch timeout (seconds). +# +fetch_timeout=60 + trap "{ exit 1; }" ERR set -o errtrace # Trap ERR in functions. @@ -80,14 +85,20 @@ fi spec="$spec$repository" +# Exit with the 'CI request is queued' response if simulating. +# +# Note that we can't assume a real repository URL is specified if simulating +# so trying to query the repository info is not a good idea. +# if [ -n "$simulate" ]; then - rm -r "$data_dir" + run rm -r "$data_dir" trace "CI request for '$spec' is simulated" else + dump_repository_manifests "$repository" "$data_dir" "$fetch_timeout" trace "CI request for '$spec' is queued" fi -# The spec normally contains the full commit id and so feels too hairy for -# including in the message. +# The spec normally contains the full commit id and so feels too hairy to +# include in the result manifest message. # exit_with_manifest 200 "CI request is queued" diff --git a/brep/handler/ci/ci-load.in b/brep/handler/ci/ci-load.in new file mode 100644 index 0000000..98078b3 --- /dev/null +++ b/brep/handler/ci/ci-load.in @@ -0,0 +1,186 @@ +#!/usr/bin/env bash + +# file : brep/handler/ci/ci-load.in +# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd +# license : MIT; see accompanying LICENSE file + +# Package CI request handler that loads the packages into the brep database. +# +# --result-url +# Result URL base for the response. If specified, the handler will append the +# brep tenant id to this value and include the resulting URL in the response +# message. +# +# +# Loader program (normally brep-load(1)). +# +# +# Loader options (normally --db-*). +# +usage="usage: $0 [--result-url ] [] " + +verbose= #true + +# Repository information fetch timeout (seconds). +# +fetch_timeout=60 + +trap "{ exit 1; }" ERR +set -o errtrace # Trap ERR in functions. + +@import brep/handler/handler@ +@import brep/handler/ci/ci@ + +# The handler's own options. +# +result_url= +while [ $# -gt 0 ]; do + case $1 in + --result-url) + shift + result_url="${1%/}" + shift + ;; + *) + break + ;; + esac +done + +# The loader path. +# +loader="$1" + +if [ -z "$loader" ]; then + error "$usage" +fi + +shift + +# Assume that the remaining arguments except the last one are the loader +# options. +# +loader_options=() +while [ $# -gt 1 ]; do + loader_options+=("$1") + shift +done + +# CI request data directory (last argument). +# +data_dir="$1" + +if [ -z "$data_dir" ]; then + error "$usage" +fi + +if [ ! -d "$data_dir" ]; then + error "'$data_dir' does not exist or is not a directory" +fi + +reference="$(basename "$data_dir")" + +# Parse the CI request manifest and obtain the repository URL, package names +# with optional versions, as well as the simulate value. +# +manifest_parser_start "$data_dir/request.manifest" + +repository= +packages=() +simulate= + +while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + case "$n" in + repository) repository="$v" ;; + package) packages+=("$v") ;; + simulate) simulate="$v" ;; + esac +done + +manifest_parser_finish + +if [ -z "$repository" ]; then + error "repository manifest value expected" +fi + +if [ -n "$simulate" -a "$simulate" != "success" ]; then + exit_with_manifest 400 "unrecognized simulation outcome '$simulate'" +fi + +# Produce the bpkg-build(1)-like package spec for tracing. +# +# The spec normally contains the full commit id and so feels too hairy to +# include in the result manifest message. +# +spec= +for p in "${packages[@]}"; do + if [ -n "$spec" ]; then + spec="$spec," + fi + spec="$spec$p" +done + +if [ -n "$spec" ]; then + spec="$spec@" +fi + +spec="$spec$repository" + +message_suffix= +if [ -n "$result_url" ]; then + message_suffix=": $result_url" +fi + +# Exit with the 'CI request is queued' response if simulating. +# +# Note that we can't assume a real repository URL is specified if simulating +# so trying to query the repository info is not a good idea. +# +if [ -n "$simulate" ]; then + run rm -r "$data_dir" + + trace "CI request for '$spec' is simulated$message_suffix" + exit_with_manifest 200 "CI request is queued$message_suffix" +fi + +# Dump the repositories.manifest and packages.manifest files. +# +run mkdir "$data_dir/cache" +dump_repository_manifests "$repository" "$data_dir/cache" "$fetch_timeout" + +# In most cases all the requested for CI packages belong to the same project, +# which would be nice to use as the repository display name. However, that +# would require parsing packages.manifest just to get this information. Which +# feels like a bit of an overkill. So for now let's just use the leaf +# component of the repository URL since it will be the same as the project +# name in most (sane) cases. +# +# First, strip the URL query and fragment parts, then prefix, and, finally, +# extension. +# +display_name="$(sed -r \ +-e 's%^([^?#]*).*$%\1%' \ +-e 's%^.*/([^/]+)/?$%\1%' \ +-e 's%(\.[^.]*)$%%' \ +<<<"$repository")" + +# Create the brep-load(1) loadtab file. +# +loadtab="$data_dir/loadtab" +run echo "$repository $display_name cache:cache" >"$loadtab" + +# Load the repository into the brep package database. +# +# Note that for now we load all the packages the repository contains without +# regard to the request manifest package values. Later, we could add filtering +# of the packages.manifest file against the request manifest values. While at +# it, we could also deduce the repository display name (see above). @@ TODO +# +run "$loader" "${loader_options[@]}" --force --shallow "$loadtab" + +# Remove the no longer needed CI request data directory. +# +run rm -r "$data_dir" + +trace "CI request for '$spec' is queued$message_suffix" +exit_with_manifest 200 "CI request is queued$message_suffix" diff --git a/brep/handler/ci/ci.bash.in b/brep/handler/ci/ci.bash.in index 023e98e..d5af34b 100644 --- a/brep/handler/ci/ci.bash.in +++ b/brep/handler/ci/ci.bash.in @@ -39,3 +39,27 @@ function exit_with_manifest () # manifest_serializer_finish run exit 0 } + +# Dump the repositories.manifest and packages.manifest files into the +# specified directory by running bpkg-rep-info command for the specified +# repository and using the specified fetch timeout (in seconds). +# +function dump_repository_manifests () # +{ + local url="$1" + local dir="$2" + local tmo="$3" + + if ! run_silent bpkg rep-info --fetch-timeout "$tmo" --manifest \ +--repositories --repositories-file "$dir/repositories.manifest" \ +--packages --packages-file "$dir/packages.manifest" "$url"; then + + # Perform the sanity check to make sure that bpkg is runnable. + # + if ! run bpkg --version >/dev/null; then + error "unable to run bpkg" + fi + + exit_with_manifest 422 "unable to fetch repository information (run 'bpkg rep-info $url' for details)" + fi +} diff --git a/brep/handler/submit/submit-dir.in b/brep/handler/submit/submit-dir.in index bda0bf8..685cf06 100644 --- a/brep/handler/submit/submit-dir.in +++ b/brep/handler/submit/submit-dir.in @@ -99,7 +99,7 @@ if [ -z "$project" ]; then fi if [ -n "$simulate" ]; then - rm -r "$data_dir" + run rm -r "$data_dir" trace "$name/$version submission is simulated" else trace "$name/$version submission is queued" diff --git a/brep/handler/submit/submit-git.bash.in b/brep/handler/submit/submit-git.bash.in index c77f18d..b1bb357 100644 --- a/brep/handler/submit/submit-git.bash.in +++ b/brep/handler/submit/submit-git.bash.in @@ -127,10 +127,7 @@ function repository_base () # { # First, strip the URL query part, then component. # - sed -n \ --e 's%^\([^?]*\).*$%\1%' \ --e 's%^\(.*/\)[^/]\{1,\}/\{0,1\}$%\1%p' \ -<<<"$1" + sed -n -r -e 's%^([^?]*).*$%\1%' -e 's%^(.*/)[^/]+/?$%\1%p' <<<"$1" } # Authenticate the project name owner. Make sure that the control manifest @@ -296,7 +293,7 @@ function auth_package_unknown () # # function url_scheme () # { - sed -n -e 's%^\(.*\)://.*$%\L\1%p' <<<"$1" + sed -n -re 's%^(.*)://.*$%\L\1%p' <<<"$1" } # Check that the repository properly responds to the probing request before @@ -320,8 +317,8 @@ function check_connectivity () # if [ "$s" == "http" -o "$s" == "https" ]; then local u q - u="$(sed -n -e 's%^\([^?]*\).*$%\1%p' <<<"$url")" # Strips query part. - q="$(sed -n -e 's%^[^?]*\(.*\)$%\1%p' <<<"$url")" # Query part. + u="$(sed -n -re 's%^([^?]*).*$%\1%p' <<<"$url")" # Strips query part. + q="$(sed -n -re 's%^[^?]*(.*)$%\1%p' <<<"$url")" # Query part. if [ -z "$q" ]; then u="$u/info/refs?service=git-upload-pack" diff --git a/brep/handler/submit/submit-git.in b/brep/handler/submit/submit-git.in index a403377..bc8ecf0 100644 --- a/brep/handler/submit/submit-git.in +++ b/brep/handler/submit/submit-git.in @@ -387,7 +387,7 @@ function git_add () # ... run git -C "$d" add $gvo "$@" >&2 } -# Dor now we make 10 re-tries to add the package and push to target. Push can +# For now we make 10 re-tries to add the package and push to target. Push can # fail due to the target-to-reference information move race (see the above # notes for details) or because concurrent submissions. We may want to make it # configurable in the future. diff --git a/brep/handler/submit/submit.bash.in b/brep/handler/submit/submit.bash.in index 38a4e06..8ea510f 100644 --- a/brep/handler/submit/submit.bash.in +++ b/brep/handler/submit/submit.bash.in @@ -49,6 +49,7 @@ function extract_package_manifest () # local man="$2" if ! run_silent bpkg pkg-verify --manifest "$arc" >"$man"; then + # Perform the sanity check to make sure that bpkg is runnable. # if ! run bpkg --version >/dev/null; then -- cgit v1.1