From e97c6b9b9c90a6f97019021d6bfcc73ed92580cc Mon Sep 17 00:00:00 2001 From: Karen Arutyunov Date: Fri, 14 Sep 2018 22:33:30 +0300 Subject: Filter packages manifest against CI request manifest in ci-load --- brep/handler/ci/ci-dir.in | 2 +- brep/handler/ci/ci-load.in | 208 ++++++++++++++++----- brep/handler/handler.bash.in | 2 +- brep/handler/submit/submit-dir.in | 2 +- brep/handler/submit/submit-git.in | 2 +- .../request.manifest | 1 - tests/ci/README | 29 ++- tests/ci/ci-dir.testscript | 8 + tests/ci/ci-load.testscript | 172 ++++++++++++++--- tests/ci/data.testscript | 9 + tests/ci/hello.tar.gz | Bin 0 -> 102400 bytes tests/submit/README | 22 ++- tests/submit/submit-dir.testscript | 8 + tests/submit/submit-git.testscript | 8 + 14 files changed, 377 insertions(+), 96 deletions(-) create mode 100644 tests/ci/hello.tar.gz diff --git a/brep/handler/ci/ci-dir.in b/brep/handler/ci/ci-dir.in index 2915b25..ee4e88d 100644 --- a/brep/handler/ci/ci-dir.in +++ b/brep/handler/ci/ci-dir.in @@ -30,7 +30,7 @@ fi # CI request data directory (last and the only argument). # -data_dir="${!#/}" +data_dir="${!#%/}" if [ -z "$data_dir" ]; then error "$usage" diff --git a/brep/handler/ci/ci-load.in b/brep/handler/ci/ci-load.in index ad96943..e1eaa9c 100644 --- a/brep/handler/ci/ci-load.in +++ b/brep/handler/ci/ci-load.in @@ -68,7 +68,7 @@ done # CI request data directory (last argument). # -data_dir="$1" +data_dir="${1%/}" if [ -z "$data_dir" ]; then error "$usage" @@ -85,20 +85,50 @@ reference="$(basename "$data_dir")" # manifest_parser_start "$data_dir/request.manifest" -repository= -packages=() simulate= +repository= + +# Package map. We first enter packages from the request manifest as keys and +# setting the values to true. Then we go through the repository package list +# consulting this map and, if found, clearing the value to empty. Finally, we +# go through the map looking for any "unhandled" packages (value is still +# true). +# +# Note that keys can be in both and / forms. +# +declare -A packages + +# While at it, produce the bpkg-build(1)-like package spec for tracing. +# +# The spec normally contains the full commit id and so feels too hairy to +# include in the result manifest message. +# +spec= while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do case "$n" in - repository) repository="$v" ;; - package) packages+=("$v") ;; - simulate) simulate="$v" ;; + simulate) simulate="$v" ;; + repository) repository="$v" ;; + + package) + packages["$v"]=true + + if [ -n "$spec" ]; then + spec="$spec," + fi + spec="$spec$v" + ;; esac done manifest_parser_finish +if [ -n "$spec" ]; then + spec="$spec@" +fi + +spec="$spec$repository" + if [ -z "$repository" ]; then error "repository manifest value expected" fi @@ -107,25 +137,6 @@ if [ -n "$simulate" -a "$simulate" != "success" ]; then exit_with_manifest 400 "unrecognized simulation outcome '$simulate'" fi -# Produce the bpkg-build(1)-like package spec for tracing. -# -# The spec normally contains the full commit id and so feels too hairy to -# include in the result manifest message. -# -spec= -for p in "${packages[@]}"; do - if [ -n "$spec" ]; then - spec="$spec," - fi - spec="$spec$p" -done - -if [ -n "$spec" ]; then - spec="$spec@" -fi - -spec="$spec$repository" - message_suffix= if [ -n "$result_url" ]; then message_suffix=": $result_url/@$reference" # Append the tenant id. @@ -145,37 +156,140 @@ fi # Dump the repositories.manifest and packages.manifest files. # -run mkdir "$data_dir/cache" -dump_repository_manifests "$repository" "$data_dir/cache" "$fetch_timeout" +cache_dir="$data_dir/cache" +run mkdir "$cache_dir" +dump_repository_manifests "$repository" "$cache_dir" "$fetch_timeout" + +# Filter the packages manifest keeping only the packages listed in the request +# manifest. Keep all the packages if the request specified no packages. +# + +# Resulting manifest. +# +packages_manifest_names=() +packages_manifest_values=() -# In most cases all the requested for CI packages belong to the same project, -# which would be nice to use as the repository display name. However, that -# would require parsing packages.manifest just to get this information. Which -# feels like a bit of an overkill. So for now let's just use the leaf -# component of the repository URL since it will be the same as the project -# name in most (sane) cases. +# While at it, set the repository display name to the first package's project +# name. # -# First, strip the URL query and fragment parts, then prefix, and, finally, -# extension. +display_name= + +manifest_parser_start "$cache_dir/packages.manifest" + +# The outer loop iterates over package manifests while the inner loop iterates +# over manifest values in each such manifest. +# +# Note that the first manifest case is special in that we will see its version +# value (empty name) first while for others -- last, as part of the previous +# manifest. We try to deal with this irregularity by reducing the first case +# (manifest_version is empty) to "as-if" it followed another manifest. +# +manifest_names=() +manifest_values=() +manifest_version= + +more=true +while [ "$more" ]; do + + if [ -n "$manifest_version" ]; then + manifest_names=("") + manifest_values=("$manifest_version") + fi + + more= + project= + + while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + case "$n" in + "") # Start of (next) manifest. + more=true + manifest_version="$v" + break + ;; + + name) name="$v" ;; + version) version="$v" ;; + project) project="$v" ;; + esac + + manifest_names+=("$n") + manifest_values+=("$v") + + done + + # Reduce the first manifest case. + # + if [ ${#manifest_names[@]} -eq 0 ]; then + continue + fi + + # Add or filter out the manifest, if present. + # + if [ ${#packages[@]} -ne 0 ]; then + + if [[ -v packages["$name"] ]]; then + packages["$name"]= + packages["$name/$version"]= # Clear it either, as may also be present. + elif [[ -v packages["$name/$version"] ]]; then + packages["$name/$version"]= + else + continue # Skip. + fi + + fi + + packages_manifest_names+=("${manifest_names[@]}") + packages_manifest_values+=("${manifest_values[@]}") + + if [ -z "$display_name" ]; then + if [ -n "$project" ]; then + display_name="$project" + else + display_name="$name" + fi + fi +done + +manifest_parser_finish + +# Verify that all the listed in the request manifest packages are present in +# the repository. # -display_name="$(sed -r \ --e 's%^([^?#]*).*$%\1%' \ --e 's%^.*/([^/]+)/?$%\1%' \ --e 's%(\.[^.]*)$%%' \ -<<<"$repository")" +for p in "${!packages[@]}"; do + if [ "${packages[$p]}" ]; then + exit_with_manifest 422 "unknown package $p" + fi +done + +# Verify that the repository is not empty. Failed that, the repository display +# name wouldn't be set. +# +if [ -z "$display_name" ]; then + exit_with_manifest 422 "no packages in repository" +fi + +# Stash the original packages manifest file for troubleshooting. +# +run mv "$cache_dir/packages.manifest" "$cache_dir/packages.manifest.orig" + +# Serialize the filtered packages manifest. +# +manifest_serializer_start "$cache_dir/packages.manifest" + +for ((i=0; i <= ${#packages_manifest_names[@]}; ++i)); do + manifest_serialize "${packages_manifest_names[$i]}" \ + "${packages_manifest_values[$i]}" +done + +manifest_serializer_finish # Create the brep-load(1) loadtab file. # loadtab="$data_dir/loadtab" run echo "$repository $display_name cache:cache" >"$loadtab" -# Load the repository into the brep package database for the tenant identified -# by the reference. -# -# Note that for now we load all the packages the repository contains without -# regard to the request manifest package values. Later, we could add filtering -# of the packages.manifest file against the request manifest values. While at -# it, we could also deduce the repository display name (see above). @@ TODO +# Load the requested repository packages into the brep package database for +# the tenant identified by the reference. # run "$loader" "${loader_options[@]}" --force --shallow --tenant "$reference" \ "$loadtab" diff --git a/brep/handler/handler.bash.in b/brep/handler/handler.bash.in index 89e7e21..fa287a3 100644 --- a/brep/handler/handler.bash.in +++ b/brep/handler/handler.bash.in @@ -37,7 +37,7 @@ if [ "$#" -gt 0 ]; then # identifies the posted entity. A handler may overwrite this value if that's # not the case. # - info_ref="$(basename "${!#/}")" + info_ref="$(basename "${!#%/}")" fi function info () # diff --git a/brep/handler/submit/submit-dir.in b/brep/handler/submit/submit-dir.in index 1f677e4..63c191c 100644 --- a/brep/handler/submit/submit-dir.in +++ b/brep/handler/submit/submit-dir.in @@ -27,7 +27,7 @@ fi # Submission data directory (last and the only argument). # -data_dir="${!#/}" +data_dir="${!#%/}" if [ -z "$data_dir" ]; then error "$usage" diff --git a/brep/handler/submit/submit-git.in b/brep/handler/submit/submit-git.in index cc9927b..2835f7a 100644 --- a/brep/handler/submit/submit-git.in +++ b/brep/handler/submit/submit-git.in @@ -261,7 +261,7 @@ fi # Submission data directory. # -data_dir="$1" +data_dir="${1%/}" shift if [ -z "$data_dir" ]; then diff --git a/tests/ci/8716f424-fd94-4def-9e2e-687203bbf4ad/request.manifest b/tests/ci/8716f424-fd94-4def-9e2e-687203bbf4ad/request.manifest index 3ef553a..229e5ed 100644 --- a/tests/ci/8716f424-fd94-4def-9e2e-687203bbf4ad/request.manifest +++ b/tests/ci/8716f424-fd94-4def-9e2e-687203bbf4ad/request.manifest @@ -1,7 +1,6 @@ : 1 id: 8716f424-fd94-4def-9e2e-687203bbf4ad repository: https://git.build2.org/hello/hello.git#master -package: hello timestamp: 2018-09-01T08:38:55Z client-ip: fe80::56e1:adff:fe83:82f5 user-agent: curl/7.59.0 diff --git a/tests/ci/README b/tests/ci/README index 14defe5..94a87b6 100644 --- a/tests/ci/README +++ b/tests/ci/README @@ -1,11 +1,34 @@ Prepare the test data with the following instructions. +Create the git repository: + +$ mkdir hello.git +$ git -C hello.git/ init --bare + +Create the project: + +$ bdep new -t empty hello +$ BDEP_AUTHOR_EMAIL=user@example.org bdep new --package -t lib libhello -d hello +$ BDEP_AUTHOR_EMAIL=user@example.org bdep new --package -t exe hello -d hello + +Edit hello/libhello/manifest and hello/hello/manifest files setting version to +0.1.0. + +$ git -C hello remote add origin "$(pwd)/hello.git" +$ git -C hello add '*' +$ git -C hello commit -m "Create" +$ git -C hello push --set-upstream origin master + +$ tar cf hello.tar.gz hello.git/ + +Move the archive into tests/ci/ directory. + Locally run brep server configured to use ci-dir handler. $ curl \ --form-string repository=https://git.build2.org/hello/hello.git#master \ ---form-string package=hello \ http://localhost/pkg?ci -Replace the submission data directory in brep/tests/ci/ with the one produced -with the above command, removing all files it contains except request.manifest. +Replace the CI request data directory in tests/ci/ with the one produced by +the above command removing all entries it contains except request.manifest +file. diff --git a/tests/ci/ci-dir.testscript b/tests/ci/ci-dir.testscript index 8388616..f6ff9d3 100644 --- a/tests/ci/ci-dir.testscript +++ b/tests/ci/ci-dir.testscript @@ -17,6 +17,14 @@ $* $~/dir 2>>~%EOE% != 0 %\[.+\] \[brep:error\] \[ref dir\] \[brep-ci-dir\]: '.+dir' does not exist or is not a directory% EOE + + : slash-stripped + : + : Test that the trailing slash is stripped from the data directory path. + : + $* $~/dir/ 2>>~%EOE% != 0 + %\[.+\] \[brep:error\] \[ref dir\] \[brep-ci-dir\]: '.+dir' does not exist or is not a directory% + EOE } : success diff --git a/tests/ci/ci-load.testscript b/tests/ci/ci-load.testscript index c574a32..ce8daf9 100644 --- a/tests/ci/ci-load.testscript +++ b/tests/ci/ci-load.testscript @@ -23,6 +23,14 @@ $* brep-load $~/dir 2>>~%EOE% != 0 %\[.+\] \[brep:error\] \[ref dir\] \[brep-ci-load\]: '.+dir' does not exist or is not a directory% EOE + + : slash-stripped + : + : Test that the trailing slash is stripped from the data directory path. + : + $* brep-load $~/dir/ 2>>~%EOE% != 0 + %\[.+\] \[brep:error\] \[ref dir\] \[brep-ci-load\]: '.+dir' does not exist or is not a directory% + EOE } : success @@ -48,12 +56,13 @@ : for-real : { - # Create the loader script that validates the arguments passed and the - # files produced by the handler. + # Create the loader script that validates the arguments passed and prints + # the loadtab, repositories manifest, and packages manifest files to + # stderr. # loader=$~/brep-load - cat <<"EOI" >=$loader; + +cat <<"EOI" >=$loader #!/usr/bin/env bash if [ "\$#" != 7 -o \ "\$1" != "--db-host=localhost" -o \ @@ -62,38 +71,139 @@ "\$4" != "--shallow" -o \ "\$5" != "--tenant" -o \ -z "\$6" -o \ - "\$7" != "$data_dir/loadtab" -o \ - ! -f "$data_dir/cache/repositories.manifest" -o \ - ! -f "$data_dir/cache/packages.manifest" ]; then - exit 1 - fi - if ! diff "$data_dir/loadtab" - <<<"https://git.build2.org/hello/hello.git#master hello cache:cache"; then + "\$7" != "$data_dir/loadtab" ]; then exit 1 fi + cat \ + "$data_dir/loadtab" \ + "$data_dir/cache/repositories.manifest" \ + "$data_dir/cache/packages.manifest" >&2 EOI - chmod 755 $loader; - - $clone_root_data; - - $* $loader --db-host=localhost --db-port=8432 $data_dir >>"EOO"; - : 1 - status: 200 - message: CI request is queued - reference: $request_id - EOO - - $clone_root_data; - - $* --result-url "http://example.com/" \ - $loader --db-host=localhost --db-port=8432 \ - $data_dir >>~"%EOO%" - : 1 - status: 200 - %message: CI request is queued: http://example.com/@.+% - %. - reference: $request_id - EOO + +chmod 755 $loader + + : whole-repo + : + { + $clone_root_data; + + $* $loader --db-host=localhost --db-port=8432 $data_dir >>"EOO" 2>>~"%EOE%" + : 1 + status: 200 + message: CI request is queued + reference: $request_id + EOO + $rep hello cache:cache + : 1 + summary: hello project repository + : 1 + name: libhello + version: 0.1.0 + project: hello + summary: hello library + license: TODO + url: https://example.org/hello + email: user@example.org + %depends: \\* build2 .+% + %depends: \\* bpkg .+% + location: libhello + %fragment: .+% + : + name: hello + version: 0.1.0 + summary: hello executable + license: TODO + url: https://example.org/hello + email: user@example.org + %depends: \\* build2 .+% + %depends: \\* bpkg .+% + location: hello + %fragment: .+% + EOE + } + + : package + : + { + $clone_root_data; + + cat <+$data_dir/request.manifest; + package: hello + EOI + + $* $loader --db-host=localhost --db-port=8432 $data_dir >>"EOO" 2>>~"%EOE%" + : 1 + status: 200 + message: CI request is queued + reference: $request_id + EOO + $rep hello cache:cache + : 1 + summary: hello project repository + : 1 + name: hello + version: 0.1.0 + summary: hello executable + license: TODO + url: https://example.org/hello + email: user@example.org + %depends: \\* build2 .+% + %depends: \\* bpkg .+% + location: hello + %fragment: .+% + EOE + } + + : package-version + : + { + $clone_root_data; + + cat <+$data_dir/request.manifest; + package: libhello/0.1.0 + EOI + + $* $loader --db-host=localhost --db-port=8432 $data_dir >>"EOO" 2>>~"%EOE%" + : 1 + status: 200 + message: CI request is queued + reference: $request_id + EOO + $rep hello cache:cache + : 1 + summary: hello project repository + : 1 + name: libhello + version: 0.1.0 + project: hello + summary: hello library + license: TODO + url: https://example.org/hello + email: user@example.org + %depends: \\* build2 .+% + %depends: \\* bpkg .+% + location: libhello + %fragment: .+% + EOE + } + + : result-url + : + { + $clone_root_data; + + $* --result-url "http://example.com/" \ + $loader --db-host=localhost --db-port=8432 \ + $data_dir >>~"%EOO%" 2>>~%EOE% + : 1 + status: 200 + %message: CI request is queued: http://example.com/@.+% + %. + reference: $request_id + EOO + %.*:.*%+ + EOE + } } } diff --git a/tests/ci/data.testscript b/tests/ci/data.testscript index 711c7b1..662a14c 100644 --- a/tests/ci/data.testscript +++ b/tests/ci/data.testscript @@ -17,10 +17,19 @@ data_dir = $regex.replace($path_search('*/request.manifest', $src_base), \ request_id = "$data_dir" +# Prepare the repository. +# +rep="file:$~/hello.git#master" ++tar -C $~ -xf $src_base/hello.tar.gz &hello.git/*** + # Copy the original CI request data directory to the root scope. # +cp -r $src_base/$data_dir ./ +# Fix-up the repository value in the request manifest. +# ++sed -i -e "s%^\(repository:\) .+\$%\\1 $rep%" $data_dir/request.manifest + root_data_dir = $~/$data_dir # The most commonly used submission data directory cloning command that copies diff --git a/tests/ci/hello.tar.gz b/tests/ci/hello.tar.gz new file mode 100644 index 0000000..d264888 Binary files /dev/null and b/tests/ci/hello.tar.gz differ diff --git a/tests/submit/README b/tests/submit/README index 49ebd1e..1aecbd7 100644 --- a/tests/submit/README +++ b/tests/submit/README @@ -1,24 +1,26 @@ Prepare the test data with the following instructions. -In an empty directory run: +Create the git repository: + +$ mkdir hello.git +$ git -C hello.git/ init --bare + +Create the project: $ bdep new -t empty -C @cfg hello -$ BDEP_EMAIL=user@example.org bdep new --package -t lib libhello -d hello +$ BDEP_AUTHOR_EMAIL=user@example.org bdep new --package -t lib libhello -d hello $ bdep init -d hello/libhello Edit hello/libhello/manifest setting version to 0.1.0. -$ mkdir hello.git -$ git -C hello.git/ init --bare - $ git -C hello remote add origin "$(pwd)/hello.git" $ git -C hello add '*' $ git -C hello commit -m "Create" $ git -C hello push --set-upstream origin master -tar cf hello.tar.gz hello.git/ +$ tar cf hello.tar.gz hello.git/ -Move the archive into brep/tests/submit/ directory. +Move the archive into tests/submit/ directory. Locally run brep server configured to use submit-dir handler. @@ -29,6 +31,6 @@ $ bdep publish \ --yes \ -d hello -Replace the submission data directory in brep/tests/submit/ with the one -produced with the above command, removing package.manifest and result.manifest -files it contains. +Replace the submission data directory in tests/submit/ with the one produced +with the above command removing package.manifest and result.manifest files it +contains. diff --git a/tests/submit/submit-dir.testscript b/tests/submit/submit-dir.testscript index 0b43aa7..8018d01 100644 --- a/tests/submit/submit-dir.testscript +++ b/tests/submit/submit-dir.testscript @@ -17,6 +17,14 @@ $* $~/dir 2>>~%EOE% != 0 %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-dir\]: '.+dir' does not exist or is not a directory% EOE + + : slash-stripped + : + : Test that the trailing slash is stripped from the data directory path. + : + $* $~/dir/ 2>>~%EOE% != 0 + %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-dir\]: '.+dir' does not exist or is not a directory% + EOE } : success diff --git a/tests/submit/submit-git.testscript b/tests/submit/submit-git.testscript index e99c1d5..a0d76f5 100644 --- a/tests/submit/submit-git.testscript +++ b/tests/submit/submit-git.testscript @@ -82,6 +82,14 @@ pkg_ctl="$prj_ctl/hello.git" $* "$root_tgt_url" ref dir 2>>~%EOE% != 0 %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-git\]: 'dir' does not exist or is not a directory% EOE + + : slash-stripped + : + : Test that the trailing slash is stripped from the data directory path. + : + $* "$root_tgt_url" dir/ 2>>~%EOE% != 0 + %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-git\]: 'dir' does not exist or is not a directory% + EOE } : success -- cgit v1.1