diff options
| -rw-r--r-- | INSTALL | 2 | ||||
| -rw-r--r-- | brep/handler/handler.bash.in | 2 | ||||
| -rw-r--r-- | brep/handler/submit/.gitignore | 1 | ||||
| -rw-r--r-- | brep/handler/submit/buildfile | 4 | ||||
| -rw-r--r-- | brep/handler/submit/submit-dir.in | 9 | ||||
| -rw-r--r-- | brep/handler/submit/submit-git.in | 6 | ||||
| -rw-r--r-- | brep/handler/submit/submit-pub.in | 413 | ||||
| -rw-r--r-- | etc/buildfile | 21 | ||||
| -rw-r--r-- | etc/private/install/brep-apache2.conf | 99 | ||||
| -rwxr-xr-x | etc/private/install/brep-install | 433 | ||||
| -rw-r--r-- | etc/private/install/brep-load.service | 11 | ||||
| -rw-r--r-- | etc/private/install/brep-load.timer | 33 | ||||
| -rw-r--r-- | etc/private/install/brep-logrotate | 20 | ||||
| -rw-r--r-- | etc/private/install/brep-module.conf | 387 | ||||
| -rwxr-xr-x | etc/private/install/brep-startup | 67 | ||||
| -rw-r--r-- | etc/private/install/brep-startup.service | 17 | ||||
| -rwxr-xr-x | etc/private/install/vm-gen-service | 204 | ||||
| -rwxr-xr-x | etc/private/vm-login | 30 | ||||
| -rwxr-xr-x | etc/private/vm-start | 98 | ||||
| -rwxr-xr-x | etc/private/vm-start-base | 187 | ||||
| -rwxr-xr-x | etc/private/vm-stop | 24 | ||||
| -rw-r--r-- | mod/external-handler.cxx | 16 | ||||
| -rw-r--r-- | tests/submit/buildfile | 9 | ||||
| -rw-r--r-- | tests/submit/submit-pub.testscript | 175 |
24 files changed, 2243 insertions, 25 deletions
@@ -44,7 +44,7 @@ b) Install PostgreSQL 9.3 or above (including the contrib package containing the postgres_fdw extension) as well as Apache2 using your distribution's packages. Below are the names of these packages for some distributions: - Debian/Ubuntu: postgresql-server postgresql-contrib apache2 + Debian/Ubuntu: postgresql postgresql-contrib apache2 Fedora/RHEL: postgresql-server postgresql-contrib httpd FreeBSD: postgresqlXY-server postgresqlXY-contrib apache24 diff --git a/brep/handler/handler.bash.in b/brep/handler/handler.bash.in index b61bbdb..1169b99 100644 --- a/brep/handler/handler.bash.in +++ b/brep/handler/handler.bash.in @@ -51,7 +51,7 @@ function info () # <severity> <text> ts= fi - echo "[$ts] [brep:$severity] [ref $info_ref] [$info_self]: $*" 1>&2; + echo "[$ts] [brep:$severity] [ref $info_ref] [$info_self]: $*" 1>&2 } function error () { info "error" "$*"; exit 1; } diff --git a/brep/handler/submit/.gitignore b/brep/handler/submit/.gitignore index cbbd541..098bf75 100644 --- a/brep/handler/submit/.gitignore +++ b/brep/handler/submit/.gitignore @@ -1,2 +1,3 @@ brep-submit-dir brep-submit-git +brep-submit-pub diff --git a/brep/handler/submit/buildfile b/brep/handler/submit/buildfile index d6e04dc..1747c64 100644 --- a/brep/handler/submit/buildfile +++ b/brep/handler/submit/buildfile @@ -1,7 +1,7 @@ # file : brep/handler/submit/buildfile # license : MIT; see accompanying LICENSE file -./: exe{brep-submit-dir} exe{brep-submit-git} +./: exe{brep-submit-dir} exe{brep-submit-git} exe{brep-submit-pub} include ../ @@ -10,5 +10,7 @@ exe{brep-submit-dir}: in{submit-dir} bash{submit} ../bash{handler} exe{brep-submit-git}: in{submit-git} \ bash{submit-git} bash{submit} ../bash{handler} +exe{brep-submit-pub}: in{submit-pub} bash{submit} ../bash{handler} + bash{submit}: in{submit} ../bash{handler} bash{submit-git}: in{submit-git} bash{submit} ../bash{handler} diff --git a/brep/handler/submit/submit-dir.in b/brep/handler/submit/submit-dir.in index 16065e6..b28ab38 100644 --- a/brep/handler/submit/submit-dir.in +++ b/brep/handler/submit/submit-dir.in @@ -66,20 +66,17 @@ fi m="$data_dir/package.manifest" extract_package_manifest "$data_dir/$archive" "$m" -# Parse the package manifest and obtain the package name, version, and -# project. +# Parse the package manifest and obtain the package name and version. # manifest_parser_start "$m" name= version= -project= while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do case "$n" in name) name="$v" ;; version) version="$v" ;; - project) project="$v" ;; esac done @@ -93,10 +90,6 @@ if [ -z "$version" ]; then error "version manifest value expected" fi -if [ -z "$project" ]; then - project="$name" -fi - if [ -n "$simulate" ]; then run rm -r "$data_dir" trace "package submission is simulated: $name/$version" diff --git a/brep/handler/submit/submit-git.in b/brep/handler/submit/submit-git.in index 34b1e90..54cd230 100644 --- a/brep/handler/submit/submit-git.in +++ b/brep/handler/submit/submit-git.in @@ -439,8 +439,12 @@ for i in {1..11}; do trace "+ exec {fd}<$l" exec {fd}<"$l" + # Note that on the locking failure we don't suggest the user to try again, + # since the client program may suggest to re-try later for all server + # errors (as bdep-publish(1) does). + # if ! run flock -w "$ref_lock_timeout" "$fd"; then - exit_with_manifest 503 "submission service temporarily unavailable" + exit_with_manifest 503 "submission service is busy" fi # Pull the reference repository. diff --git a/brep/handler/submit/submit-pub.in b/brep/handler/submit/submit-pub.in new file mode 100644 index 0000000..d262ae9 --- /dev/null +++ b/brep/handler/submit/submit-pub.in @@ -0,0 +1,413 @@ +#!/usr/bin/env bash + +# file : brep/handler/submit/submit-pub.in +# license : MIT; see accompanying LICENSE file + +# Package submission handler with direct repository publishing. +# +# The overall idea behind this handler is to directly add the package to a +# private/trusted (unsigned) pkg repository with a simple structure (no +# sections). Upon successful execution of this handler no additional steps are +# required. +# +# Specifically, the handler performs the following steps: +# +# - Lock the repository directory for the duraton of the package submission. +# +# - Check for the package duplicate. +# +# - Create the new repository as a hardlink-copy of the current one. +# +# - Remove any package revisions, if present. +# +# - Validate and add the package archive to the new repository (with project +# subdirectory). +# +# - Re-generate the new repository without signing. +# +# - Verify that the new repository is loadable into the brep package database. +# +# - Atomically switch the repository symlink to refer to the new repository. +# +# - Release the lock and remove the old repository. +# +# The repository argument (<repo>) should be an absolute path to a symbolic +# link to the pkg repository directory, with the archive and manifest files +# residing in its 1/ subdirectory. The base name of the <repo> path is used +# as a base for new repository directories. +# +# Unless the handler is called for testing, the loader program's absolute path +# and options should be specified so that the handler can verify that the +# package is loadable into the brep package database (this makes sure the +# package dependencies are resolvable, etc). +# +# Notes: +# +# - Filesystem entries that exist or are created in the data directory: +# +# <pkg>-<ver>.tar.gz saved by brep (could be other archives in the future) +# request.manifest created by brep +# package.manifest extracted by the handler +# loadtab created by the handler +# result.manifest saved by brep +# +# Options: +# +# --user <name> +# +# Re-execute itself under the specified user. +# +# Note that the repository can also be modified manually (e.g., to remove +# packages). This option is normally specified to make sure that all the +# repository filesystem entries belong to a single user, which, in +# particular, can simplify their permissions handling (avoid extra ACLs, +# etc). +# +# Note that if this option is specified, then current user (normally the +# user under which Apache2 is running) must be allowed to execute sudo +# without a password, which is only recommended in private/trusted +# environments. +# +# --result-url <url> +# +# Result URL base for the response. If specified, the handler appends the +# <package>/<version> to this value and includes the resulting URL in the +# response message. +# +usage="usage: $0 [<options>] [<loader-path> <loader-options>] <repo> <dir>" + +# Diagnostics. +# +verbose= #true + +# The repository lock timeout (seconds). +# +rep_lock_timeout=60 + +trap "{ exit 1; }" ERR +set -o errtrace # Trap ERR in functions. + +@import brep/handler/handler@ +@import brep/handler/submit/submit@ + +# Parse the command line options and, while at it, compose the arguments array +# for potential re-execution under a different user. +# +user= +result_url= + +scr_exe="$(realpath "${BASH_SOURCE[0]}")" +scr_dir="$(dirname "$scr_exe")" + +args=("$scr_exe") + +while [ "$#" -gt 0 ]; do + case $1 in + --user) + shift + user="$1" + shift + ;; + --result-url) + args+=("$1") + shift + result_url="${1%/}" + args+=("$1") + shift + ;; + *) + break; # The end of options is encountered. + ;; + esac +done + +loader_args=() # The loader path and options. + +# Assume all the remaining arguments except for the last two (repository +# symlink and data directory) as the loader program path and arguments. +# +while [ "$#" -gt 2 ]; do + loader_args+=("$1") + args+=("$1") + shift +done + +if [ "$#" -ne 2 ]; then + error "$usage" +fi + +# pkg repository symlink. +# +repo="${1%/}" +shift + +if [ -z "$repo" ]; then + error "$usage" +fi + +# Submission data directory. +# +data_dir="${1%/}" +shift + +if [ -z "$data_dir" ]; then + error "$usage" +fi + +# Re-execute itself under a different user, if requested. +# +if [ -n "$user" ]; then + args+=("$repo" "$data_dir") + + # Compose the arguments string to pass to the su program, quoting empty + # arguments as well as those that contain spaces. Note that here, for + # simplicity, we assume that the arguments may not contain '"'. + # + as= + for a in "${args[@]}"; do + if [ -z "$a" -o -z "${a##* *}" ]; then + a="\"$a\"" + fi + if [ -n "$as" ]; then + a=" $a" + fi + as="$as$a" + done + + run exec sudo --non-interactive su -l "$user" -c "$as" +fi + +# Check path presence (do it after user switch for permissions). +# +if [ ! -L "$repo" ]; then + error "'$repo' does not exist or is not a symlink" +fi + +if [ ! -d "$data_dir" ]; then + error "'$data_dir' does not exist or is not a directory" +fi + +reference="$(basename "$data_dir")" + +# Parse the submission request manifest and obtain the archive path as well as +# the simulate value. +# +manifest_parser_start "$data_dir/request.manifest" + +archive= +simulate= + +while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + case "$n" in + archive) archive="$v" ;; + simulate) simulate="$v" ;; + esac +done + +manifest_parser_finish + +if [ -z "$archive" ]; then + error "archive manifest value expected" +fi + +if [ -n "$simulate" -a "$simulate" != "success" ]; then + exit_with_manifest 400 "unrecognized simulation outcome '$simulate'" +fi + +m="$data_dir/package.manifest" +extract_package_manifest "$data_dir/$archive" "$m" + +# Parse the package manifest and obtain the package name, version, and +# project. +# +manifest_parser_start "$m" + +name= +version= +project= + +while IFS=: read -ru "$manifest_parser_ofd" -d '' n v; do + case "$n" in + name) name="$v" ;; + version) version="$v" ;; + project) project="$v" ;; + esac +done + +manifest_parser_finish + +if [ -z "$name" ]; then + error "name manifest value expected" +fi + +if [ -z "$version" ]; then + error "version manifest value expected" +fi + +if [ -z "$project" ]; then + project="$name" +fi + +if [ -n "$result_url" ]; then + message_suffix=": $result_url/$name/$version" +else + message_suffix=": $name/$version" +fi + +# Open the reading file descriptor and lock the repository. Fail if unable to +# lock before timeout. +# +l="$repo.lock" +run touch "$l" +trace "+ exec {lfd}<$l" +exec {lfd}<"$l" + +# Note that on the locking failure we don't suggest the user to try again, +# since the client program may suggest to re-try later for all server errors +# (as bdep-publish(1) does). +# +if ! run flock -w "$rep_lock_timeout" "$lfd"; then + exit_with_manifest 503 "submission service is busy" +fi + +repo_old="$(realpath "$repo")" # Old repo path. +repo_name="$(basename "$repo")-$(date "+%Y%m%d-%H%M%S-%N")" # New repo name. +repo_new="$(dirname "$repo_old")/$repo_name" # New repo path. +repo_link="$repo_new.link" # New repo symlink. + +# On exit, remove the new repository symlink and directory, unless the link +# doesn't exist or the directory removal is canceled (for example, the new +# repository is made current). +# +function exit_trap () +{ + if [ -L "$repo_link" ]; then + run rm -r -f "$repo_link" + fi + + if [ -n "$repo_new" -a -d "$repo_new" ]; then + run rm -r -f "$repo_new" + fi +} + +trap exit_trap EXIT + +# Check for the package duplicate (in all projects). +# +if [ -n "$(run find "$repo_old/1" -name "$archive")" ]; then + exit_with_manifest 422 "duplicate submission" +fi + +# Copy the current repository using hardlinks. +# +# -r (recursive) +# -t (preserve timestamps) +# -O (omit dir timestamps) +# --link-dest (hardlink files instead of copying) +# +# We also exclude the packages.manifest file that will be re-generated anyway. +# +run rsync -rtO --exclude 'packages.manifest' --link-dest="$repo_old" \ + "$repo_old/" "$repo_new" + +# Remove the package version revisions that may exist in the repository. +# +# Strips the version revision part, if present. +# +v="$(sed -n -re 's%^(\+?[^+]+)(\+[0-9]+)?$%\1%p' <<<"$version")" + +# Go through the potentially matching archives (for example, for foo-1.2.3+2: +# foo-1.2.3.tar.gz, foo-1.2.3+1.tar.gz, foo-1.2.30.tar.gz, etc) and remove +# those that match exactly. +# +# Change CWD to the section directory to make sure that the found archive +# paths don't contain spaces. +# +fs=($(run cd "$repo_new/1" && run find -name "$name-$v*")) + +for f in "${fs[@]}"; do + if [[ "$f" =~ ^\./[^/]+/"$name-$v"(\+[0-9]+)?\.[^/]+$ ]]; then + run rm "$repo_new/1/$f" >&2 + fi +done + +# Copy the archive rather than moving it since we may need it for +# troubleshooting. Note: the data and repository directories can be on +# different filesystems and so hardlinking could fail. +# +run mkdir -p "$repo_new/1/$project" +run cp "$data_dir/$archive" "$repo_new/1/$project" + +# Create the new repository. +# +# Note that if bpkg-rep-create fails, we can't reliably distinguish if this is +# a user or internal error (broken package archive vs broken repository). +# Thus, we always treat is as a user error, providing the full error +# description in the response and assuming that the submitter can either fix +# the issue or report it to the repository maintainers. This again assumes +# private/trusted environment. +# +trace "+ bpkg rep-create '$repo_new/1' 2>&1" + +if ! e="$(bpkg rep-create "$repo_new/1" 2>&1)"; then + exit_with_manifest 400 "submitted archive is not a valid package +$e" +fi + +# If requested, verify that the new repository is loadable into the package +# database and, as in the above case, treat the potential error as a user +# error. +# +if [ "${#loader_args[@]}" -ne 0 ]; then + f="$data_dir/loadtab" + echo "http://testrepo/1 private cache:$repo_new/1" >"$f" + + trace "+ ${loader_args[@]} '$f' 2>&1" + + if ! e="$("${loader_args[@]}" "$f" 2>&1)"; then + + # Sanitize the error message, removing the confusing lines. + # + e="$(run sed -re '/testrepo/d' <<<"$e")" + exit_with_manifest 400 "unable to add package to repository +$e" + fi +fi + +# Finally, create the new repository symlink and replace the current symlink +# with it, unless we are simulating. +# +run ln -sf "$repo_name" "$repo_link" + +if [ -z "$simulate" ]; then + run mv -T "$repo_link" "$repo" # Switch the repository symlink atomically. + + # Now, when the repository link is switched, disable the new repository + # removal. + # + # Note that we still can respond with an error status. However, the + # remaining operations are all cleanups and thus unlikely to fail. + # + repo_new= +fi + +trace "+ exec {lfd}<&-" +exec {lfd}<&- # Close the file descriptor and unlock the repository. + +# Remove the old repository, unless we are simulating. +# +# Note that if simulating, we leave the new repository directory/symlink +# removal to the exit trap (see above). +# +if [ -z "$simulate" ]; then + run rm -r "$repo_old" + + what="published" +else + what="simulated" +fi + +run rm -r "$data_dir" + +trace "package is $what$message_suffix" +exit_with_manifest 200 "package is published$message_suffix" diff --git a/etc/buildfile b/etc/buildfile index 1b78f65..f3157f2 100644 --- a/etc/buildfile +++ b/etc/buildfile @@ -3,11 +3,28 @@ ./: file{** -buildfile} -# Install into the etc/ subdirectory of, say, /usr/share/ -# recreating subdirectories. +# Install into the etc/ subdirectory of, say, /usr/share/ recreating +# subdirectories. # *: { install = data/etc/ install.subdirs = true } + +# Keep the executable permission for the installed script files. +# +private/ +{ + file{vm-start}@./ \ + file{vm-start-base}@./ \ + file{vm-login}@./ \ + file{vm-stop}@./: install.mode=755 + + install/ + { + file{brep-install}@./ \ + file{brep-startup}@./ \ + file{vm-gen-service}@./ : install.mode=755 + } +} diff --git a/etc/private/install/brep-apache2.conf b/etc/private/install/brep-apache2.conf new file mode 100644 index 0000000..99186d1 --- /dev/null +++ b/etc/private/install/brep-apache2.conf @@ -0,0 +1,99 @@ +# Keep in the main server configuration context. This way the directive will +# be in effect during module initialization and request handling. +# +# Note that initialization log messages are written to the main server log +# file (/var/log/apache2/error.log), and request handling messages to the +# virtual server log file (/var/www/brep/log/error.log). +# +LogLevel brep:info + +<VirtualHost *:80> + #ServerName <brep-hostname> + #ServerAdmin <brep-admin-email> + + #DocumentRoot /var/www/brep/public + #Options +Indexes + + AddOutputFilterByType DEFLATE application/xhtml+xml + AddOutputFilterByType DEFLATE text/manifest + AddOutputFilterByType DEFLATE text/plain + AddOutputFilterByType DEFLATE text/css + + Alias "/1" "/var/brep/bpkg/pkg/1" + + ErrorLog /var/www/brep/log/error.log + CustomLog /var/www/brep/log/access.log combined + + # brep configuration + # + + # Load the brep module. + # + <IfModule !brep_module> + LoadModule brep_module /home/brep/install/libexec/brep/mod_brep.so + </IfModule> + + # Repository email. This email is used for the From: header in emails send + # by brep (for example, build failure notifications). + # + #brep-email <brep-admin-email> + + # Repository host. It specifies the scheme and the host address (but not the + # root path; see brep-root below) that will be used whenever brep needs to + # construct an absolute URL to one of its locations (for example, a link to + # a build log that is being send via email). + # + #brep-host http://<brep-hostname> + + # Repository root. This is the part of the URL between the host name and the + # start of the repository. For example, root value /pkg means the repository + # URL is http://example.org/pkg/. Specify / to use the web server root + # (e.g., http://example.org/). If using a different repository root, don't + # forget to also change Location and Alias directives below. + # + brep-root / + + <Location "/"> + SetHandler brep + + <IfModule dir_module> + DirectoryIndex disabled + DirectorySlash Off + </IfModule> + </Location> + + # Brep module configuration. If you prefer, you can paste the contents of + # this file here. However, you will need to prefix every option with + # 'brep-'. + # + brep-conf /home/brep/config/brep-module.conf + + # Static brep content (CSS files). + # + <IfModule !alias_module> + Error "mod_alias is not enabled" + </IfModule> + + # Note: trailing slashes are important! + # + Alias /@/ /home/brep/install/share/brep/www/ + + <Directory "/home/brep/install/share/brep/www"> + Require all granted + </Directory> + + # brep config override (must come after). + # + <LocationMatch "^/([0-9]|icons)(/.*)?$"> + SetHandler none + + DirectoryIndex enabled + DirectorySlash On + </LocationMatch> +</VirtualHost> + +<Directory /var/brep/bpkg/pkg/> + Options Indexes FollowSymLinks + AllowOverride None + Require all granted +</Directory> diff --git a/etc/private/install/brep-install b/etc/private/install/brep-install new file mode 100755 index 0000000..0c6c2e6 --- /dev/null +++ b/etc/private/install/brep-install @@ -0,0 +1,433 @@ +#! /usr/bin/env bash + +# file : etc/private/install/brep-install +# license : MIT; see accompanying LICENSE file + +# Setup HTTP-only brep instance with unsigned package submission support via +# direct repository publishing (brep/handler/submit/submit-pub). +# +# NOTE: this setup should only be used in private/trusted environments. +# +# Unless the --setup option is specified, create the 'brep' group and user and +# re-run itself with the --setup option under this user. In the setup mode +# install and configure the brep instance, automating the instructions from +# the INSTALL file, including: +# +# - Build the build2 toolchain (installing it to /usr/local/) and brep +# (installing it to ~brep/install/). +# +# - Install PostgreSQL and create brep users/databases. +# +# - Installing Apache2 and configure HTTP server with the brep module. +# +# Note that the script is written for use on Debian-based distributions so you +# will need to adjust it to match other distributions or operating systems. +# +# Options: +# +# --mount +# +# Mount the virtio-9p device with id 'state' as the /var/brep directory. +# This directory is expected to either contain the pkg repository or be +# empty, in which case an empty repository will be automatically +# initialized. If this option is unspecified, the directory will be created +# in the local filesystem. +# +# --brep-user +# +# User and group ids to use when creating the 'brep' group and user. If +# unspecified, 63700 is used. +# +# --setup +# +# Install and configure the brep instance, assuming that the 'brep' user +# already exists and this script is executed as this user. +# +usage="Usage: $0 [<options>]" + +# build2 toolchain repository certificate fingerprint. Note: this is a +# repository the toolchain installation script downloads the build2 packages +# from. +# +toolchain_repo_cert_fp="86:BA:D4:DE:2C:87:1A:EE:38:C7:F1:64:7F:65:77:02:15:79:F3:C4:83:C0:AB:5A:EA:F4:F7:8C:1D:63:30:C6" +#toolchain_repo_cert_fp="37:CE:2C:A5:1D:CF:93:81:D7:07:46:AD:66:B3:C3:90:83:B8:96:9E:34:F0:E7:B3:A2:B0:6C:EF:66:A4:BE:65" + +# brep package repository URL and certificate fingerprint. +# +#brep_repo_url="https://pkg.cppget.org/1/alpha" +#brep_repo_cert_fp="86:BA:D4:DE:2C:87:1A:EE:38:C7:F1:64:7F:65:77:02:15:79:F3:C4:83:C0:AB:5A:EA:F4:F7:8C:1D:63:30:C6" +brep_repo_url="https://stage.build2.org/1" +brep_repo_cert_fp="37:CE:2C:A5:1D:CF:93:81:D7:07:46:AD:66:B3:C3:90:83:B8:96:9E:34:F0:E7:B3:A2:B0:6C:EF:66:A4:BE:65" + +owd=`pwd` +trap "{ exit 1; }" ERR +trap "{ cd $owd; }" EXIT +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "error: $*"; exit 1; } + +# Trace a command line, quoting empty arguments as well as those that contain +# spaces. +# +function trace () # <cmd> <arg>... +{ + local s="+" + while [ "$#" -gt 0 ]; do + if [ -z "$1" -o -z "${1##* *}" ]; then + s="$s \"$1\"" + else + s="$s $1" + fi + + shift + done + + info "$s" +} + +# Trace and run a command. +# +run () # <args>... +{ + trace "$@" + "$@" +} + +# The chosen fixed id for the 'brep' user. Note: must match the id of the +# 'brep' user on the host. +# +# Note that Linux assigns the [0 99] range for the statically allocated system +# users and [100 499] -- for dynamic allocations by administrators and post- +# install scripts. Debian, in turn, assigns the [100 999] range for the +# dynamically allocated system users and [60000 64999] -- for statically +# allocated on demand "obscure package users". +# +brep_id=63700 # Update the README file on change. + +# Parse the command line options and, while at it, compose the options array +# for potential re-execution as the 'brep' user. +# +mount= +setup= +ops=() + +while [ "$#" -gt 0 ]; do + case "$1" in + --mount) + mount=true + ops+=("$1") + shift + ;; + --brep-user) + shift + brep_id="$1" + shift + ;; + --setup) + setup=true + shift + ;; + *) + break # The end of options is encountered. + ;; + esac +done + +if [ "$#" -ne 0 ]; then + error "$usage" +fi + +scr_exe="$(realpath "${BASH_SOURCE[0]}")" +scr_dir="$(dirname "$scr_exe")" + +# Unless we are not in the setup mode, non-interactively add the 'brep' +# user/group and re-execute the script in the setup mode as this user. +# +if [ ! "$setup" ]; then + run sudo addgroup --gid "$brep_id" brep + + run sudo adduser --uid "$brep_id" --gid "$brep_id" --disabled-password \ + --gecos "" brep + + run sudo tee -a /etc/sudoers.d/brep >/dev/null <<EOF +brep ALL=(ALL) NOPASSWD:ALL +EOF + + run sudo chmod 0440 /etc/sudoers.d/brep + + # Use --session-command rather than --command|-c to make sure that when the + # su program receives SIGINT (Ctrl-C) it kills not just its child process + # but also all its descendants. + # + # Note: here we rely on ops to not contain spaces or be empty. + # + run exec sudo su -l brep --session-command "'$scr_exe' --setup ${ops[@]}" +fi + +# Here we assume that we are executed as brep user. +# +run cd "$HOME" + +# Mount the brep state directory, if requested. Note that otherwise, the +# directory will be created later, in the local filesystem by the brep-startup +# script. +# +if [ "$mount" ]; then + run sudo mkdir -p /var/brep + + run sudo tee -a /etc/fstab >/dev/null <<EOF +state /var/brep 9p trans=virtio,version=9p2000.L,posixacl,cache=none,_netdev 0 0 +EOF + + run sudo mount -a +fi + +# Install the prerequisite binary packages. +# +run sudo apt-get --yes update +run sudo apt-get --yes install --no-install-recommends g++ +run sudo apt-get --yes install --no-install-recommends postgresql postgresql-contrib libpq-dev +run sudo apt-get --yes install --no-install-recommends apache2 libapr1-dev libapreq2-dev apache2-dev +run sudo apt-get --yes install --no-install-recommends acl rsync +run sudo apt-get clean + +# Install build2 toolchain. +# +run mkdir build2-build +run cd build2-build + +# Look for the toolchain installation script in this script directory. +# +run cp "$(echo "$scr_dir"/build2-install-*.sh)" . +run sh ./build2-install-*.sh --no-check --yes --trust "$toolchain_repo_cert_fp" +#run sh ./build2-install-*.sh --no-check --yes --local + +run cd .. # Back to brep home. + +# Grant Apache2 read access to the module and configuration. +# +run setfacl -m g:www-data:rx "$HOME" +run setfacl -dm g:www-data:rx "$HOME" + +# Install brep. +# +run mkdir brep +run cd brep + +run bpkg create \ + cc \ + config.cc.coptions="-O3" \ + config.cc.poptions="-I$(apxs -q includedir)" \ + config.bin.lib=shared \ + config.bin.rpath="$HOME/install/lib" \ + config.install.root="$HOME/install" + +run bpkg add "$brep_repo_url" +run bpkg fetch --trust "$brep_repo_cert_fp" +run bpkg build --yes brep ?sys:libapr1 ?sys:libapreq2 ?sys:libpq +run bpkg install brep + +run cd .. # Back to brep home. + +# Create PostgreSQL user and databases. +# +# Note that while we could probably omit the build-related setup, let's keep +# it to stay close to the instructions in the INSTALL file and to simplify the +# potential future configuration of the brep instance as a build2 build bot +# controller. +# +run sudo sudo -u postgres psql <<EOF +CREATE DATABASE brep_package +TEMPLATE template0 +ENCODING 'UTF8' +LC_COLLATE 'en_US.UTF8' +LC_CTYPE 'en_US.UTF8'; + +CREATE DATABASE brep_build +TEMPLATE template0 +ENCODING 'UTF8' +LC_COLLATE 'en_US.UTF8' +LC_CTYPE 'en_US.UTF8'; + +CREATE USER brep; + +GRANT ALL PRIVILEGES ON DATABASE brep_package, brep_build TO brep; + +CREATE USER "www-data" INHERIT IN ROLE brep; + +CREATE USER "brep-build" INHERIT IN ROLE brep PASSWORD '-'; +EOF + +# Create the "staging" package database for the submit-pub package submission +# handler. +# +run sudo sudo -u postgres psql <<EOF +CREATE DATABASE brep_submit_package +TEMPLATE template0 +ENCODING 'UTF8' +LC_COLLATE 'en_US.UTF8' +LC_CTYPE 'en_US.UTF8'; + +GRANT ALL PRIVILEGES ON DATABASE brep_submit_package TO brep; +EOF + +# Make sure the 'brep' and Apache2 user's logins work properly. +# +q="SELECT current_database();" +run psql -d brep_package -c "$q" >/dev/null +run psql -d brep_build -c "$q" >/dev/null +run psql -d brep_submit_package -c "$q" >/dev/null + +run sudo sudo -u www-data psql -d brep_package -c "$q" >/dev/null +run sudo sudo -u www-data psql -d brep_build -c "$q" >/dev/null + +# Setup the connection between the databases. +# +run sudo sudo -u postgres psql -d brep_build <<EOF +CREATE EXTENSION postgres_fdw; + +CREATE SERVER package_server +FOREIGN DATA WRAPPER postgres_fdw +OPTIONS (dbname 'brep_package', updatable 'false'); + +GRANT USAGE ON FOREIGN SERVER package_server to brep; + +CREATE USER MAPPING FOR PUBLIC +SERVER package_server +OPTIONS (user 'brep-build', password '-'); +EOF + +# Allow brep-build user to access the brep_package database. +# +f="$(run sudo sudo -u postgres psql -t -A -c "show hba_file;")" +s="# TYPE DATABASE USER ADDRESS METHOD\nlocal brep_package brep-build md5\n\n" + +run sudo sed --in-place=.bak "1s/^/$s/" "$f" +run sudo systemctl restart postgresql + +# Enable creating database tables with columns of the case-insensitive +# character string type. +# +q="CREATE EXTENSION citext;" +run sudo sudo -u postgres psql -d brep_package <<<"$q" +run sudo sudo -u postgres psql -d brep_build <<<"$q" +run sudo sudo -u postgres psql -d brep_submit_package <<<"$q" + +# Copy the brep module configuration. +# +# Note: must be done before bin/brep-startup execution, which adjusts the +# configuration. +# +run mkdir config +run cp "$scr_dir/brep-module.conf" config/ + +# Initialize the brep private instance, in particular creating the database +# schemas and running the brep loader. +# +run mkdir bin/ +run cp "$scr_dir/brep-startup" bin/ +run bin/brep-startup + +# Smoke test the database schemas. +# +run psql -d brep_package -c 'SELECT canonical_name, summary FROM repository' >/dev/null +run psql -d brep_build -c 'SELECT package_name FROM build' >/dev/null +run psql -d brep_build -c 'SELECT DISTINCT name FROM build_package' >/dev/null +run psql -d brep_submit_package -c 'SELECT canonical_name, summary FROM repository' >/dev/null + +# Setup executing the brep-startup script on boot. +# +run sudo cp "$scr_dir/brep-startup.service" /etc/systemd/system/ + +run sudo systemctl start brep-startup.service # Make sure there are no issues. +run sudo systemctl enable brep-startup.service + +# Prepare directories for the package submission service. +# +run mkdir submit-data +run mkdir submit-temp +run setfacl -m g:www-data:rwx submit-data +run setfacl -m g:www-data:rwx submit-temp + +# Make the Apache2 user owned directories fully accessible by the 'brep' user +# (which the submit-pub submission handler will run as). +# +run setfacl -dm g:brep:rwx submit-data +run setfacl -dm g:brep:rwx submit-temp + +# Add the Apache2 user to sudoers, so the submission handler can re-execute +# itself as the 'brep' user. +# +run sudo tee -a /etc/sudoers.d/www-data >/dev/null <<EOF +www-data ALL=(ALL) NOPASSWD:ALL +EOF + +run sudo chmod 0440 /etc/sudoers.d/www-data + +# Setup the Apache2 module. +# +run sudo mkdir -p /var/www/brep/log/ + +run sudo cp "$scr_dir/brep-apache2.conf" /etc/apache2/sites-available/000-brep.conf +run sudo cp "$scr_dir/brep-logrotate" /etc/logrotate.d/brep + +run sudo a2dissite --purge -q 000-default +run sudo a2ensite -q 000-brep + +run sudo systemctl restart apache2 +run sudo systemctl status apache2 >/dev/null + +# Make sure the Apache2 service depends on PostgreSQL and +# brep-startup.service, so that they are started in proper order. +# +run sudo mkdir -p /etc/systemd/system/apache2.service.d/ +run sudo tee /etc/systemd/system/apache2.service.d/postgresql.conf >/dev/null <<EOF +[Unit] +Requires=postgresql.service +After=postgresql.service +EOF + +run sudo tee /etc/systemd/system/apache2.service.d/brep-startup.conf >/dev/null <<EOF +[Unit] +Requires=brep-startup.service +After=brep-startup.service +EOF + +run sudo mkdir -p /etc/systemd/system/postgresql.service.d/ +run sudo tee /etc/systemd/system/postgresql.service.d/apache2.conf >/dev/null <<EOF +[Unit] +Wants=apache2.service +EOF + +run sudo systemctl daemon-reload + +# Verify that Apache2 is stopped after PostgreSQL is stopped. +# +run sudo systemctl stop postgresql + +ec="0" +run sudo systemctl status apache2 >/dev/null || ec="$?" + +if [ "$ec" -ne 3 ]; then + error "exit code 3 (unit is not active) is expected instead of $ec" +fi + +# Verify that Apache2 is started after PostgreSQL is started. +# +run sudo systemctl start postgresql + +run sleep 3 +run sudo systemctl status apache2 >/dev/null + +# Setup periodic loader execution. +# +run sudo cp "$scr_dir/brep-load.service" /etc/systemd/system/ +run sudo cp "$scr_dir/brep-load.timer" /etc/systemd/system/ + +run sudo systemctl start brep-load.service # Make sure there are no issues. + +run sudo systemctl start brep-load.timer +run sudo systemctl status brep-load.timer >/dev/null +run sudo systemctl enable brep-load.timer +run sudo systemctl status brep-load.timer >/dev/null diff --git a/etc/private/install/brep-load.service b/etc/private/install/brep-load.service new file mode 100644 index 0000000..272a124 --- /dev/null +++ b/etc/private/install/brep-load.service @@ -0,0 +1,11 @@ +[Unit] +Description=brep repository loader service + +[Service] +Type=oneshot +User=brep +Group=brep +ExecStart=/home/brep/install/bin/brep-load /home/brep/config/loadtab + +[Install] +WantedBy=default.target diff --git a/etc/private/install/brep-load.timer b/etc/private/install/brep-load.timer new file mode 100644 index 0000000..1bf78c4 --- /dev/null +++ b/etc/private/install/brep-load.timer @@ -0,0 +1,33 @@ +[Unit] +Description=brep repository loader timer +RefuseManualStart=no +RefuseManualStop=no + +# Note that due to brep-startup service's oneshot type, this unit won't be +# started until the brep-startup process exits successfully. +# +# Also note that if brep-startup fails and is restarted manually, similar to +# services, the timer is not started automatically. Instead, it has to be +# started manually with `systemctl start brep-load.timer`. +# +Requires=brep-startup.service +After=brep-startup.service + +[Timer] +Unit=brep-load.service + +# Don't keep track of the timer across reboots. +# +Persistent=false + +# Start the timer for the first time. +# +OnBootSec=1 + +# Then wait 4-5 seconds until the next run. +# +OnUnitInactiveSec=4 +AccuracySec=1 + +[Install] +WantedBy=timers.target diff --git a/etc/private/install/brep-logrotate b/etc/private/install/brep-logrotate new file mode 100644 index 0000000..67c7c90 --- /dev/null +++ b/etc/private/install/brep-logrotate @@ -0,0 +1,20 @@ +/var/www/brep/log/*.log { + weekly + missingok + rotate 4 + compress + delaycompress + notifempty + create 640 root adm + sharedscripts + postrotate + if /etc/init.d/apache2 status > /dev/null ; then \ + /etc/init.d/apache2 reload > /dev/null; \ + fi; + endscript + prerotate + if [ -d /etc/logrotate.d/httpd-prerotate ]; then \ + run-parts /etc/logrotate.d/httpd-prerotate; \ + fi; \ + endscript +} diff --git a/etc/private/install/brep-module.conf b/etc/private/install/brep-module.conf new file mode 100644 index 0000000..0bff58d --- /dev/null +++ b/etc/private/install/brep-module.conf @@ -0,0 +1,387 @@ +# Configuration file for the brep module (note: this is not an apache2 .conf +# file but it can be converted to one by prefixing all the options with +# brep-). See brep(1) for detailed description of each configuration option. +# Commented out options indicate their default values. +# +# Besides being parsed by the brep module, this file may also be parsed by +# brep utilities that are normally only interested in the subset of the +# options. To simplify skipping of unrecognized, this file must always have an +# option name and its value on the same line. +# + +# Package search page title. It is placed inside XHTML5 <title> element. +# +# search-title Packages + + +# Web page logo. It is displayed in the page header aligned to the left edge. +# The value is treated as an XHTML5 fragment. +# +# logo "" + + +# Web page menu. Each entry is displayed in the page header in the order +# specified and aligned to the right edge. A link target that starts with '/' +# or contains ':' is used as is. Otherwise, it is prefixed with the repository +# web interface root. +# +menu Packages= +# menu Builds=?builds +# menu Configs=?build-configs +menu Submit=?submit +# menu CI=?ci +menu About=?about + + +# Number of packages per page. +# +# search-page-entries 20 + + +# Number of pages in navigation (pager). +# +# search-pages 5 + + +# Number of package description characters to display in brief pages. +# +# package-description 500 + + +# Number of package changes characters to display in brief pages. +# +# package-changes 5000 + + +# The package database connection configuration. By default, brep will try to +# connect to the local instance of PostgreSQL with the operating system- +# default mechanism (Unix-domain socket, etc) and use operating system +# (login) user name and the database called 'brep_package'. If the role name +# is not empty then the login user will be switched (with SET ROLE) to this +# user prior to executing any statements. If not specified, then 'brep' is +# used. See brep(1) for details. +# +# package-db-user +# package-db-role brep +# package-db-password +# package-db-name brep_package +# package-db-host +# package-db-port + + +# The maximum number of concurrent package database connections per web server +# process. If 0, then no limitation is applied. +# +# package-db-max-connections 5 + + +# The maximum number of times to retry package database transactions in the +# face of recoverable failures (deadlock, loss of connection, etc). +# +# package-db-retry 10 + + +# Build configuration file. If not specified (default), then the package +# building functionality will be disabled. If specified, then the build +# database must be configured (see next). Note: must be an absolute path. +# +# build-config + + +# Number of build configurations per page. +# +# build-config-page-entries 20 + + +# Number of pages in navigation (pager). +# +# build-config-pages 5 + + +# Directory containing build bot agent public keys. If specified, then brep +# will perform agent authentication and will reject build results from +# unauthenticated ones. If not specified, then build results are accepted from +# all agents (which will be a security risk if the brep instance is publicly +# accessible). +# +# The directory is expected to contain one PEM-encoded public key per file with +# the .pem extension. All other files and subdirectories are ignored. The brep +# instance needs to be restarted after adding new key files for the changes to +# take effect. +# +# build-bot-agent-keys + + +# Number of builds per page. +# +# build-page-entries 20 + + +# Number of pages in navigation (pager). +# +# build-pages 5 + + +# Time to wait before considering a package for a forced rebuild. Must be +# specified in seconds. Default is 10 minutes. +# +# build-forced-rebuild-timeout 600 + + +# Time to wait before considering a package for a normal rebuild. Must be +# specified in seconds. Default is 24 hours. +# +# build-normal-rebuild-timeout 86400 + + +# Alternative package rebuild timeout to use instead of the normal rebuild +# timeout (see the build-normal-rebuild-timeout option for details) during +# the specified time interval. Must be specified in seconds. Default is the +# time interval length. +# +# The alternative rebuild timeout can be used to "pull" the rebuild window to +# the specified time of day, for example, to optimize load and/or power +# consumption of the build infrastructure (off-work hours, solar, off-peak +# electricity tariffs, etc). A shorter than the time interval rebuild timeout +# can also be used to force continuous rebuilds, for example, to shake out +# flaky tests. Note also that if the alternative rebuild timeout is greater +# than the normal rebuild timeout, then this will result in slower rebuilds +# during the alternative time interval. In this case, if the build +# infrastructure is monitored for delayed package builds, then the alternative +# rebuild timeout should only be made slightly greater than the normal timeout +# (see brep-monitor(1) for details). +# +# The time interval boundaries must be specified as times of day (in the local +# timezone) in the <hours>:<minutes> form. If the stop time is less than the +# start time then the interval extends through midnight. The start and stop +# times must both be either specified or absent. If unspecified, then no +# alternative rebuild timeout will be used. +# +# build-alt-rebuild-timeout +# build-alt-rebuild-start +# build-alt-rebuild-stop + + +# The maximum size of the build task request manifest accepted. Note that the +# HTTP POST request body is cached to retry database transactions in the face +# of recoverable failures (deadlock, loss of connection, etc). Default is +# 100K. +# +# build-task-request-max-size 102400 + + +# Time to wait before considering the expected task result lost. Must be +# specified in seconds. Default is 3 hours. +# +# build-result-timeout 10800 + + +# The maximum size of the build result manifest accepted. Note that the HTTP +# POST request body is cached to retry database transactions in the face of +# recoverable failures (deadlock, loss of connection, etc). Default is 10M. +# +# build-result-request-max-size 10485760 + + +# The build database connection configuration. By default, brep will try to +# connect to the local instance of PostgreSQL with the operating system-default +# mechanism (Unix-domain socket, etc) and use operating system (login) user +# name and the database called 'brep_build'. If the role name is not empty +# then the login user will be switched (with SET ROLE) to this user prior +# to executing any statements. If not specified, then 'brep' is used. See +# brep(1) for details. +# +# build-db-user +# build-db-role brep +# build-db-password +# build-db-name brep_build +# build-db-host +# build-db-port + + +# The maximum number of concurrent build database connections per web server +# process. If 0, then no limitation is applied. +# +# build-db-max-connections 5 + + +# The maximum number of times to retry build database transactions in the +# face of recoverable failures (deadlock, loss of connection, etc). +# +# build-db-retry 10 + + +# The openssl program to be used for crypto operations. You can also specify +# additional options that should be passed to the openssl program with +# openssl-option. If the openssl program is not explicitly specified, then brep +# will use openssl by default. +# +# openssl openssl + + +# Additional option to be passed to the openssl program (see openssl for +# details). Repeat this option to specify multiple openssl options. +# +# openssl-option + + +# Environment variable to be set (<name>=<value>) or unset (just <name>) for +# the openssl program (see openssl for details). Repeat this option to specify +# multiple openssl variables. Note that unspecified variables are inherited +# from the web server process. +# +# You need to at least set the RANDFILE environment variable to change the +# default location of the openssl program seed file and maybe also the +# OPENSSL_CONF variable if you would like to use a custom openssl configuration +# file. +# +# openssl-envvar RANDFILE=/home/brep/www-data-openssl.rnd +# openssl-envvar OPENSSL_CONF=/home/brep/www-data-openssl.cnf +# +# To create www-data-openssl.rnd with suitable permissions, run (as user brep): +# +# $ touch www-data-openssl.rnd +# $ setfacl -b -m g:www-data:rw www-data-openssl.rnd +# + + +# The directory to save final submission data to. If unspecified, the package +# submission functionality will be disabled. If specified, then submit-temp +# must be specified as well. +# +# Note that the directory path must be absolute and the directory itself must +# exist and have read, write, and execute permissions granted to the user that +# runs the web server. +# +submit-data /home/brep/submit-data + + +# The directory to save temporary submission data to. Must be specified if the +# package submission functionality is enabled. +# +# Note that this directory must be on the same filesystem and satisfy the same +# requirements as submit-data. It is also the user's responsibility to clean +# it up after an unclean web server shutdown. +# +submit-temp /home/brep/submit-temp + + +# The maximum size of the submission data accepted. Note that currently the +# entire submission request is read into memory. Default is 10M. +# +# 100M. +# +submit-max-size 104857600 + + +# The package submission form fragment. If specified, then its contents are +# treated as an XHTML5 fragment that is inserted into the <body> element of +# the submission page. If unspecified, then no submission page will be +# displayed. Note that the file path must be absolute. +# +submit-form /home/brep/install/share/brep/www/submit.xhtml + + +# The package submission email. If specified, the submission request and +# result manifests will be sent to this address. +# +# submit-email + + +# The handler program to be executed on package submission. The handler is +# executed as part of the submission request and is passed additional +# arguments that can be specified with submit-handler-argument followed by +# the absolute path to the submission directory. Note that the program path +# must be absolute. +# +submit-handler /home/brep/install/bin/brep-submit-pub + + +# Additional arguments to be passed to the submission handler program (see +# submit-handler for details). Repeat this option to specify multiple +# arguments. +# +submit-handler-argument --user +submit-handler-argument brep +submit-handler-argument --result-url +submit-handler-argument http://unknown +submit-handler-argument /home/brep/install/bin/brep-load +submit-handler-argument --db-name=brep_submit_package +submit-handler-argument /var/brep/bpkg/pkg + + +# The handler program timeout in seconds. If specified and the handler does +# not exit in the alloted time, then it is killed and its termination is +# treated as abnormal. +# +submit-handler-timeout 120 + + +# The directory to save CI request data to. If unspecified, the package CI +# functionality will be disabled. +# +# Note that the directory path must be absolute and the directory itself must +# exist and have read, write, and execute permissions granted to the user that +# runs the web server. +# +# ci-data + + +# The package CI form fragment. If specified, then its contents are treated as +# an XHTML5 fragment that is inserted into the <body> element of the CI page. +# If unspecified, then no CI page will be displayed. Note that the file path +# must be absolute. +# +# ci-form + + +# The package CI email. If specified, the CI request and result manifests will +# be sent to this address. +# +# ci-email + + +# The handler program to be executed on CI request. The handler is executed as +# part of the HTTP request and is passed additional arguments that can be +# specified with ci-handler-argument followed by the absolute path to the CI +# request directory. Note that the program path must be absolute. +# +# ci-handler + + +# Additional arguments to be passed to the CI handler program (see ci-handler +# for details). Repeat this option to specify multiple arguments. +# +# ci-handler-argument + + +# The CI handler program timeout in seconds. If specified and the handler does +# not exit in the allotted time, then it is killed and its termination is +# treated as abnormal. +# +# ci-handler-timeout + + +# The default view to display for the global repository root. The value is one +# of the supported services (packages, builds, submit, ci, etc). Default is +# packages. +# +# root-global-view packages + + +# The default view to display for the tenant repository root. The value is one +# of the supported services (packages, builds, submit, ci, etc). Default is +# packages. +# +# root-tenant-view packages + + +# Name to call the tenant values on web pages. If not specified, then 'tenant' +# is used. +# +# tenant-name tenant + + +# Trace verbosity. Disabled by default. +# +# verbosity 0 diff --git a/etc/private/install/brep-startup b/etc/private/install/brep-startup new file mode 100755 index 0000000..80cb255 --- /dev/null +++ b/etc/private/install/brep-startup @@ -0,0 +1,67 @@ +#! /usr/bin/env bash + +# file : etc/private/install/brep-startup +# license : MIT; see accompanying LICENSE file + +# (Re-)initialize the brep private instance, normally on the machine startup. +# +# Specifically: +# +# - Create the pkg repository and symlink to it, unless already exists. +# +# - Migrate the brep databases as a sanity check. +# +# - Adjust the brep module configuration file using the current host name/IP. +# +# - Generate the loadtab using the current host name/IP and run the loader. +# +trap "{ exit 1; }" ERR + +# Create the pkg repository, if required. +# +d=/var/brep/bpkg + +if [ ! -L "$d/pkg" ]; then + rd="$(date "+pkg-%Y%m%d-%H%M%S-%N")" + + mkdir -p "$d/$rd/1" + ln -s "$rd" "$d/pkg" +fi + +r="$d/pkg/1" + +if [ ! -f "$r/repositories.manifest" ]; then + echo ": 1" >"$r/repositories.manifest" +fi + +if [ ! -f "$r/packages.manifest" ]; then + bpkg rep-create -q "$r" +fi + +# Migrate the databases. +# +"$HOME/install/bin/brep-migrate" package +"$HOME/install/bin/brep-migrate" build +"$HOME/install/bin/brep-migrate" -n brep_submit_package package + +# Deduce the machine host name. +# +h="$(hostname -f)" +if [ "$h" == "localhost" ]; then + h="$(hostname -I | sed 's/ *$//')" # Strip the potential trailing space(s). +fi + +# Adjust the submission result URL host name in the brep module configuration +# file. +# +sed --in-place -re \ +"\$!N;s%^\s*(submit-handler-argument\s+--result-url\s*\\n)\ +\s*(submit-handler-argument\s+https?://)[^/]+(.*)\$%\1\2$h\3%;P;D" \ +"$HOME/config/brep-module.conf" + +# (Re-)generate the loadtab file and reload the repository. +# +f="$HOME/config/loadtab" + +echo "http://$h/1 private cache:$r" >"$f" +"$HOME/install/bin/brep-load" "$f" diff --git a/etc/private/install/brep-startup.service b/etc/private/install/brep-startup.service new file mode 100644 index 0000000..a3dc546 --- /dev/null +++ b/etc/private/install/brep-startup.service @@ -0,0 +1,17 @@ +[Unit] +Description=brep instance initialization service + +Wants=network-online.target +After=network-online.target + +Requires=postgresql.service +After=postgresql.service + +[Service] +Type=oneshot +User=brep +Group=brep +ExecStart=/home/brep/bin/brep-startup + +[Install] +WantedBy=default.target diff --git a/etc/private/install/vm-gen-service b/etc/private/install/vm-gen-service new file mode 100755 index 0000000..c4938c2 --- /dev/null +++ b/etc/private/install/vm-gen-service @@ -0,0 +1,204 @@ +#! /usr/bin/env bash + +# Generate systemd .service file for QEMU/KVM virtual machines. +# +# Normally the machines are run from a dedicated user account with its home +# directory containing all the relevant files (management scripts, images, +# configurations, and sockets). However, this can be overriden with the +# following options +# +# --home <dir> +# The virtual machines "home" directory. If unspecified, the user's home +# directory is assumed. +# +# --bin <dir> +# The virtual machines management scripts directory. If unspecified, +# <home>/bin is assumed. If specified as relative, then assumed relative +# to <home>. +# +# --etc <dir> +# The virtual machines configuration files directory. If unspecified, +# <home>/etc is assumed. If specified as relative, then assumed relative +# to <home>. +# +# --var <dir> +# The virtual machines image files directory. If unspecified, <home>/var is +# assumed. If specified as relative, then assumed relative to <home>. +# +# --run <dir> +# The virtual machines sockets directory. If unspecified, <home>/run is +# assumed. If specified as relative, then assumed relative to <home>. +# +# If <user> is unspecified, the current user is assumed. If <group> is +# unspecified, the user's primary group is assumed. +# +usage="usage: $0 [<options>] [<user>] [<group>]" + +owd="$(pwd)" +trap "{ cd '$owd'; exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +home= +bin= +etc= +var= +run= + +while [ "$#" -gt 0 ]; do + case "$1" in + --home) + shift + home="$1" + shift + ;; + --bin) + shift + bin="$1" + shift + ;; + --etc) + shift + etc="$1" + shift + ;; + --var) + shift + var="$1" + shift + ;; + --run) + shift + run="$1" + shift + ;; + *) + break + ;; + esac +done + +user="$1" + +if [ -z "$user" ]; then + user="$(id -un)" +fi + +group="$2" + +if [ -z "$group" ]; then + group="$(id -un "$user")" +fi + +if [ -z "$home" ]; then + home="$(eval echo ~$user)" +fi + +function complete_dir () # <default> <home> <dir> +{ + local r + if [ -z "$3" ]; then + r="$2/$1" + elif [ "${3:0:1}" != "/" ]; then + r="$2/$3" + else + r="$3" + fi + echo "$(realpath --no-symlinks --canonicalize-missing "$r")" +} + +bin="$(complete_dir bin "$home" "$bin")" +etc="$(complete_dir etc "$home" "$etc")" +var="$(complete_dir var "$home" "$var")" +run="$(complete_dir run "$home" "$run")" + +name="vm-$user" +file="$name@.service" + +# Thinks that must be \-escaped: +# +# - $ (including in comments) +# - \ (e.g., in line continuations) +# +cat <<EOF >"$file" +# $file -- QEMU/KVM machine service template for systemd +# +# user: $user +# group: $group +# bin: $bin +# etc: $etc +# var: $var +# run: $run +# +# To install: +# +# sudo cp $file /etc/systemd/system/ +# sudo chmod 644 /etc/systemd/system/$file +# +# cp ... $var/<machine>.img +# nano $etc/<machine>.conf # Specify RAM, CPU, TAP, MAC, etc. +# +# sudo systemctl start $name@<machine> +# sudo systemctl status $name@<machine> +# login-machine $run/<machine>-con.sock +# sudo systemctl stop $name@<machine> +# +# sudo systemctl enable $name@<machine> + +[Unit] +Description=QEMU/KVM virtual machine %I + +Wants=network-online.target +#After=network-online.target +After=multi-user.target + +[Service] +User=$user +Group=$user +UMask=0007 +WorkingDirectory=~ + +Environment=CPU=1 +Environment=RAM=2G + +# These MUST be specific in EnvironmentFile! +# +#Environment=TAP= +#Environment=MAC= + +# Note that using variable expansion in EnvironmentFile does not work (at +# least not with systemd 229). +# +EnvironmentFile=$etc/%i.conf + +# Note that the first word of ExecStart cannot contain variable expansions. +# +ExecStart=$bin/vm-start \\ + --cpu \${CPU} \\ + --ram \${RAM} \\ + --tap \${TAP} \\ + --mac \${MAC} \\ + --monitor $run/%i-mon.sock \\ + --console $run/%i-con.sock \\ + $var/%i.img + +ExecStop=$bin/vm-stop $run/%i-mon.sock + +# Make systemd wait for ExecStop completion. +# +KillMode=none +TimeoutStopSec=60 + +[Install] +WantedBy=multi-user.target +EOF + +info "generated $file for" +info " user: $user" +info " group: $group" +info " bin: $bin" +info " etc: $etc" +info " var: $var" +info " run: $run" diff --git a/etc/private/vm-login b/etc/private/vm-login new file mode 100755 index 0000000..3b501ca --- /dev/null +++ b/etc/private/vm-login @@ -0,0 +1,30 @@ +#! /usr/bin/env bash + +# Get virtual machine console. +# +usage="usage: $0 <console-socket>" + +owd="$(pwd)" +trap "{ cd '$owd'; exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +con="$1" + +if [ -z "$con" ]; then + error "missing console socket" +fi + +pty="$(dirname "$con")/$(basename -s .sock "$con").pty" + +socat "UNIX-CONNECT:$con" "PTY,link=$pty" & +pid="$!" + +screen "$pty" + +# Note: socat may have already terminated (e.g., VM was shut down). +# +kill "$pid" 2>/dev/null || true +wait diff --git a/etc/private/vm-start b/etc/private/vm-start new file mode 100755 index 0000000..41c4247 --- /dev/null +++ b/etc/private/vm-start @@ -0,0 +1,98 @@ +#! /usr/bin/env bash + +# file : etc/private/vm-start +# license : MIT; see accompanying LICENSE file + +# Start the brep virtual machine (VM) for installing or running the previously +# installed brep private instance. Must be executed on the host as brep user. +# +# Share with the VM the brep state directory as the 9p filesystem with the +# passthrough security model enabled. This directory is expected to be owned +# by the brep user and either contain the pkg repository maintained by the +# brep instance or be empty, in which case the empty repository will be +# automatically initialized. +# +# Note that you can signal to the VM to regenerate the repository on startup +# (e.g., after package removal) by removing the packages.manifest file from +# the repository. +# +# Options: +# +# --state <dir> +# +# State directory to share with the VM. If unspecified, $HOME/state is +# assumed. +# +# --install <dir> +# +# Also share with the VM the install directory (contains the brep private +# instance installation script and auxiliary files). +# +# Note that this script wraps the generic vm-start-base script and passes +# through any arguments that follows these options to that script. +# +usage="usage: $0 [<options>] [<base-options>] <machine-img> [<extra-qemu-options>]" + +trap "{ exit 1; }" ERR +set -o errtrace # Trap ERR in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +install= +state="$HOME/state" + +while [ "$#" -gt 0 ]; do + case "$1" in + --install) + shift + install="${1%/}" + shift + ;; + --state) + shift + state="${1%/}" + shift + ;; + *) + break # The end of options is encountered. + ;; + esac +done + +if [ "$#" -eq 0 ]; then + error "missing machine image" +fi + +# Verify the state directory existence. +# +if [ ! -d "$state" ]; then + error "state directory '$state' does not exist or is not a directory" +fi + +# Compute the start and QEMU options. +# +start_ops=() +qemu_ops=(\ + -fsdev "local,id=state,path=$state,security_model=passthrough" \ + -device "virtio-9p-pci,fsdev=state,mount_tag=state") + +if [ -n "$install" ]; then + + # Verify the toolchain install script existence in the install directory. + # + if [ ! -f "$(echo "$install"/build2-install-*.sh)" ]; then + error "missing toolchain installation script in '$install' directory" + fi + + start_ops+=(--stdio) + qemu_ops+=(\ + -fsdev "local,id=install,path=$install,security_model=passthrough" \ + -device "virtio-9p-pci,fsdev=install,mount_tag=install") +fi + +# Finally, forward execution to the base script. +# +scr_dir="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" + +exec "$scr_dir/vm-start-base" "${start_ops[@]}" "$@" "${qemu_ops[@]}" diff --git a/etc/private/vm-start-base b/etc/private/vm-start-base new file mode 100755 index 0000000..df59d6a --- /dev/null +++ b/etc/private/vm-start-base @@ -0,0 +1,187 @@ +#! /usr/bin/env bash + +# Start a QEMU/KVM virtual machine. +# +# --cpu <num> +# CPU hardware threads to allocate to the VM, 1 by default. +# +# --ram <num> +# RAM to allocate to the VM, 2G by default (can be specified with G, +# M suffixes). +# +# --tap <tap> +# Existing tap interface to use instead of creating a new one. +# +# --mac <addr> +# MAC address to use for the machine. +# +# --monitor <path> +# Monitor UNIX socket path, /tmp/machine-<tap>-mon.sock if unspecified. +# +# --console <path> +# Console UNIX socket path, /tmp/machine-<tap>-con.sock if unspecified. +# +# --stdio +# Connect both console and monitor to stdio (multiplexed). +# +# --stdio-monior +# Connect only monitor to stdio. +# +usage="usage: $0 [<options>] <machine-img> [<extra-qemu-options>]" + +owd="$(pwd)" +trap "{ cd '$owd'; exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +qemu=(qemu-system-x86_64 -enable-kvm) + +# The bridge is only used if we are cretaing the tap. +# +br=br0 + +cpu=1 +ram=2G +tap= +mac="de:ad:be:ef:b8:da" +mon= +con= +stdio= +stdio_monitor= + +while [ "$#" -gt 0 ]; do + case "$1" in + --cpu) + shift + cpu="$1" + shift + ;; + --ram) + shift + ram="$1" + shift + ;; + --tap) + shift + tap="$1" + shift + ;; + --mac) + shift + mac="$1" + shift + ;; + --monitor) + shift + mon="$1" + shift + ;; + --console) + shift + con="$1" + shift + ;; + --stdio) + stdio=true + stdio_monitor= + shift + ;; + --stdio-monitor) + stdio= + stdio_monitor=true + shift + ;; + *) + break + ;; + esac +done + +img="$1" +shift + +if [ -z "$img" ]; then + error "missing machine image" +fi + +if [ ! -f "$img" ]; then + error "machine image '$img' does not exist" +fi + +# Open the reading file descriptor and lock the machine image. Fail if unable +# to lock. +# +# Note that the file descriptor is automatically closed on the script exit and +# the lock is released. +# +exec {lfd}<"$img" + +if ! flock -n "$lfd"; then + error "machine is already running" +fi + +del_tap= +if [ -z "$tap" ]; then + tap=tap9 + sudo ip tuntap delete "$tap" mode tap || true + sudo ip tuntap add "$tap" mode tap user "$(whoami)" + sudo ip link set "$tap" up + #sleep 0.5s + sudo ip link set "$tap" master "$br" + del_tap=true +fi + +if [ -z "$mon" ]; then + mon="/tmp/machine-$tap-mon.sock" +fi + +if [ -z "$con" ]; then + con="/tmp/machine-$tap-con.sock" +fi + +ops=(\ + -m "$ram" \ + -cpu host -smp "$cpu,sockets=1,cores=$cpu,threads=1" \ + \ + -netdev "tap,id=net0,ifname=$tap,script=no" \ + -device "virtio-net-pci,netdev=net0,mac=$mac" \ + \ + -drive "if=none,id=disk0,file=$img,format=raw" \ + -device "virtio-blk-pci,scsi=off,drive=disk0" \ + \ + -nographic \ +) + +# Console/monitor options. +# +if [ "$stdio" ]; then + # Multiplex the monitor and serial console onto stdio. In particular, this + # makes sure Ctrl-c is passed to the guest (rather than termination the QEMU + # process). To switch between monitor and console, Ctrl-a,c (to terminate + # QEMU, type quit in the monitor). + # + ops+=(-serial mon:stdio) +else + # Monitor. + # + if [ "$stdio_monitor" ]; then + ops+=(-chardev stdio,id=mon) + else + ops+=(-chardev "socket,id=mon,path=$mon,server,nowait") + fi + + ops+=(-mon chardev=mon,mode=readline) + + # Console. + # + ops+=(-chardev "socket,id=con,path=$con,server,nowait" \ + -serial chardev:con) +fi + +"${qemu[@]}" "${ops[@]}" -boot c "$@" + +if [ "$del_tap" ]; then + sudo ip tuntap delete "$tap" mode tap +fi diff --git a/etc/private/vm-stop b/etc/private/vm-stop new file mode 100755 index 0000000..d3b8330 --- /dev/null +++ b/etc/private/vm-stop @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +# Stop virtual machine started with vm-start. +# +usage="usage: $0 <monitor-socket>" + +owd="$(pwd)" +trap "{ cd '$owd'; exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +mon="$1" + +if [ -z "$mon" ]; then + error "missing monitor socket" +fi + +echo system_powerdown | socat - "UNIX-CONNECT:$mon" >/dev/null + +# Wait for QEMU to close the socket. This is racy so ignore errors. +# +socat "UNIX-CONNECT:$mon" - >/dev/null 2>&1 || true diff --git a/mod/external-handler.cxx b/mod/external-handler.cxx index 4c1a86f..7f26680 100644 --- a/mod/external-handler.cxx +++ b/mod/external-handler.cxx @@ -96,14 +96,14 @@ namespace brep pipe.out.close (); auto kill = [&pr, &warn, &handler, &ref] () - { - // We may still end up well (see below), thus this is a warning. - // - warn << "ref " << ref << ": process " << handler - << " execution timeout expired"; + { + // We may still end up well (see below), thus this is a warning. + // + warn << "ref " << ref << ": process " << handler + << " execution timeout expired"; - pr.kill (); - }; + pr.kill (); + }; try { @@ -126,7 +126,7 @@ namespace brep } timeval tm {wd.count () / 1000 /* seconds */, - wd.count () % 1000 * 1000 /* microseconds */}; + wd.count () % 1000 * 1000 /* microseconds */}; fd_set rd; FD_ZERO (&rd); diff --git a/tests/submit/buildfile b/tests/submit/buildfile index 1845d23..a90115a 100644 --- a/tests/submit/buildfile +++ b/tests/submit/buildfile @@ -7,9 +7,12 @@ include $dir commons = data -./: testscript{* -{$commons}} common_testscript{$commons} {*/ -test/}{**} \ - $dir/exe{brep-submit-dir} $dir/exe{brep-submit-git} file{hello.tar.gz} \ - doc{README} +./: testscript{* -{$commons}} common_testscript{$commons} {*/ -test/}{**} \ + $dir/exe{brep-submit-dir} \ + $dir/exe{brep-submit-git} \ + $dir/exe{brep-submit-pub} \ + file{hello.tar.gz} doc{README} testscript{submit-dir}@./: test = $out_base/$dir/brep-submit-dir testscript{submit-git}@./: test = $out_base/$dir/brep-submit-git +testscript{submit-pub}@./: test = $out_base/$dir/brep-submit-pub diff --git a/tests/submit/submit-pub.testscript b/tests/submit/submit-pub.testscript new file mode 100644 index 0000000..b73d108 --- /dev/null +++ b/tests/submit/submit-pub.testscript @@ -0,0 +1,175 @@ +# file : tests/submit/submit-pub.testscript +# license : MIT; see accompanying LICENSE file + +.include data.testscript + +user=$getenv("USER") + +# Disable tests if the password-less sudo is not enabled for the user. +# ++if! sudo -u "$user" --non-interactive echo 'test' >'test' + exit +end + +root_rep=$~/pkg-1 + ++mkdir -p $root_rep/1 ++echo ": 1" >=$root_rep/1/repositories.manifest ++bpkg rep-create $root_rep/1 2>! &$root_rep/1/packages.manifest + +clone_root_rep = cp --no-cleanup -r $root_rep ./ &pkg-1/*** &?pkg.lock + +: args +{ + : none + : + $* 2>>~%EOE% != 0 + %\[.+\] \[brep:error\] \[ref \] \[brep-submit-pub\]: usage: .+brep-submit-pub \[<options>\] \[<loader-path> <loader-options>\] <repo> <dir>% + EOE + + : few + : + $* 2>>~%EOE% != 0 + %\[.+\] \[brep:error\] \[ref \] \[brep-submit-pub\]: usage: .+brep-submit-pub \[<options>\] \[<loader-path> <loader-options>\] <repo> <dir>% + EOE + + : repo-not-exist + : + $* $~/repo $~/dir 2>>~%EOE% != 0 + %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-pub\]: '.+repo' does not exist or is not a symlink% + EOE + + : dir-not-exist + : + { + $clone_root_rep; + ln -s pkg-1 pkg; + $* $~/pkg $~/dir 2>>~%EOE% != 0 + %\[.+\] \[brep:error\] \[ref dir\] \[brep-submit-pub\]: '.+dir' does not exist or is not a directory% + EOE + } +} + +: success +: +{ + test.options += --user "$user" + + : simulate + : + { + $clone_root_data; + echo 'simulate: success' >+$data_dir/request.manifest; + + $clone_root_rep; + ln -s pkg-1 pkg; + + $* $~/pkg $~/$data_dir >>"EOO"; + : 1 + status: 200 + message: package is published: libhello/0.1.0 + reference: $checksum + EOO + + test -d $data_dir != 0 + } + + : for-real + : + : Here we create the (fake) package revision which is expected to be removed + : by the handler. + : + { + $clone_root_data; + $clone_root_rep; + ln -s pkg-1 pkg; + + mkdir --no-cleanup pkg-1/1/prj; + touch --no-cleanup pkg-1/1/prj/libhello-0.1.0+1.tar.gz; + + $* $~/pkg $~/$data_dir &!pkg-1/*** &pkg-*/*** >>"EOO"; + : 1 + status: 200 + message: package is published: libhello/0.1.0 + reference: $checksum + EOO + + # While at it, test the duplicate submission. + # + $clone_root_data_clean; + + $* $~/pkg $~/$data_dir >>"EOO" + : 1 + status: 422 + message: duplicate submission + reference: $checksum + EOO + } + + : result-url + : + { + $clone_root_data; + $clone_root_rep; + ln -s pkg-1 pkg; + + test.options += --result-url 'https://example.com/'; + + $* $~/pkg $~/$data_dir &!pkg-1/*** &pkg-*/*** >>"EOO" + : 1 + status: 200 + message: package is published: https://example.com/libhello/0.1.0 + reference: $checksum + EOO + } +} + +: failure +: +{ + test.options += --user "$user" + + : invalid-package + : + : Here we remove repositories.manifest to make sure that the bpkg-rep-create + : failure ends up with a proper response. + : + { + $clone_root_data_clean; + $clone_root_rep; + ln -s pkg-1 pkg; + + rm pkg/1/repositories.manifest; + + $* $~/brep-loader $~/pkg $~/$data_dir >>~"%EOO%" + : 1 + status: 400 + message: \\ + submitted archive is not a valid package + %.+ + \\ + reference: $checksum + EOO + } + + : not-loadable + : + : Here we specify a non-existing brep loader program to make sure that the + : brep database load failure ends up with a proper response. + : + { + $clone_root_data_clean; + $clone_root_rep; + ln -s pkg-1 pkg; + + $* $~/brep-loader $~/pkg $~/$data_dir >>~"%EOO%" + : 1 + status: 400 + message: \\ + unable to add package to repository + %.+ + \\ + reference: $checksum + EOO + } +} |