diff options
Diffstat (limited to 'etc/private/install')
-rw-r--r-- | etc/private/install/README | 75 | ||||
-rw-r--r-- | etc/private/install/brep-apache2.conf | 99 | ||||
-rwxr-xr-x | etc/private/install/brep-install | 479 | ||||
-rw-r--r-- | etc/private/install/brep-load.service | 11 | ||||
-rw-r--r-- | etc/private/install/brep-load.timer | 33 | ||||
-rw-r--r-- | etc/private/install/brep-logrotate | 20 | ||||
-rw-r--r-- | etc/private/install/brep-module.conf | 532 | ||||
-rwxr-xr-x | etc/private/install/brep-startup | 88 | ||||
-rw-r--r-- | etc/private/install/brep-startup.service | 17 | ||||
-rwxr-xr-x | etc/private/install/vm-gen-service | 207 |
10 files changed, 1561 insertions, 0 deletions
diff --git a/etc/private/install/README b/etc/private/install/README new file mode 100644 index 0000000..ef3ae70 --- /dev/null +++ b/etc/private/install/README @@ -0,0 +1,75 @@ +This directory contains scripts, configuration files, etc., that are used to +prepare the virtual machine (VM) with a build2 repository web interface (brep) +described in ../README. This document provides terse notes on how this VM is +prepared. + +The VM is prepared and tested using user brep: + +# groupadd --gid 63700 brep +# useradd --uid 63700 --gid 63700 --create-home brep +# usermod --lock brep +# usermod -G kvm brep + +Note: different UID/GID can be used by passing the --brep-user option to the +brep-install script. + +# su - brep +$ mkdir -p bin vm state/bpkg + +Next copy the prepared VM image: + +$ cp .../linux_debian_10.img vm/brep.img + +The brep-install script assumes a Debian-based VM distribution. Other +distributions can probably be made to work but will require changes to +brep-install. The VM normally already has the following changes applied: + + # apt-get install acpid # For QEMU system_powerdown to work. + # systemctl enable acpid + + # systemctl enable serial-getty@ttyS0.service + # systemctl start serial-getty@ttyS0.service + + # nano /etc/default/grub + # # GRUB_CMDLINE_LINUX_DEFAULT="console=ttyS0" + # # GRUB_TERMINAL="serial console" + # update-grub + + # echo localhost >/etc/hostname + # nano /etc/dhcp/dhclient.conf + # # Comment out `send host-name ...`. + # # Add `send dhcp-client-identifier = hardware;` + +Clone or copy the brep repository and create the directory structure: + +$ cp -r .../brep ./ +$ cp brep/etc/private/vm-* bin/ +$ cp -r brep/etc/private ./etc + +Download the build2 toolchain installation script: + +$ curl -sSfO https://download.build2.org/X.Y.Z/build2-install-X.Y.Z.sh +$ # Verify the checksum. +$ mv build2-install-*.sh etc/install/ + +Start the VM (give as much CPU/RAM as available to speed up compilation): + +$ ~/bin/vm-start --install etc/install/ --cpu 8 --ram 8G \ + --tap tap0 --mac de:ad:be:ef:de:ad vm/brep.img + +Login into the VM as root, then perform the following steps: + + # mount -t 9p -o trans=virtio,version=9p2000.L install /mnt + # /mnt/brep-install --mount + +After the installation is complete, test the result as described in ../README. + +Note: to create a "clean" VM for distribution, pass the --clean option to +brep-install, shut the VM down immediately after installation, save the clean +VM image, then boot a copy for testing. + +Generate the systemd service template file: + +~/etc/install/vm-gen-service --bin bin --etc vm --var vm --run . + +Test starting the VM as a systemd service as described in ../README. diff --git a/etc/private/install/brep-apache2.conf b/etc/private/install/brep-apache2.conf new file mode 100644 index 0000000..99186d1 --- /dev/null +++ b/etc/private/install/brep-apache2.conf @@ -0,0 +1,99 @@ +# Keep in the main server configuration context. This way the directive will +# be in effect during module initialization and request handling. +# +# Note that initialization log messages are written to the main server log +# file (/var/log/apache2/error.log), and request handling messages to the +# virtual server log file (/var/www/brep/log/error.log). +# +LogLevel brep:info + +<VirtualHost *:80> + #ServerName <brep-hostname> + #ServerAdmin <brep-admin-email> + + #DocumentRoot /var/www/brep/public + #Options +Indexes + + AddOutputFilterByType DEFLATE application/xhtml+xml + AddOutputFilterByType DEFLATE text/manifest + AddOutputFilterByType DEFLATE text/plain + AddOutputFilterByType DEFLATE text/css + + Alias "/1" "/var/brep/bpkg/pkg/1" + + ErrorLog /var/www/brep/log/error.log + CustomLog /var/www/brep/log/access.log combined + + # brep configuration + # + + # Load the brep module. + # + <IfModule !brep_module> + LoadModule brep_module /home/brep/install/libexec/brep/mod_brep.so + </IfModule> + + # Repository email. This email is used for the From: header in emails send + # by brep (for example, build failure notifications). + # + #brep-email <brep-admin-email> + + # Repository host. It specifies the scheme and the host address (but not the + # root path; see brep-root below) that will be used whenever brep needs to + # construct an absolute URL to one of its locations (for example, a link to + # a build log that is being send via email). + # + #brep-host http://<brep-hostname> + + # Repository root. This is the part of the URL between the host name and the + # start of the repository. For example, root value /pkg means the repository + # URL is http://example.org/pkg/. Specify / to use the web server root + # (e.g., http://example.org/). If using a different repository root, don't + # forget to also change Location and Alias directives below. + # + brep-root / + + <Location "/"> + SetHandler brep + + <IfModule dir_module> + DirectoryIndex disabled + DirectorySlash Off + </IfModule> + </Location> + + # Brep module configuration. If you prefer, you can paste the contents of + # this file here. However, you will need to prefix every option with + # 'brep-'. + # + brep-conf /home/brep/config/brep-module.conf + + # Static brep content (CSS files). + # + <IfModule !alias_module> + Error "mod_alias is not enabled" + </IfModule> + + # Note: trailing slashes are important! + # + Alias /@/ /home/brep/install/share/brep/www/ + + <Directory "/home/brep/install/share/brep/www"> + Require all granted + </Directory> + + # brep config override (must come after). + # + <LocationMatch "^/([0-9]|icons)(/.*)?$"> + SetHandler none + + DirectoryIndex enabled + DirectorySlash On + </LocationMatch> +</VirtualHost> + +<Directory /var/brep/bpkg/pkg/> + Options Indexes FollowSymLinks + AllowOverride None + Require all granted +</Directory> diff --git a/etc/private/install/brep-install b/etc/private/install/brep-install new file mode 100755 index 0000000..37179c2 --- /dev/null +++ b/etc/private/install/brep-install @@ -0,0 +1,479 @@ +#! /usr/bin/env bash + +# file : etc/private/install/brep-install +# license : MIT; see accompanying LICENSE file + +# Setup HTTP-only brep instance with unsigned package submission support via +# direct repository publishing (brep/handler/submit/submit-pub). +# +# NOTE: this setup should only be used in private/trusted environments. +# +# Unless the --setup option is specified, create the 'brep' group and user and +# re-run itself with the --setup option under this user. In the setup mode +# install and configure the brep instance, automating the instructions from +# the INSTALL file, including: +# +# - Build the build2 toolchain (installing it to /usr/local/) and brep +# (installing it to ~brep/install/). +# +# - Install PostgreSQL and create brep users/databases. +# +# - Installing Apache2 and configure HTTP server with the brep module. +# +# Note that the script is written for use on Debian-based distributions so you +# will need to adjust it to match other distributions or operating systems. +# +# Options: +# +# --mount +# +# Mount the virtio-9p device with id 'state' as the /var/brep directory. +# This directory is expected to either contain the pkg repository or be +# empty, in which case an empty repository will be automatically +# initialized. If this option is unspecified, the directory will be created +# in the local filesystem. +# +# --brep-user +# +# User and group ids to use when creating the 'brep' group and user. If +# unspecified, 63700 is used. +# +# --setup +# +# Install and configure the brep instance, assuming that the 'brep' user +# already exists and this script is executed as this user. +# +# --clean +# +# At the end of the brep instance setup remove installation environment- +# specific traces (host name/IP from the configuration files, etc). Normally +# you would use this option to make the "clean" machine copy for +# distribution. Note that if this option is specified, then the brep +# instance will only be unusable after the machine reboot. +# +usage="Usage: $0 [<options>]" + +# build2 toolchain repository certificate fingerprint. Note: this is a +# repository the toolchain installation script downloads the build2 packages +# from. +# +toolchain_repo_cert_fp="70:64:FE:E4:E0:F3:60:F1:B4:51:E1:FA:12:5C:E0:B3:DB:DF:96:33:39:B9:2E:E5:C2:68:63:4C:A6:47:39:43" +#toolchain_repo_cert_fp="EC:50:13:E2:3D:F7:92:B4:50:0B:BF:2A:1F:7D:31:04:C6:57:6F:BC:BE:04:2E:E0:58:14:FA:66:66:21:1F:14" + +# brep package repository URL and certificate fingerprint. +# +#brep_repo_url="https://pkg.cppget.org/1/alpha" +#brep_repo_cert_fp="70:64:FE:E4:E0:F3:60:F1:B4:51:E1:FA:12:5C:E0:B3:DB:DF:96:33:39:B9:2E:E5:C2:68:63:4C:A6:47:39:43" +brep_repo_url="https://stage.build2.org/1" +brep_repo_cert_fp="EC:50:13:E2:3D:F7:92:B4:50:0B:BF:2A:1F:7D:31:04:C6:57:6F:BC:BE:04:2E:E0:58:14:FA:66:66:21:1F:14" + +owd=`pwd` +trap "{ exit 1; }" ERR +trap "{ cd $owd; }" EXIT +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "error: $*"; exit 1; } + +# Trace a command line, quoting empty arguments as well as those that contain +# spaces. +# +function trace () # <cmd> <arg>... +{ + local s="+" + while [ "$#" -gt 0 ]; do + if [ -z "$1" -o -z "${1##* *}" ]; then + s="$s \"$1\"" + else + s="$s $1" + fi + + shift + done + + info "$s" +} + +# Trace and run a command. +# +run () # <args>... +{ + trace "$@" + "$@" +} + +# The chosen fixed id for the 'brep' user. Note: must match the id of the +# 'brep' user on the host. +# +# Note that Linux assigns the [0 99] range for the statically allocated system +# users and [100 499] -- for dynamic allocations by administrators and post- +# install scripts. Debian, in turn, assigns the [100 999] range for the +# dynamically allocated system users and [60000 64999] -- for statically +# allocated on demand "obscure package users". +# +brep_id=63700 # Update the README file on change. + +# Parse the command line options and, while at it, compose the options array +# for potential re-execution as the 'brep' user. +# +mount= +setup= +clean= +ops=() + +while [ "$#" -gt 0 ]; do + case "$1" in + --mount) + mount=true + ops+=("$1") + shift + ;; + --brep-user) + shift + brep_id="$1" + shift + ;; + --setup) + setup=true + shift + ;; + --clean) + clean=true + ops+=("$1") + shift + ;; + *) + break # The end of options is encountered. + ;; + esac +done + +if [ "$#" -ne 0 ]; then + error "$usage" +fi + +scr_exe="$(realpath "${BASH_SOURCE[0]}")" +scr_dir="$(dirname "$scr_exe")" + +# Unless we are not in the setup mode, non-interactively add the 'brep' +# user/group and re-execute the script in the setup mode as this user. +# +if [ ! "$setup" ]; then + run sudo addgroup --gid "$brep_id" brep + + run sudo adduser --uid "$brep_id" --gid "$brep_id" --disabled-password \ + --gecos "" brep + + run sudo tee -a /etc/sudoers.d/brep >/dev/null <<EOF +brep ALL=(ALL) NOPASSWD:ALL +EOF + + run sudo chmod 0440 /etc/sudoers.d/brep + + # Use --session-command rather than --command|-c to make sure that when the + # su program receives SIGINT (Ctrl-C) it kills not just its child process + # but also all its descendants. + # + # Note: here we rely on ops to not contain spaces or be empty. + # + run exec sudo su -l brep --session-command "'$scr_exe' --setup ${ops[*]}" +fi + +# Here we assume that we are executed as brep user. +# +run cd "$HOME" + +# Mount the brep state directory, if requested. Note that otherwise, the +# directory will be created later, in the local filesystem by the brep-startup +# script. +# +if [ "$mount" ]; then + run sudo mkdir -p /var/brep + + run sudo tee -a /etc/fstab >/dev/null <<EOF +state /var/brep 9p trans=virtio,version=9p2000.L,posixacl,cache=none,_netdev 0 0 +EOF + + run sudo mount -a +fi + +# Install the prerequisite binary packages. +# +run sudo apt-get --yes update +run sudo apt-get --yes install --no-install-recommends g++ +run sudo apt-get --yes install --no-install-recommends postgresql postgresql-contrib libpq-dev +run sudo apt-get --yes install --no-install-recommends apache2 libapr1-dev libapreq2-dev apache2-dev +run sudo apt-get --yes install --no-install-recommends acl rsync +run sudo apt-get clean + +# Install build2 toolchain. +# +run mkdir build2-build +run cd build2-build + +# Look for the toolchain installation script in this script directory. +# +run cp "$(echo "$scr_dir"/build2-install-*.sh)" . +run sh ./build2-install-*.sh --no-check --yes --trust "$toolchain_repo_cert_fp" +#run sh ./build2-install-*.sh --no-check --yes --local + +run cd .. # Back to brep home. + +# Grant Apache2 read access to the module and configuration. +# +run setfacl -m g:www-data:rx "$HOME" +run setfacl -dm g:www-data:rx "$HOME" + +# Install brep. +# +run mkdir brep +run cd brep + +run bpkg create \ + cc \ + config.cc.coptions="-O3" \ + config.cc.poptions="-I$(apxs -q includedir)" \ + config.bin.lib=shared \ + config.bin.rpath="$HOME/install/lib" \ + config.install.root="$HOME/install" + +run bpkg add "$brep_repo_url" +run bpkg fetch --trust "$brep_repo_cert_fp" +run bpkg build --yes brep ?sys:libapr1 ?sys:libapreq2 ?sys:libpq +run bpkg install brep + +run cd .. # Back to brep home. + +# Create PostgreSQL user and databases. +# +# Note that while we could probably omit the build-related setup, let's keep +# it to stay close to the instructions in the INSTALL file and to simplify the +# potential future configuration of the brep instance as a build2 build bot +# controller. +# +run sudo sudo -u postgres psql <<EOF +CREATE DATABASE brep_package +TEMPLATE template0 +ENCODING 'UTF8' +LC_COLLATE 'en_US.UTF8' +LC_CTYPE 'en_US.UTF8'; + +CREATE DATABASE brep_build +TEMPLATE template0 +ENCODING 'UTF8' +LC_COLLATE 'en_US.UTF8' +LC_CTYPE 'en_US.UTF8'; + +CREATE USER brep; + +GRANT ALL PRIVILEGES ON DATABASE brep_package, brep_build TO brep; + +CREATE USER "www-data" INHERIT IN ROLE brep; + +CREATE USER "brep-build" INHERIT IN ROLE brep PASSWORD '-'; + +\c brep_package +GRANT ALL PRIVILEGES ON SCHEMA public TO brep; + +\c brep_build +GRANT ALL PRIVILEGES ON SCHEMA public TO brep; +EOF + +# Create the "staging" package database for the submit-pub package submission +# handler. +# +run sudo sudo -u postgres psql <<EOF +CREATE DATABASE brep_submit_package +TEMPLATE template0 +ENCODING 'UTF8' +LC_COLLATE 'en_US.UTF8' +LC_CTYPE 'en_US.UTF8'; + +GRANT ALL PRIVILEGES ON DATABASE brep_submit_package TO brep; + +\c brep_submit_package +GRANT ALL PRIVILEGES ON SCHEMA public TO brep; +EOF + +# Make sure the 'brep' and Apache2 user's logins work properly. +# +q="SELECT current_database();" +run psql -d brep_package -c "$q" >/dev/null +run psql -d brep_build -c "$q" >/dev/null +run psql -d brep_submit_package -c "$q" >/dev/null + +run sudo sudo -u www-data psql -d brep_package -c "$q" >/dev/null +run sudo sudo -u www-data psql -d brep_build -c "$q" >/dev/null + +# Setup the connection between the databases. +# +run sudo sudo -u postgres psql -d brep_build <<EOF +CREATE EXTENSION postgres_fdw; + +CREATE SERVER package_server +FOREIGN DATA WRAPPER postgres_fdw +OPTIONS (dbname 'brep_package', updatable 'true'); + +GRANT USAGE ON FOREIGN SERVER package_server to brep; + +CREATE USER MAPPING FOR PUBLIC +SERVER package_server +OPTIONS (user 'brep-build', password '-'); +EOF + +# Allow brep-build user to access the brep_package database. +# +f="$(run sudo sudo -u postgres psql -t -A -c "show hba_file;")" +s="# TYPE DATABASE USER ADDRESS METHOD\nlocal brep_package brep-build md5\n\n" + +run sudo sed --in-place=.bak "1s/^/$s/" "$f" +run sudo systemctl restart postgresql + +# Enable creating database tables with columns of the case-insensitive +# character string type. +# +q="CREATE EXTENSION citext;" +run sudo sudo -u postgres psql -d brep_package <<<"$q" +run sudo sudo -u postgres psql -d brep_build <<<"$q" +run sudo sudo -u postgres psql -d brep_submit_package <<<"$q" + +# Copy the brep module configuration. +# +# Note: must be done before bin/brep-startup execution, which adjusts the +# configuration. +# +run mkdir config +run cp "$scr_dir/brep-module.conf" config/ + +# Initialize the brep private instance, in particular creating the database +# schemas and running the brep loader. +# +run mkdir bin/ +run cp "$scr_dir/brep-startup" bin/ +run bin/brep-startup + +# Smoke test the database schemas. +# +run psql -d brep_package -c 'SELECT canonical_name, summary FROM repository' >/dev/null +run psql -d brep_build -c 'SELECT package_name FROM build' >/dev/null +run psql -d brep_build -c 'SELECT DISTINCT name FROM build_package' >/dev/null +run psql -d brep_submit_package -c 'SELECT canonical_name, summary FROM repository' >/dev/null + +# Setup executing the brep-startup script on boot. +# +run sudo cp "$scr_dir/brep-startup.service" /etc/systemd/system/ + +run sudo systemctl start brep-startup.service # Make sure there are no issues. +run sudo systemctl enable brep-startup.service + +# Prepare directories for the package submission service. +# +run mkdir submit-data +run mkdir submit-temp +run setfacl -m g:www-data:rwx submit-data +run setfacl -m g:www-data:rwx submit-temp + +# Make the Apache2 user owned directories fully accessible by the 'brep' user +# (which the submit-pub submission handler will run as). +# +run setfacl -dm g:brep:rwx submit-data +run setfacl -dm g:brep:rwx submit-temp + +# Add the Apache2 user to sudoers, so the submission handler can re-execute +# itself as the 'brep' user. +# +run sudo tee -a /etc/sudoers.d/www-data >/dev/null <<EOF +www-data ALL=(ALL) NOPASSWD:ALL +EOF + +run sudo chmod 0440 /etc/sudoers.d/www-data + +# Setup the Apache2 module. +# +run sudo mkdir -p /var/www/brep/log/ + +run sudo cp "$scr_dir/brep-apache2.conf" /etc/apache2/sites-available/000-brep.conf +run sudo cp "$scr_dir/brep-logrotate" /etc/logrotate.d/brep + +run sudo a2dissite --purge -q 000-default +run sudo a2ensite -q 000-brep + +run sudo systemctl restart apache2 +run sudo systemctl status apache2 >/dev/null + +# Make sure the Apache2 service depends on PostgreSQL and +# brep-startup.service, so that they are started in proper order. +# +run sudo mkdir -p /etc/systemd/system/apache2.service.d/ +run sudo tee /etc/systemd/system/apache2.service.d/postgresql.conf >/dev/null <<EOF +[Unit] +Requires=postgresql.service +After=postgresql.service +EOF + +run sudo tee /etc/systemd/system/apache2.service.d/brep-startup.conf >/dev/null <<EOF +[Unit] +Requires=brep-startup.service +After=brep-startup.service +EOF + +run sudo mkdir -p /etc/systemd/system/postgresql.service.d/ +run sudo tee /etc/systemd/system/postgresql.service.d/apache2.conf >/dev/null <<EOF +[Unit] +Wants=apache2.service +EOF + +run sudo systemctl daemon-reload + +# Verify that Apache2 is stopped after PostgreSQL is stopped. +# +run sudo systemctl stop postgresql + +ec="0" +run sudo systemctl status apache2 >/dev/null || ec="$?" + +if [ "$ec" -ne 3 ]; then + error "exit code 3 (unit is not active) is expected instead of $ec" +fi + +# Verify that Apache2 is started after PostgreSQL is started. +# +run sudo systemctl start postgresql + +run sleep 3 +run sudo systemctl status apache2 >/dev/null + +# Setup periodic loader execution. +# +run sudo cp "$scr_dir/brep-load.service" /etc/systemd/system/ +run sudo cp "$scr_dir/brep-load.timer" /etc/systemd/system/ + +run sudo systemctl start brep-load.service # Make sure there are no issues. + +run sudo systemctl start brep-load.timer +run sudo systemctl status brep-load.timer >/dev/null +run sudo systemctl enable brep-load.timer +run sudo systemctl status brep-load.timer >/dev/null + +# Cleanup the installation environment-specific traces, if requested. +# +if [ "$clean" ]; then + + # Stop the relevant services. + # + run sudo systemctl stop brep-load.timer + run sudo systemctl stop apache2 + + # Remove the host name/IP from the configuration. + # + run cp "$scr_dir/brep-module.conf" config/ # Adjusted by brep-startup. + run rm config/loadtab # Recreated by brep-startup. + + # Finally, stop networking and cleanup the DHCP lease information. + # + # Note that after networking is stopped, sudo prints the 'unable to resolve + # host' diagnostics while trying to obtain the host IP. Thus, we execute the + # last two commands via a single sudo call. + # + run sudo bash -c "systemctl stop networking && rm -rf /var/lib/dhcp/*.leases" +fi diff --git a/etc/private/install/brep-load.service b/etc/private/install/brep-load.service new file mode 100644 index 0000000..272a124 --- /dev/null +++ b/etc/private/install/brep-load.service @@ -0,0 +1,11 @@ +[Unit] +Description=brep repository loader service + +[Service] +Type=oneshot +User=brep +Group=brep +ExecStart=/home/brep/install/bin/brep-load /home/brep/config/loadtab + +[Install] +WantedBy=default.target diff --git a/etc/private/install/brep-load.timer b/etc/private/install/brep-load.timer new file mode 100644 index 0000000..1bf78c4 --- /dev/null +++ b/etc/private/install/brep-load.timer @@ -0,0 +1,33 @@ +[Unit] +Description=brep repository loader timer +RefuseManualStart=no +RefuseManualStop=no + +# Note that due to brep-startup service's oneshot type, this unit won't be +# started until the brep-startup process exits successfully. +# +# Also note that if brep-startup fails and is restarted manually, similar to +# services, the timer is not started automatically. Instead, it has to be +# started manually with `systemctl start brep-load.timer`. +# +Requires=brep-startup.service +After=brep-startup.service + +[Timer] +Unit=brep-load.service + +# Don't keep track of the timer across reboots. +# +Persistent=false + +# Start the timer for the first time. +# +OnBootSec=1 + +# Then wait 4-5 seconds until the next run. +# +OnUnitInactiveSec=4 +AccuracySec=1 + +[Install] +WantedBy=timers.target diff --git a/etc/private/install/brep-logrotate b/etc/private/install/brep-logrotate new file mode 100644 index 0000000..67c7c90 --- /dev/null +++ b/etc/private/install/brep-logrotate @@ -0,0 +1,20 @@ +/var/www/brep/log/*.log { + weekly + missingok + rotate 4 + compress + delaycompress + notifempty + create 640 root adm + sharedscripts + postrotate + if /etc/init.d/apache2 status > /dev/null ; then \ + /etc/init.d/apache2 reload > /dev/null; \ + fi; + endscript + prerotate + if [ -d /etc/logrotate.d/httpd-prerotate ]; then \ + run-parts /etc/logrotate.d/httpd-prerotate; \ + fi; \ + endscript +} diff --git a/etc/private/install/brep-module.conf b/etc/private/install/brep-module.conf new file mode 100644 index 0000000..bfaa8f6 --- /dev/null +++ b/etc/private/install/brep-module.conf @@ -0,0 +1,532 @@ +# Configuration file for the brep module (note: this is not an apache2 .conf +# file but it can be converted to one by prefixing all the options with +# brep-). See brep(1) for detailed description of each configuration option. +# Commented out options indicate their default values. +# +# Besides being parsed by the brep module, this file may also be parsed by +# brep utilities that are normally only interested in the subset of the +# options. To simplify skipping of unrecognized, this file must always have an +# option name and its value on the same line. +# + +# Package search page title. It is placed inside XHTML5 <title> element. +# +# search-title Packages + + +# Package search page description. If specified, it is displayed before the +# search form on the first page only. The value is treated as an XHTML5 +# fragment. +# +# search-description "" + + +# Web page logo. It is displayed in the page header aligned to the left edge. +# The value is treated as an XHTML5 fragment. +# +# logo "" + + +# Web page menu. Each entry is displayed in the page header in the order +# specified and aligned to the right edge. A link target that starts with '/' +# or contains ':' is used as is. Otherwise, it is prefixed with the repository +# web interface root. +# +menu Packages= +# menu Builds=?builds +# menu Configs=?build-configs +menu Submit=?submit +# menu CI=?ci +menu About=?about + + +# Number of packages per page. +# +# search-page-entries 20 + + +# Number of pages in navigation (pager). +# +# search-pages 5 + + +# Number of package description characters to display in brief pages. +# +# package-description 500 + + +# Number of package changes characters to display in brief pages. +# +# package-changes 5000 + + +# The package database connection configuration. By default, brep will try to +# connect to the local instance of PostgreSQL with the operating system- +# default mechanism (Unix-domain socket, etc) and use operating system +# (login) user name and the database called 'brep_package'. If the role name +# is not empty then the login user will be switched (with SET ROLE) to this +# user prior to executing any statements. If not specified, then 'brep' is +# used. See brep(1) for details. +# +# package-db-user +# package-db-role brep +# package-db-password +# package-db-name brep_package +# package-db-host +# package-db-port + + +# The maximum number of concurrent package database connections per web server +# process. If 0, then no limitation is applied. +# +# package-db-max-connections 5 + + +# The maximum number of times to retry package database transactions in the +# face of recoverable failures (deadlock, loss of connection, etc). +# +# package-db-retry 10 + + +# Build configuration file. If not specified (default), then the package +# building functionality will be disabled. If specified, then the build +# database must be configured (see next). Note: must be an absolute path. +# +# build-config + + +# Number of build configurations per page. +# +# build-config-page-entries 20 + + +# Number of pages in navigation (pager). +# +# build-config-pages 5 + + +# Directory containing build bot agent public keys. If specified, then brep +# will perform agent authentication and will reject build results from +# unauthenticated ones. If not specified, then build results are accepted from +# all agents (which will be a security risk if the brep instance is publicly +# accessible). +# +# The directory is expected to contain one PEM-encoded public key per file with +# the .pem extension. All other files and subdirectories are ignored. The brep +# instance needs to be restarted after adding new key files for the changes to +# take effect. +# +# build-bot-agent-keys + + +# Regular expressions in the /<regex>/<replacement>/ form for transforming the +# interactive build login information, for example, into the actual command +# that can be used by the user. The regular expressions are matched against +# the "<agent> <interactive-login>" string containing the respective task +# request manifest values. The first matching expression is used for the +# transformation. If no expression matches, then the task request is +# considered invalid, unless no expressions are specified. Repeat this option +# to specify multiple expressions. +# +# build-interactive-login + + +# Order in which packages are considered for build. The valid values are +# 'stable' and 'random'. If not specified, then 'stable' is assumed. Note that +# interactive builds are always preferred. +# +#build-package-order stable + + +# Number of builds per page. +# +# build-page-entries 20 + + +# Number of pages in navigation (pager). +# +# build-pages 5 + + +# Time to wait before considering a package for a forced rebuild. Must be +# specified in seconds. Default is 10 minutes. +# +# build-forced-rebuild-timeout 600 + + +# Time to wait before considering a package for a soft rebuild (only to be +# performed if the build environment or any of the package dependencies have +# changed). Must be specified in seconds. The special zero value disables soft +# rebuilds. Default is 24 hours. +# +# build-soft-rebuild-timeout 86400 + + +# Alternative package soft rebuild timeout to use instead of the soft rebuild +# timeout (see the build-soft-rebuild-timeout option for details) during the +# specified time interval. Must be specified in seconds. Default is the time +# interval length plus (build-soft-rebuild-timeout - 24h) if soft rebuild +# timeout is greater than 24 hours (thus the rebuild is only triggered within +# the last 24 hours of the build-soft-rebuild-timeout expiration). +# +# The alternative rebuild timeout can be used to "pull" the rebuild window to +# the specified time of day, for example, to optimize load and/or power +# consumption of the build infrastructure (off-work hours, solar, off-peak +# electricity tariffs, etc). A shorter than the time interval rebuild timeout +# can also be used to force continuous rebuilds, for example, to shake out +# flaky tests. Note also that if the alternative rebuild timeout is greater +# than the normal rebuild timeout, then this will result in slower rebuilds +# during the alternative time interval. In this case, if the build +# infrastructure is monitored for delayed package builds, then the alternative +# rebuild timeout should only be made slightly greater than the normal timeout +# (see brep-monitor(1) for details). +# +# The time interval boundaries must be specified as times of day (in the local +# timezone) in the <hours>:<minutes> form. If the stop time is less than the +# start time then the interval extends through midnight. The start and stop +# times must both be either specified or absent. If unspecified, then no +# alternative rebuild timeout will be used. +# +# build-alt-soft-rebuild-timeout +# build-alt-soft-rebuild-start +# build-alt-soft-rebuild-stop + + +# Time to wait before considering a package for a hard rebuild (to be +# performed unconditionally). Must be specified in seconds. The special zero +# value disables hard rebuilds. Default is 7 days. +# +# build-hard-rebuild-timeout 604800 + + +# Alternative package hard rebuild timeout. The semantics is the same as for +# the build-alt-soft-rebuild-* options but for the build-hard-rebuild-timeout +# option. +# +# build-alt-hard-rebuild-timeout +# build-alt-hard-rebuild-start +# build-alt-hard-rebuild-stop + + +# Time to wait before assuming the 'queued' notifications are delivered for +# package CI requests submitted via third-party services (GitHub, etc). During +# this time a package is not considered for a build. Must be specified in +# seconds. Default is 30 seconds. +# +# build-queued-timeout 30 + + +# The maximum size of the build task request manifest accepted. Note that the +# HTTP POST request body is cached to retry database transactions in the face +# of recoverable failures (deadlock, loss of connection, etc). Default is +# 100K. +# +# build-task-request-max-size 102400 + + +# Time to wait before considering the expected task result lost. Must be +# specified in seconds. Default is 3 hours. +# +# build-result-timeout 10800 + + +# The maximum size of the build result manifest accepted. Note that the HTTP +# POST request body is cached to retry database transactions in the face of +# recoverable failures (deadlock, loss of connection, etc). Default is 10M. +# +# build-result-request-max-size 10485760 + + +# Enable or disable package build notification emails in the <name>=<mode> +# form. The valid <mode> values are 'none', 'latest', and 'all'. If 'all' is +# specified for a toolchain name, then emails are sent according to the +# build-*email package manifest values when all versions of a package are +# built with this toolchain. If 'latest' is specified, then for this toolchain +# name the emails are only sent for the latest version of a package. If 'none' +# is specified, then no emails are sent for this toolchain name. By default +# the 'latest' mode is assumed. Repeat this option to enable/disable emails +# for multiple toolchains. +# +# build-toolchain-email <toolchain-name>=latest|none|all + + +# The build database connection configuration. By default, brep will try to +# connect to the local instance of PostgreSQL with the operating system-default +# mechanism (Unix-domain socket, etc) and use operating system (login) user +# name and the database called 'brep_build'. If the role name is not empty +# then the login user will be switched (with SET ROLE) to this user prior +# to executing any statements. If not specified, then 'brep' is used. See +# brep(1) for details. +# +# build-db-user +# build-db-role brep +# build-db-password +# build-db-name brep_build +# build-db-host +# build-db-port + + +# The maximum number of concurrent build database connections per web server +# process. If 0, then no limitation is applied. +# +# build-db-max-connections 5 + + +# The maximum number of times to retry build database transactions in the +# face of recoverable failures (deadlock, loss of connection, etc). +# +# build-db-retry 10 + + +# The root directory where the uploaded binary distribution packages are +# saved to under the following directory hierarchy: +# +# [<tenant>/]<distribution>/<os-release>/<project>/<package>/<version>/<package-config> +# +# The package configuration directory symlinks that match these paths are +# mapped to web URLs based on the bindist-url value and displayed on the +# package version details page. If this option is specified, then bindist-url +# must be specified as well." +# +# bindist-root + + +# The root URL of the directory specified with the bindist-root option. This +# option must be specified if bindist-root is specified. +# +# bindist-url + + +# The openssl program to be used for crypto operations. You can also specify +# additional options that should be passed to the openssl program with +# openssl-option. If the openssl program is not explicitly specified, then brep +# will use openssl by default. +# +# openssl openssl + + +# Additional option to be passed to the openssl program (see openssl for +# details). Repeat this option to specify multiple openssl options. +# +# openssl-option + + +# Environment variable to be set (<name>=<value>) or unset (just <name>) for +# the openssl program (see openssl for details). Repeat this option to specify +# multiple openssl variables. Note that unspecified variables are inherited +# from the web server process. +# +# You need to at least set the RANDFILE environment variable to change the +# default location of the openssl program seed file and maybe also the +# OPENSSL_CONF variable if you would like to use a custom openssl configuration +# file. +# +# openssl-envvar RANDFILE=/home/brep/www-data-openssl.rnd +# openssl-envvar OPENSSL_CONF=/home/brep/www-data-openssl.cnf +# +# To create www-data-openssl.rnd with suitable permissions, run (as user brep): +# +# $ touch www-data-openssl.rnd +# $ setfacl -b -m g:www-data:rw www-data-openssl.rnd +# + + +# The directory to save final submission data to. If unspecified, the package +# submission functionality will be disabled. If specified, then submit-temp +# must be specified as well. +# +# Note that the directory path must be absolute and the directory itself must +# exist and have read, write, and execute permissions granted to the user that +# runs the web server. +# +submit-data /home/brep/submit-data + + +# The directory to save temporary submission data to. Must be specified if the +# package submission functionality is enabled. +# +# Note that this directory must be on the same filesystem and satisfy the same +# requirements as submit-data. It is also the user's responsibility to clean +# it up after an unclean web server shutdown. +# +submit-temp /home/brep/submit-temp + + +# The maximum size of the submission data accepted. Note that currently the +# entire submission request is read into memory. Default is 10M. +# +# 100M. +# +submit-max-size 104857600 + + +# The package submission form fragment. If specified, then its contents are +# treated as an XHTML5 fragment that is inserted into the <body> element of +# the submission page. If unspecified, then no submission page will be +# displayed. Note that the file path must be absolute. +# +submit-form /home/brep/install/share/brep/www/submit.xhtml + + +# The package submission email. If specified, the submission request and +# result manifests will be sent to this address. +# +# submit-email + + +# The handler program to be executed on package submission. The handler is +# executed as part of the HTTP request and is passed additional arguments that +# can be specified with submit-handler-argument followed by the absolute path +# to the submission directory. Note that the program path must be absolute. +# +submit-handler /home/brep/install/bin/brep-submit-pub + + +# Additional arguments to be passed to the submission handler program (see +# submit-handler for details). Repeat this option to specify multiple +# arguments. +# +submit-handler-argument --user +submit-handler-argument brep +submit-handler-argument --result-url +submit-handler-argument http://unknown +submit-handler-argument /home/brep/install/bin/brep-load +submit-handler-argument --db-name=brep_submit_package +submit-handler-argument /var/brep/bpkg/pkg + + +# The handler program timeout in seconds. If specified and the handler does +# not exit in the alloted time, then it is killed and its termination is +# treated as abnormal. +# +submit-handler-timeout 120 + + +# The directory to save CI request data to. If unspecified, the package CI +# functionality will be disabled. +# +# Note that the directory path must be absolute and the directory itself must +# exist and have read, write, and execute permissions granted to the user that +# runs the web server. +# +# ci-data + + +# The package CI form fragment. If specified, then its contents are treated as +# an XHTML5 fragment that is inserted into the <body> element of the CI page. +# If unspecified, then no CI page will be displayed. Note that the file path +# must be absolute. +# +# ci-form + + +# The package CI email. If specified, the CI request and result manifests will +# be sent to this address. +# +# ci-email + + +# The handler program to be executed on CI request. The handler is executed as +# part of the HTTP request and is passed additional arguments that can be +# specified with ci-handler-argument followed by the absolute path to the CI +# request directory. Note that the program path must be absolute. +# +# ci-handler + + +# Additional arguments to be passed to the CI handler program (see ci-handler +# for details). Repeat this option to specify multiple arguments. +# +# ci-handler-argument + + +# The CI handler program timeout in seconds. If specified and the handler does +# not exit in the allotted time, then it is killed and its termination is +# treated as abnormal. +# +# ci-handler-timeout + + +# The directory to save upload data to for the specified upload type. If +# unspecified, the build artifacts upload functionality will be disabled for +# this type. +# +# Note that the directory path must be absolute and the directory itself must +# exist and have read, write, and execute permissions granted to the user that +# runs the web server. +# +# upload-data <type>=<dir> + + +# The maximum size of the upload data accepted for the specified upload type. +# Note that currently the entire upload request is read into memory. The +# default is 10M. +# +# upload-max-size <type>=10485760 + + +# The build artifacts upload email. If specified, the upload request and +# result manifests will be sent to this address. +# +# upload-email <type>=<email> + + +# The handler program to be executed on build artifacts upload of the +# specified type. The handler is executed as part of the HTTP request and is +# passed additional arguments that can be specified with +# upload-handler-argument followed by the absolute path to the upload +# directory (upload-data). Note that the program path must be absolute. +# +# upload-handler <type>=<path> + + +# Additional arguments to be passed to the upload handler program for the +# specified upload type (see upload-handler for details). Repeat this option +# to specify multiple arguments. +# +# upload-handler-argument <type>=<arg> + + +# The upload handler program timeout in seconds for the specified upload type. +# If specified and the handler does not exit in the allotted time, then it is +# killed and its termination is treated as abnormal. +# +# upload-handler-timeout <type>=<seconds> + + +# Disable upload of the specified type for the specified toolchain name. +# Repeat this option to disable uploads for multiple toolchains. +# +# upload-toolchain-exclude <type>=<name> + + +# Disable upload of the specified type for packages from the repository with +# the specified canonical name. Repeat this option to disable uploads for +# multiple repositories. +# +# upload-repository-exclude <type>=<name> + + +# The default view to display for the global repository root. The value is one +# of the supported services (packages, builds, submit, ci, etc). Default is +# packages. +# +# root-global-view packages + + +# The default view to display for the tenant repository root. The value is one +# of the supported services (packages, builds, submit, ci, etc). Default is +# packages. +# +# root-tenant-view packages + + +# Name to call the tenant values on web pages. If not specified, then 'tenant' +# is used. +# +# tenant-name tenant + + +# Trace verbosity. Disabled by default. +# +# verbosity 0 diff --git a/etc/private/install/brep-startup b/etc/private/install/brep-startup new file mode 100755 index 0000000..780a2c0 --- /dev/null +++ b/etc/private/install/brep-startup @@ -0,0 +1,88 @@ +#! /usr/bin/env bash + +# file : etc/private/install/brep-startup +# license : MIT; see accompanying LICENSE file + +# (Re-)initialize the brep private instance, normally on the machine startup. +# +# Specifically: +# +# - Create the pkg repository and symlink to it, unless already exists. +# +# - Migrate the brep databases as a sanity check. +# +# - Adjust the brep module configuration file using the current host name/IP. +# +# - Generate the loadtab using the current host name/IP and run the loader. +# +trap "{ exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "error: $*"; exit 1; } + +# Create the pkg repository, if required. +# +d=/var/brep/bpkg + +if [ ! -L "$d/pkg" ]; then + rd="$(date "+pkg-%Y%m%d-%H%M%S-%N")" + + mkdir -p "$d/$rd/1" + ln -s "$rd" "$d/pkg" +fi + +r="$d/pkg/1" + +if [ ! -f "$r/repositories.manifest" ]; then + cat <<EOF >"$r/repositories.manifest" +: 1 +#summary: Private repository +#description: \\ +#This is a private repository. +#And this description can contain multiple lines. +#\\ +#email: admin@example.org + +#: +#role: prerequisite +#location: https://pkg.cppget.org/1/stable +#trust: ... +EOF +fi + +if [ ! -f "$r/packages.manifest" ]; then + bpkg rep-create -q "$r" +fi + +# Migrate the databases. +# +"$HOME/install/bin/brep-migrate" package +"$HOME/install/bin/brep-migrate" build +"$HOME/install/bin/brep-migrate" -n brep_submit_package package + +# Deduce the machine host name. +# +h="$(hostname -f)" +if [ "$h" == "localhost" ]; then + h="$(hostname -I | sed 's/ *$//')" # Strip the potential trailing space(s). +fi + +if [ -z "$h" ]; then + error "unable to obtain host name or IP address" +fi + +# Adjust the submission result URL host name in the brep module configuration +# file. +# +sed --in-place -re \ +"\$!N;s%^\s*(submit-handler-argument\s+--result-url\s*\\n)\ +\s*(submit-handler-argument\s+https?://)[^/]*(.*)\$%\1\2$h\3%;P;D" \ +"$HOME/config/brep-module.conf" + +# (Re-)generate the loadtab file and reload the repository. +# +f="$HOME/config/loadtab" + +echo "http://$h/1 private cache:$r" >"$f" +"$HOME/install/bin/brep-load" "$f" diff --git a/etc/private/install/brep-startup.service b/etc/private/install/brep-startup.service new file mode 100644 index 0000000..a3dc546 --- /dev/null +++ b/etc/private/install/brep-startup.service @@ -0,0 +1,17 @@ +[Unit] +Description=brep instance initialization service + +Wants=network-online.target +After=network-online.target + +Requires=postgresql.service +After=postgresql.service + +[Service] +Type=oneshot +User=brep +Group=brep +ExecStart=/home/brep/bin/brep-startup + +[Install] +WantedBy=default.target diff --git a/etc/private/install/vm-gen-service b/etc/private/install/vm-gen-service new file mode 100755 index 0000000..ae49a49 --- /dev/null +++ b/etc/private/install/vm-gen-service @@ -0,0 +1,207 @@ +#! /usr/bin/env bash + +# Generate systemd .service file for QEMU/KVM virtual machines. +# +# Normally the machines are run from a dedicated user account with its home +# directory containing all the relevant files (management scripts, images, +# configurations, and sockets). However, this can be overriden with the +# following options +# +# --home <dir> +# The virtual machines "home" directory. If unspecified, the user's home +# directory is assumed. +# +# --bin <dir> +# The virtual machines management scripts directory. If unspecified, +# <home>/bin is assumed. If specified as relative, then assumed relative +# to <home>. +# +# --etc <dir> +# The virtual machines configuration files directory. If unspecified, +# <home>/etc is assumed. If specified as relative, then assumed relative +# to <home>. +# +# --var <dir> +# The virtual machines image files directory. If unspecified, <home>/var is +# assumed. If specified as relative, then assumed relative to <home>. +# +# --run <dir> +# The virtual machines sockets directory. If unspecified, <home>/run is +# assumed. If specified as relative, then assumed relative to <home>. +# +# If <user> is unspecified, the current user is assumed. If <group> is +# unspecified, the user's primary group is assumed. +# +usage="usage: $0 [<options>] [<user>] [<group>]" + +owd="$(pwd)" +trap "{ cd '$owd'; exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +home= +bin= +etc= +var= +run= + +while [ "$#" -gt 0 ]; do + case "$1" in + --home) + shift + home="$1" + shift + ;; + --bin) + shift + bin="$1" + shift + ;; + --etc) + shift + etc="$1" + shift + ;; + --var) + shift + var="$1" + shift + ;; + --run) + shift + run="$1" + shift + ;; + *) + break + ;; + esac +done + +user="$1" + +if [ -z "$user" ]; then + user="$(id -un)" +fi + +group="$2" + +if [ -z "$group" ]; then + group="$(id -un "$user")" +fi + +if [ -z "$home" ]; then + home="$(eval echo ~$user)" +fi + +function complete_dir () # <default> <home> <dir> +{ + local r + if [ -z "$3" ]; then + r="$2/$1" + elif [ "${3:0:1}" != "/" ]; then + r="$2/$3" + else + r="$3" + fi + echo "$(realpath --no-symlinks --canonicalize-missing "$r")" +} + +bin="$(complete_dir bin "$home" "$bin")" +etc="$(complete_dir etc "$home" "$etc")" +var="$(complete_dir var "$home" "$var")" +run="$(complete_dir run "$home" "$run")" + +name="vm-$user" +file="$name@.service" + +# Thinks that must be \-escaped: +# +# - $ (including in comments) +# - \ (e.g., in line continuations) +# +cat <<EOF >"$file" +# $file -- QEMU/KVM machine service template for systemd +# +# user: $user +# group: $group +# bin: $bin +# etc: $etc +# var: $var +# run: $run +# +# To install: +# +# sudo cp $file /etc/systemd/system/ +# sudo chmod 644 /etc/systemd/system/$file +# +# cp ... $var/<machine>.img +# nano $etc/<machine>.conf # Specify RAM, CPU, TAP, MAC, etc. +# +# sudo systemctl start $name@<machine> +# sudo systemctl status $name@<machine> +# login-machine $run/<machine>-con.sock +# sudo systemctl stop $name@<machine> +# +# sudo systemctl enable $name@<machine> + +[Unit] +Description=QEMU/KVM virtual machine %I + +Wants=network-online.target +#After=network-online.target +After=multi-user.target + +[Service] +User=$user +Group=$user +UMask=0007 +WorkingDirectory=~ + +Environment=CPU=1 +Environment=RAM=2G + +# These MUST be specific in EnvironmentFile! +# +#Environment=TAP= +#Environment=MAC= + +# Note that using variable expansion in EnvironmentFile does not work (at +# least not with systemd 229). +# +EnvironmentFile=$etc/%i.conf + +# Note that the first word of ExecStart cannot contain variable expansions. +# +ExecStart=$bin/vm-start \\ + --cpu \${CPU} \\ + --ram \${RAM} \\ + --tap \${TAP} \\ + --mac \${MAC} \\ + --pid $run/%i.pid \\ + --monitor $run/%i-mon.sock \\ + --console $run/%i-con.sock \\ + $var/%i.img + +ExecStop=$bin/vm-stop $run/%i.pid $run/%i-mon.sock + +# This makes sure systemd waits for the ExecStart command to exit rather +# than killing it as soon as ExecStop exits (this is necessary since our +# vm-stop may exit just before vm-start). +# +KillMode=none +TimeoutStopSec=60 + +[Install] +WantedBy=multi-user.target +EOF + +info "generated $file for" +info " user: $user" +info " group: $group" +info " bin: $bin" +info " etc: $etc" +info " var: $var" +info " run: $run" |