diff options
Diffstat (limited to 'etc')
29 files changed, 2892 insertions, 13 deletions
diff --git a/etc/brep-module.conf b/etc/brep-module.conf index 458261e..d5a5e78 100644 --- a/etc/brep-module.conf +++ b/etc/brep-module.conf @@ -3,12 +3,24 @@ # brep-). See brep(1) for detailed description of each configuration option. # Commented out options indicate their default values. # +# Besides being parsed by the brep module, this file may also be parsed by +# brep utilities that are normally only interested in the subset of the +# options. To simplify skipping of unrecognized, this file must always have an +# option name and its value on the same line. +# # Package search page title. It is placed inside XHTML5 <title> element. # # search-title Packages +# Package search page description. If specified, it is displayed before the +# search form on the first page only. The value is treated as an XHTML5 +# fragment. +# +# search-description "" + + # Web page logo. It is displayed in the page header aligned to the left edge. # The value is treated as an XHTML5 fragment. # @@ -107,6 +119,25 @@ menu About=?about # build-bot-agent-keys +# Regular expressions in the /<regex>/<replacement>/ form for transforming the +# interactive build login information, for example, into the actual command +# that can be used by the user. The regular expressions are matched against +# the "<agent> <interactive-login>" string containing the respective task +# request manifest values. The first matching expression is used for the +# transformation. If no expression matches, then the task request is +# considered invalid, unless no expressions are specified. Repeat this option +# to specify multiple expressions. +# +# build-interactive-login + + +# Order in which packages are considered for build. The valid values are +# 'stable' and 'random'. If not specified, then 'stable' is assumed. Note that +# interactive builds are always preferred. +# +#build-package-order stable + + # Number of builds per page. # # build-page-entries 20 @@ -123,10 +154,66 @@ menu About=?about # build-forced-rebuild-timeout 600 -# Time to wait before considering a package for a normal rebuild. Must be -# specified in seconds. Default is 24 hours. +# Time to wait before considering a package for a soft rebuild (only to be +# performed if the build environment or any of the package dependencies have +# changed). Must be specified in seconds. The special zero value disables soft +# rebuilds. Default is 24 hours. +# +# build-soft-rebuild-timeout 86400 + + +# Alternative package soft rebuild timeout to use instead of the soft rebuild +# timeout (see the build-soft-rebuild-timeout option for details) during the +# specified time interval. Must be specified in seconds. Default is the time +# interval length plus (build-soft-rebuild-timeout - 24h) if soft rebuild +# timeout is greater than 24 hours (thus the rebuild is only triggered within +# the last 24 hours of the build-soft-rebuild-timeout expiration). # -# build-normal-rebuild-timeout 86400 +# The alternative rebuild timeout can be used to "pull" the rebuild window to +# the specified time of day, for example, to optimize load and/or power +# consumption of the build infrastructure (off-work hours, solar, off-peak +# electricity tariffs, etc). A shorter than the time interval rebuild timeout +# can also be used to force continuous rebuilds, for example, to shake out +# flaky tests. Note also that if the alternative rebuild timeout is greater +# than the normal rebuild timeout, then this will result in slower rebuilds +# during the alternative time interval. In this case, if the build +# infrastructure is monitored for delayed package builds, then the alternative +# rebuild timeout should only be made slightly greater than the normal timeout +# (see brep-monitor(1) for details). +# +# The time interval boundaries must be specified as times of day (in the local +# timezone) in the <hours>:<minutes> form. If the stop time is less than the +# start time then the interval extends through midnight. The start and stop +# times must both be either specified or absent. If unspecified, then no +# alternative rebuild timeout will be used. +# +# build-alt-soft-rebuild-timeout +# build-alt-soft-rebuild-start +# build-alt-soft-rebuild-stop + + +# Time to wait before considering a package for a hard rebuild (to be +# performed unconditionally). Must be specified in seconds. The special zero +# value disables hard rebuilds. Default is 7 days. +# +# build-hard-rebuild-timeout 604800 + + +# Alternative package hard rebuild timeout. The semantics is the same as for +# the build-alt-soft-rebuild-* options but for the build-hard-rebuild-timeout +# option. +# +# build-alt-hard-rebuild-timeout +# build-alt-hard-rebuild-start +# build-alt-hard-rebuild-stop + + +# Time to wait before assuming the 'queued' notifications are delivered for +# package CI requests submitted via third-party services (GitHub, etc). During +# this time a package is not considered for a build. Must be specified in +# seconds. Default is 30 seconds. +# +# build-queued-timeout 30 # The maximum size of the build task request manifest accepted. Note that the @@ -150,6 +237,19 @@ menu About=?about # build-result-request-max-size 10485760 +# Enable or disable package build notification emails in the <name>=<mode> +# form. The valid <mode> values are 'none', 'latest', and 'all'. If 'all' is +# specified for a toolchain name, then emails are sent according to the +# build-*email package manifest values when all versions of a package are +# built with this toolchain. If 'latest' is specified, then for this toolchain +# name the emails are only sent for the latest version of a package. If 'none' +# is specified, then no emails are sent for this toolchain name. By default +# the 'latest' mode is assumed. Repeat this option to enable/disable emails +# for multiple toolchains. +# +# build-toolchain-email <toolchain-name>=latest|none|all + + # The build database connection configuration. By default, brep will try to # connect to the local instance of PostgreSQL with the operating system-default # mechanism (Unix-domain socket, etc) and use operating system (login) user @@ -178,6 +278,25 @@ menu About=?about # build-db-retry 10 +# The root directory where the uploaded binary distribution packages are +# saved to under the following directory hierarchy: +# +# [<tenant>/]<distribution>/<os-release>/<project>/<package>/<version>/<package-config> +# +# The package configuration directory symlinks that match these paths are +# mapped to web URLs based on the bindist-url value and displayed on the +# package version details page. If this option is specified, then bindist-url +# must be specified as well." +# +# bindist-root + + +# The root URL of the directory specified with the bindist-root option. This +# option must be specified if bindist-root is specified. +# +# bindist-url + + # The openssl program to be used for crypto operations. You can also specify # additional options that should be passed to the openssl program with # openssl-option. If the openssl program is not explicitly specified, then brep @@ -254,10 +373,9 @@ menu About=?about # The handler program to be executed on package submission. The handler is -# executed as part of the submission request and is passed additional -# arguments that can be specified with submit-handler-argument followed by -# the absolute path to the submission directory. Note that the program path -# must be absolute. +# executed as part of the HTTP request and is passed additional arguments that +# can be specified with submit-handler-argument followed by the absolute path +# to the submission directory. Note that the program path must be absolute. # # submit-handler @@ -321,6 +439,66 @@ menu About=?about # ci-handler-timeout +# The directory to save upload data to for the specified upload type. If +# unspecified, the build artifacts upload functionality will be disabled for +# this type. +# +# Note that the directory path must be absolute and the directory itself must +# exist and have read, write, and execute permissions granted to the user that +# runs the web server. +# +# upload-data <type>=<dir> + + +# The maximum size of the upload data accepted for the specified upload type. +# Note that currently the entire upload request is read into memory. The +# default is 10M. +# +# upload-max-size <type>=10485760 + + +# The build artifacts upload email. If specified, the upload request and +# result manifests will be sent to this address. +# +# upload-email <type>=<email> + + +# The handler program to be executed on build artifacts upload of the +# specified type. The handler is executed as part of the HTTP request and is +# passed additional arguments that can be specified with +# upload-handler-argument followed by the absolute path to the upload +# directory (upload-data). Note that the program path must be absolute. +# +# upload-handler <type>=<path> + + +# Additional arguments to be passed to the upload handler program for the +# specified upload type (see upload-handler for details). Repeat this option +# to specify multiple arguments. +# +# upload-handler-argument <type>=<arg> + + +# The upload handler program timeout in seconds for the specified upload type. +# If specified and the handler does not exit in the allotted time, then it is +# killed and its termination is treated as abnormal. +# +# upload-handler-timeout <type>=<seconds> + + +# Disable upload of the specified type for the specified toolchain name. +# Repeat this option to disable uploads for multiple toolchains. +# +# upload-toolchain-exclude <type>=<name> + + +# Disable upload of the specified type for packages from the repository with +# the specified canonical name. Repeat this option to disable uploads for +# multiple repositories. +# +# upload-repository-exclude <type>=<name> + + # The default view to display for the global repository root. The value is one # of the supported services (packages, builds, submit, ci, etc). Default is # packages. diff --git a/etc/buildfile b/etc/buildfile index 8518b2b..f3157f2 100644 --- a/etc/buildfile +++ b/etc/buildfile @@ -1,14 +1,30 @@ # file : etc/buildfile -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file ./: file{** -buildfile} -# Install into the etc/ subdirectory of, say, /usr/share/ -# recreating subdirectories. +# Install into the etc/ subdirectory of, say, /usr/share/ recreating +# subdirectories. # *: { install = data/etc/ install.subdirs = true } + +# Keep the executable permission for the installed script files. +# +private/ +{ + file{vm-start}@./ \ + file{vm-start-base}@./ \ + file{vm-login}@./ \ + file{vm-stop}@./: install.mode=755 + + install/ + { + file{brep-install}@./ \ + file{brep-startup}@./ \ + file{vm-gen-service}@./ : install.mode=755 + } +} diff --git a/etc/private/README b/etc/private/README new file mode 100644 index 0000000..d9a702e --- /dev/null +++ b/etc/private/README @@ -0,0 +1,335 @@ +This directory contains a virtual machine (VM) with a build2 repository web +interface (brep) installed and configured for a private package repository. +It also includes a number of scripts and configuration files for running +this VM as a systemd service. + +A brep installation consists of a web server (Apache2), a database server +(PostgreSQL), and a number of auxiliary processes (repository loader, +submission handler, etc). While all this can be installed and configured +manually (as described in brep/INSTALL), this VM has everything pre-installed +and pre-configured which makes it possible to quickly get a private +repository up and running. + +Note that the configuration offered by this VM is only suitable for a +private/trusted environment, normally either for personal use (host-local) +or for use within an organization's private network. Specifically: + + - The repository is accessed via HTTP. + + - The repository is not signed. + + - Submitted packages are published directly and without ownership + authentication. + + - The VM does not auto-update (since it does not assume the presence of an + Internet connection) and therefore may not have the latest security + patches. + +The below setup instructions are for host machines that run systemd-based +Linux distributions. Note, however, that it should be possible to use other +distributions or operating systems provided they are capable of running +QEMU/KVM virtual machines. The following utilities are expected to be +available on the host machine: + + - systemd >= 229 (systemd --version) + - bash >= 4.3 (bash --version) + - qemu >= 2.5.0 (qemu-system-x86_64 --version) + - screen, socat (screen --version, socat -V) + +Consult your distribution's package manager if any of these utilities are +missing. + +The host machine is also expected to have KVM virtualization support as +well as at least 1G or RAM (2G recommended) and at least 5G of disk space +(4G for VM image and the rest for package storage) that can be dedicated +to the VM. + +Commands shown in this guide use several prompts with the following meaning: + + # -- must be executed as root on the host machine + $ -- must be execute as user brep on the host machine + > -- can be executed for testing on any other machine with build2 installed + + +1. Create the brep user and group +--------------------------------- + +In this setup, the VM image, scripts, etc., as well as the repository packages +are all kept in the home directory of the special user brep. In particular, +the packages are stored on the host machine (as opposed to inside the VM +image) and are shared with the VM (using the virtio-9p filesystem). As a +result, if necessary, you can manipulate the package repository from the host +machine (but see Step 6 below for the rules). This setup also makes it easier +to upgrade VM images by simply replacing the old image with a new (see Step 7 +below for details). + +However, to make this arrangement work reliably, the brep user/group IDs on +the host machine must match those inside the VM. As a result, we create the +brep user/group with specific IDs: + +# groupadd --gid 63700 brep +# useradd --uid 63700 --gid 63700 --create-home brep +# usermod --lock brep # disable password login (if desired) + +Additionally, if your distribution requires users that are allowed to use KVM +to belong to a special group (normally kvm), then add the brep user to this +group: + +# usermod -G kvm brep + +If unsure whether this is required, skip this step and come back to it if you +get the 'KVM: permission denied' error on Step 4. + + +2. Download and unpack the VM archive into the brep user's home directory +------------------------------------------------------------------------- + +# su - brep +$ pwd +/home/brep + +$ curl -fO https://download.build2.org/X.Y.Z/linux-debian-N-brep-X.Y.Z.tar.xz +$ sha256sum -b linux-debian-N-brep-X.Y.Z.tar.xz + +Verify the checksum matches the one from https://build2.org/download.xhtml + +$ tar -xf linux-debian-N-brep-X.Y.Z.tar.xz --strip-components=1 +$ ls +bin/ etc/ vm/ vm-brep@.service README NEWS + + +3. Configure networking for the VM +---------------------------------- + +This setup expects the VM to use bridged networking with a persistent tap +interface. This allows for a wide variety of configurations ranging between +host-local (private bridge without routing), subnet (private bridge with +routing/NAT), and local area network (public bridge over host's Ethernet +adapter). In particular, the last configuration would make the repository +accessible from other machines on the same local network. + +The exact steps on how to setup bridged networking and create a persistent tap +interface depend the network manager used thus consult your distribution's +documentation for details. The guide found in etc/systemd-networkd/README +shows how to setup the local area network configuration mentioned above using +the systemd-networkd network manager available on most systemd-based +distributions. + +In the rest of this guide we assume that tap interface called tap0 is +appropriately configured and is owned by user/group brep. + + +4. Generate a MAC address and start the VM for testing +------------------------------------------------------ + +The recommended way to obtain a MAC address for the VM is to generate it based +on the address of the host's Ethernet adapter (see inside vm-gen-macaddress +for details): + +$ ~/bin/vm-gen-macaddress xx:yy:yy:yy:yy:yy 0 + +Where xx:yy:yy:yy:yy:yy is the MAC address of the host's Ethernet adapter +which can can be viewed with the following command: + +# ip link show + +The address printed by vm-gen-macaddress will be in the 02:yy:yy:yy:yy:yy +form. + +If you are using a local network configuration, then now is a good time to +assign the VM its IP address and hostname. If you need to submit a request to +your network administrator, then the following text could serve as a template: + +"I would like to run a VM on the <host> machine that needs to have its own IP + address and domain name (configured via DHCP). It will have fixed MAC address + <mac> (which was derived from <host>'s physical Ethernet address; but you are + welcome to assign a different MAC address if required). The DHCP client ID is + the same as the MAC address. I would like this machine to have the <vm> name + if possible. + + FYI, this is a QEMU/KVM virtual machine running as a systemd service. It + will use bridged networking with a tap interface." + +Where: + + <host> host machine's name, for example, myserver.lan (run hostname -f) + <mac> the generated mac address (02:yy:yy:yy:yy:yy) + <vm> VM machine's name, for example, mybrep.lan + +Note that the VM is configured to receive its hostname from DHCP server (the +DHCP protocol option 12, "Host Name"). Failed that, the repository URL will +use the IP address. + +Next, create the package repository directory and start the VM for testing +(replace 02:yy:yy:yy:yy:yy with the actual MAC address): + +$ mkdir -p state/bpkg +$ ~/bin/vm-start --stdio --tap tap0 --mac 02:yy:yy:yy:yy:yy vm/brep.img + +After booting, you will be presented with a login. Login as root with password +123 (VM command prompts are shown indented with two spaces). Then verify IP +address, hostname, and the network functionality: + + # ip addr show + # hostname -f + # ping example.org + +If everything appears correct, visit the repository web page with a browser +(for example, http://mybrep.lan). Check the About page to verify the +repository URL matches the hostname or IP address. + +Try to submit a package (for example, from your development machine): + +> bdep new hello +> cd hello +> git add . && git commit -m test +> bdep init -C @test cc +> bdep publish --control=none --repository http://mybrep.lan --force=snapshot + +Visit the repository web page and confirm the package is there. Then try to +consume the submitted package from the repository: + +> bpkg create -d test +> bpkg build -d test hello@http://mybrep.lan/1 + +If everything is working fine, shut the VM down: + + # shutdown -h now + + +5. Setup the VM to run as a systemd service +------------------------------------------- + +To start the VM as a systemd service on host boot, perform the following +steps. + +First, create the VM configuration file (replace 02:yy:yy:yy:yy:yy with the +actual MAC address):: + +$ cat <<EOF >vm/brep.conf +RAM=2G +CPU=1 +TAP=tap0 +MAC=02:yy:yy:yy:yy:yy +EOF + +Then configure the systemd service: + +# cp ~brep/vm-brep@.service /etc/systemd/system/ +# chmod 644 /etc/systemd/system/vm-brep@.service +# systemctl status vm-brep@brep +# systemctl start vm-brep@brep +# systemctl status vm-brep@brep + +If the VM fails to start, study the logs for a possible cause: + +# journalctl -u vm-brep@brep + +If the VM has started successfully, perform the same verifications as on Step +4 above. + +To login to the VM running as a systemd service (for example, to verify IP and +hostname) use the vm-login script (which uses screen(1) to connect to the VM's +console): + +$ ~/bin/vm-login ~/brep-con.sock + +Note that the screen may be blank (due to this being a serial console). In +this case, press Enter to see the login. To close the login, press 'Ctrl-a k' +(or 'Ctrl-a a k' if already running inside screen). + +If everything functions correctly, verify the VM can be stopped: + +# systemctl stop vm-brep@brep +# systemctl status vm-brep@brep + +Finally, if desired, enable the VM service to start on boot: + +# systemctl enable vm-brep@brep + +After this you may also want to reboot the host machine and confirm the VM is +started on boot. + + +6. Manage the repository state +------------------------------ + +While you can submit packages to the repository using bdep-publish(1), they +can also be added them manually. Also, currently, packages can only be removed +manually. + +The repository packages and metadata are stored in the ~brep/state/bpkg/pkg/ +directory. If you need to make any modifications in this directory, there are +two rules that you must follow: + + 1. You must stop the VM before making any modifications. + + 2. You must make any modification only as user brep. + +After performing the modifications, remove the 1/packages.manifest file to +trigger the repository metadata regeneration on the next VM startup. You can +also customize the repositories.manifest file in the same way. For example, +you could add cppget.org as a prerequisite repository for your private +repository. + +Putting it all together, the steps could look like this: + +# systemctl stop vm-brep@brep +# su - brep +$ cd state/bpkg/pkg/1 +$ <make your changes here> +$ rm packages.manifest +$ exit +# systemctl start vm-brep@brep + +Note also that it's easy to break the repository with manual modifications. +For example, you may add a package that has an unmet dependency or remove a +package that still has some dependents. In this case, the brep service inside +the VM will fail to start and the repository web interface will be +unavailable. In this case, you can login into the VM to investigate: + +$ ~/bin/vm-login ~/brep-con.sock + + # systemctl status brep-startup + # journalctl -u brep-startup + + +7. Upgrade the VM +----------------- + +To upgrade to the new version of the VM, first download and unpack the new +VM archive similar to Step 2: + +$ curl -fO https://download.build2.org/X.Y.Z/linux-debian-N-brep-X.Y.Z.tar.xz +$ sha256sum -b linux-debian-N-brep-X.Y.Z.tar.xz + +Verify the checksum matches the one from https://build2.org/download.xhtml + +$ tar -xf linux-debian-N-brep-X.Y.Z.tar.xz + +Next read the linux-debian-N-brep-X.Y.Z/NEWS file for changes and potential +backwards compatibility issues. Unless instructed otherwise by the NEWS file, +the upgrade procedure is as follows: + +# systemctl stop vm-brep@brep + +$ cd +$ mkdir bak +$ mv -t bak/ bin etc vm vm-brep@.service README NEWS +$ mv -t ./ linux-debian-N-brep-X.Y.Z/* +$ cp bak/vm/brep.conf vm/ +$ rm state/bpkg/pkg/1/packages.manifest + +# cp ~brep/vm-brep@.service /etc/systemd/system/ +# chmod 644 /etc/systemd/system/vm-brep@.service +# systemctl daemon-reload +# systemctl start vm-brep@brep +# systemctl status vm-brep@brep + +If the VM has started successfully, perform the same verifications as on Step +4 above. If everything is functioning correctly, you can remove the backup +files: + +$ rm -r bak + +If there are any issues, investigate as on Step 6. diff --git a/etc/private/install/README b/etc/private/install/README new file mode 100644 index 0000000..ef3ae70 --- /dev/null +++ b/etc/private/install/README @@ -0,0 +1,75 @@ +This directory contains scripts, configuration files, etc., that are used to +prepare the virtual machine (VM) with a build2 repository web interface (brep) +described in ../README. This document provides terse notes on how this VM is +prepared. + +The VM is prepared and tested using user brep: + +# groupadd --gid 63700 brep +# useradd --uid 63700 --gid 63700 --create-home brep +# usermod --lock brep +# usermod -G kvm brep + +Note: different UID/GID can be used by passing the --brep-user option to the +brep-install script. + +# su - brep +$ mkdir -p bin vm state/bpkg + +Next copy the prepared VM image: + +$ cp .../linux_debian_10.img vm/brep.img + +The brep-install script assumes a Debian-based VM distribution. Other +distributions can probably be made to work but will require changes to +brep-install. The VM normally already has the following changes applied: + + # apt-get install acpid # For QEMU system_powerdown to work. + # systemctl enable acpid + + # systemctl enable serial-getty@ttyS0.service + # systemctl start serial-getty@ttyS0.service + + # nano /etc/default/grub + # # GRUB_CMDLINE_LINUX_DEFAULT="console=ttyS0" + # # GRUB_TERMINAL="serial console" + # update-grub + + # echo localhost >/etc/hostname + # nano /etc/dhcp/dhclient.conf + # # Comment out `send host-name ...`. + # # Add `send dhcp-client-identifier = hardware;` + +Clone or copy the brep repository and create the directory structure: + +$ cp -r .../brep ./ +$ cp brep/etc/private/vm-* bin/ +$ cp -r brep/etc/private ./etc + +Download the build2 toolchain installation script: + +$ curl -sSfO https://download.build2.org/X.Y.Z/build2-install-X.Y.Z.sh +$ # Verify the checksum. +$ mv build2-install-*.sh etc/install/ + +Start the VM (give as much CPU/RAM as available to speed up compilation): + +$ ~/bin/vm-start --install etc/install/ --cpu 8 --ram 8G \ + --tap tap0 --mac de:ad:be:ef:de:ad vm/brep.img + +Login into the VM as root, then perform the following steps: + + # mount -t 9p -o trans=virtio,version=9p2000.L install /mnt + # /mnt/brep-install --mount + +After the installation is complete, test the result as described in ../README. + +Note: to create a "clean" VM for distribution, pass the --clean option to +brep-install, shut the VM down immediately after installation, save the clean +VM image, then boot a copy for testing. + +Generate the systemd service template file: + +~/etc/install/vm-gen-service --bin bin --etc vm --var vm --run . + +Test starting the VM as a systemd service as described in ../README. diff --git a/etc/private/install/brep-apache2.conf b/etc/private/install/brep-apache2.conf new file mode 100644 index 0000000..99186d1 --- /dev/null +++ b/etc/private/install/brep-apache2.conf @@ -0,0 +1,99 @@ +# Keep in the main server configuration context. This way the directive will +# be in effect during module initialization and request handling. +# +# Note that initialization log messages are written to the main server log +# file (/var/log/apache2/error.log), and request handling messages to the +# virtual server log file (/var/www/brep/log/error.log). +# +LogLevel brep:info + +<VirtualHost *:80> + #ServerName <brep-hostname> + #ServerAdmin <brep-admin-email> + + #DocumentRoot /var/www/brep/public + #Options +Indexes + + AddOutputFilterByType DEFLATE application/xhtml+xml + AddOutputFilterByType DEFLATE text/manifest + AddOutputFilterByType DEFLATE text/plain + AddOutputFilterByType DEFLATE text/css + + Alias "/1" "/var/brep/bpkg/pkg/1" + + ErrorLog /var/www/brep/log/error.log + CustomLog /var/www/brep/log/access.log combined + + # brep configuration + # + + # Load the brep module. + # + <IfModule !brep_module> + LoadModule brep_module /home/brep/install/libexec/brep/mod_brep.so + </IfModule> + + # Repository email. This email is used for the From: header in emails send + # by brep (for example, build failure notifications). + # + #brep-email <brep-admin-email> + + # Repository host. It specifies the scheme and the host address (but not the + # root path; see brep-root below) that will be used whenever brep needs to + # construct an absolute URL to one of its locations (for example, a link to + # a build log that is being send via email). + # + #brep-host http://<brep-hostname> + + # Repository root. This is the part of the URL between the host name and the + # start of the repository. For example, root value /pkg means the repository + # URL is http://example.org/pkg/. Specify / to use the web server root + # (e.g., http://example.org/). If using a different repository root, don't + # forget to also change Location and Alias directives below. + # + brep-root / + + <Location "/"> + SetHandler brep + + <IfModule dir_module> + DirectoryIndex disabled + DirectorySlash Off + </IfModule> + </Location> + + # Brep module configuration. If you prefer, you can paste the contents of + # this file here. However, you will need to prefix every option with + # 'brep-'. + # + brep-conf /home/brep/config/brep-module.conf + + # Static brep content (CSS files). + # + <IfModule !alias_module> + Error "mod_alias is not enabled" + </IfModule> + + # Note: trailing slashes are important! + # + Alias /@/ /home/brep/install/share/brep/www/ + + <Directory "/home/brep/install/share/brep/www"> + Require all granted + </Directory> + + # brep config override (must come after). + # + <LocationMatch "^/([0-9]|icons)(/.*)?$"> + SetHandler none + + DirectoryIndex enabled + DirectorySlash On + </LocationMatch> +</VirtualHost> + +<Directory /var/brep/bpkg/pkg/> + Options Indexes FollowSymLinks + AllowOverride None + Require all granted +</Directory> diff --git a/etc/private/install/brep-install b/etc/private/install/brep-install new file mode 100755 index 0000000..37179c2 --- /dev/null +++ b/etc/private/install/brep-install @@ -0,0 +1,479 @@ +#! /usr/bin/env bash + +# file : etc/private/install/brep-install +# license : MIT; see accompanying LICENSE file + +# Setup HTTP-only brep instance with unsigned package submission support via +# direct repository publishing (brep/handler/submit/submit-pub). +# +# NOTE: this setup should only be used in private/trusted environments. +# +# Unless the --setup option is specified, create the 'brep' group and user and +# re-run itself with the --setup option under this user. In the setup mode +# install and configure the brep instance, automating the instructions from +# the INSTALL file, including: +# +# - Build the build2 toolchain (installing it to /usr/local/) and brep +# (installing it to ~brep/install/). +# +# - Install PostgreSQL and create brep users/databases. +# +# - Installing Apache2 and configure HTTP server with the brep module. +# +# Note that the script is written for use on Debian-based distributions so you +# will need to adjust it to match other distributions or operating systems. +# +# Options: +# +# --mount +# +# Mount the virtio-9p device with id 'state' as the /var/brep directory. +# This directory is expected to either contain the pkg repository or be +# empty, in which case an empty repository will be automatically +# initialized. If this option is unspecified, the directory will be created +# in the local filesystem. +# +# --brep-user +# +# User and group ids to use when creating the 'brep' group and user. If +# unspecified, 63700 is used. +# +# --setup +# +# Install and configure the brep instance, assuming that the 'brep' user +# already exists and this script is executed as this user. +# +# --clean +# +# At the end of the brep instance setup remove installation environment- +# specific traces (host name/IP from the configuration files, etc). Normally +# you would use this option to make the "clean" machine copy for +# distribution. Note that if this option is specified, then the brep +# instance will only be unusable after the machine reboot. +# +usage="Usage: $0 [<options>]" + +# build2 toolchain repository certificate fingerprint. Note: this is a +# repository the toolchain installation script downloads the build2 packages +# from. +# +toolchain_repo_cert_fp="70:64:FE:E4:E0:F3:60:F1:B4:51:E1:FA:12:5C:E0:B3:DB:DF:96:33:39:B9:2E:E5:C2:68:63:4C:A6:47:39:43" +#toolchain_repo_cert_fp="EC:50:13:E2:3D:F7:92:B4:50:0B:BF:2A:1F:7D:31:04:C6:57:6F:BC:BE:04:2E:E0:58:14:FA:66:66:21:1F:14" + +# brep package repository URL and certificate fingerprint. +# +#brep_repo_url="https://pkg.cppget.org/1/alpha" +#brep_repo_cert_fp="70:64:FE:E4:E0:F3:60:F1:B4:51:E1:FA:12:5C:E0:B3:DB:DF:96:33:39:B9:2E:E5:C2:68:63:4C:A6:47:39:43" +brep_repo_url="https://stage.build2.org/1" +brep_repo_cert_fp="EC:50:13:E2:3D:F7:92:B4:50:0B:BF:2A:1F:7D:31:04:C6:57:6F:BC:BE:04:2E:E0:58:14:FA:66:66:21:1F:14" + +owd=`pwd` +trap "{ exit 1; }" ERR +trap "{ cd $owd; }" EXIT +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "error: $*"; exit 1; } + +# Trace a command line, quoting empty arguments as well as those that contain +# spaces. +# +function trace () # <cmd> <arg>... +{ + local s="+" + while [ "$#" -gt 0 ]; do + if [ -z "$1" -o -z "${1##* *}" ]; then + s="$s \"$1\"" + else + s="$s $1" + fi + + shift + done + + info "$s" +} + +# Trace and run a command. +# +run () # <args>... +{ + trace "$@" + "$@" +} + +# The chosen fixed id for the 'brep' user. Note: must match the id of the +# 'brep' user on the host. +# +# Note that Linux assigns the [0 99] range for the statically allocated system +# users and [100 499] -- for dynamic allocations by administrators and post- +# install scripts. Debian, in turn, assigns the [100 999] range for the +# dynamically allocated system users and [60000 64999] -- for statically +# allocated on demand "obscure package users". +# +brep_id=63700 # Update the README file on change. + +# Parse the command line options and, while at it, compose the options array +# for potential re-execution as the 'brep' user. +# +mount= +setup= +clean= +ops=() + +while [ "$#" -gt 0 ]; do + case "$1" in + --mount) + mount=true + ops+=("$1") + shift + ;; + --brep-user) + shift + brep_id="$1" + shift + ;; + --setup) + setup=true + shift + ;; + --clean) + clean=true + ops+=("$1") + shift + ;; + *) + break # The end of options is encountered. + ;; + esac +done + +if [ "$#" -ne 0 ]; then + error "$usage" +fi + +scr_exe="$(realpath "${BASH_SOURCE[0]}")" +scr_dir="$(dirname "$scr_exe")" + +# Unless we are not in the setup mode, non-interactively add the 'brep' +# user/group and re-execute the script in the setup mode as this user. +# +if [ ! "$setup" ]; then + run sudo addgroup --gid "$brep_id" brep + + run sudo adduser --uid "$brep_id" --gid "$brep_id" --disabled-password \ + --gecos "" brep + + run sudo tee -a /etc/sudoers.d/brep >/dev/null <<EOF +brep ALL=(ALL) NOPASSWD:ALL +EOF + + run sudo chmod 0440 /etc/sudoers.d/brep + + # Use --session-command rather than --command|-c to make sure that when the + # su program receives SIGINT (Ctrl-C) it kills not just its child process + # but also all its descendants. + # + # Note: here we rely on ops to not contain spaces or be empty. + # + run exec sudo su -l brep --session-command "'$scr_exe' --setup ${ops[*]}" +fi + +# Here we assume that we are executed as brep user. +# +run cd "$HOME" + +# Mount the brep state directory, if requested. Note that otherwise, the +# directory will be created later, in the local filesystem by the brep-startup +# script. +# +if [ "$mount" ]; then + run sudo mkdir -p /var/brep + + run sudo tee -a /etc/fstab >/dev/null <<EOF +state /var/brep 9p trans=virtio,version=9p2000.L,posixacl,cache=none,_netdev 0 0 +EOF + + run sudo mount -a +fi + +# Install the prerequisite binary packages. +# +run sudo apt-get --yes update +run sudo apt-get --yes install --no-install-recommends g++ +run sudo apt-get --yes install --no-install-recommends postgresql postgresql-contrib libpq-dev +run sudo apt-get --yes install --no-install-recommends apache2 libapr1-dev libapreq2-dev apache2-dev +run sudo apt-get --yes install --no-install-recommends acl rsync +run sudo apt-get clean + +# Install build2 toolchain. +# +run mkdir build2-build +run cd build2-build + +# Look for the toolchain installation script in this script directory. +# +run cp "$(echo "$scr_dir"/build2-install-*.sh)" . +run sh ./build2-install-*.sh --no-check --yes --trust "$toolchain_repo_cert_fp" +#run sh ./build2-install-*.sh --no-check --yes --local + +run cd .. # Back to brep home. + +# Grant Apache2 read access to the module and configuration. +# +run setfacl -m g:www-data:rx "$HOME" +run setfacl -dm g:www-data:rx "$HOME" + +# Install brep. +# +run mkdir brep +run cd brep + +run bpkg create \ + cc \ + config.cc.coptions="-O3" \ + config.cc.poptions="-I$(apxs -q includedir)" \ + config.bin.lib=shared \ + config.bin.rpath="$HOME/install/lib" \ + config.install.root="$HOME/install" + +run bpkg add "$brep_repo_url" +run bpkg fetch --trust "$brep_repo_cert_fp" +run bpkg build --yes brep ?sys:libapr1 ?sys:libapreq2 ?sys:libpq +run bpkg install brep + +run cd .. # Back to brep home. + +# Create PostgreSQL user and databases. +# +# Note that while we could probably omit the build-related setup, let's keep +# it to stay close to the instructions in the INSTALL file and to simplify the +# potential future configuration of the brep instance as a build2 build bot +# controller. +# +run sudo sudo -u postgres psql <<EOF +CREATE DATABASE brep_package +TEMPLATE template0 +ENCODING 'UTF8' +LC_COLLATE 'en_US.UTF8' +LC_CTYPE 'en_US.UTF8'; + +CREATE DATABASE brep_build +TEMPLATE template0 +ENCODING 'UTF8' +LC_COLLATE 'en_US.UTF8' +LC_CTYPE 'en_US.UTF8'; + +CREATE USER brep; + +GRANT ALL PRIVILEGES ON DATABASE brep_package, brep_build TO brep; + +CREATE USER "www-data" INHERIT IN ROLE brep; + +CREATE USER "brep-build" INHERIT IN ROLE brep PASSWORD '-'; + +\c brep_package +GRANT ALL PRIVILEGES ON SCHEMA public TO brep; + +\c brep_build +GRANT ALL PRIVILEGES ON SCHEMA public TO brep; +EOF + +# Create the "staging" package database for the submit-pub package submission +# handler. +# +run sudo sudo -u postgres psql <<EOF +CREATE DATABASE brep_submit_package +TEMPLATE template0 +ENCODING 'UTF8' +LC_COLLATE 'en_US.UTF8' +LC_CTYPE 'en_US.UTF8'; + +GRANT ALL PRIVILEGES ON DATABASE brep_submit_package TO brep; + +\c brep_submit_package +GRANT ALL PRIVILEGES ON SCHEMA public TO brep; +EOF + +# Make sure the 'brep' and Apache2 user's logins work properly. +# +q="SELECT current_database();" +run psql -d brep_package -c "$q" >/dev/null +run psql -d brep_build -c "$q" >/dev/null +run psql -d brep_submit_package -c "$q" >/dev/null + +run sudo sudo -u www-data psql -d brep_package -c "$q" >/dev/null +run sudo sudo -u www-data psql -d brep_build -c "$q" >/dev/null + +# Setup the connection between the databases. +# +run sudo sudo -u postgres psql -d brep_build <<EOF +CREATE EXTENSION postgres_fdw; + +CREATE SERVER package_server +FOREIGN DATA WRAPPER postgres_fdw +OPTIONS (dbname 'brep_package', updatable 'true'); + +GRANT USAGE ON FOREIGN SERVER package_server to brep; + +CREATE USER MAPPING FOR PUBLIC +SERVER package_server +OPTIONS (user 'brep-build', password '-'); +EOF + +# Allow brep-build user to access the brep_package database. +# +f="$(run sudo sudo -u postgres psql -t -A -c "show hba_file;")" +s="# TYPE DATABASE USER ADDRESS METHOD\nlocal brep_package brep-build md5\n\n" + +run sudo sed --in-place=.bak "1s/^/$s/" "$f" +run sudo systemctl restart postgresql + +# Enable creating database tables with columns of the case-insensitive +# character string type. +# +q="CREATE EXTENSION citext;" +run sudo sudo -u postgres psql -d brep_package <<<"$q" +run sudo sudo -u postgres psql -d brep_build <<<"$q" +run sudo sudo -u postgres psql -d brep_submit_package <<<"$q" + +# Copy the brep module configuration. +# +# Note: must be done before bin/brep-startup execution, which adjusts the +# configuration. +# +run mkdir config +run cp "$scr_dir/brep-module.conf" config/ + +# Initialize the brep private instance, in particular creating the database +# schemas and running the brep loader. +# +run mkdir bin/ +run cp "$scr_dir/brep-startup" bin/ +run bin/brep-startup + +# Smoke test the database schemas. +# +run psql -d brep_package -c 'SELECT canonical_name, summary FROM repository' >/dev/null +run psql -d brep_build -c 'SELECT package_name FROM build' >/dev/null +run psql -d brep_build -c 'SELECT DISTINCT name FROM build_package' >/dev/null +run psql -d brep_submit_package -c 'SELECT canonical_name, summary FROM repository' >/dev/null + +# Setup executing the brep-startup script on boot. +# +run sudo cp "$scr_dir/brep-startup.service" /etc/systemd/system/ + +run sudo systemctl start brep-startup.service # Make sure there are no issues. +run sudo systemctl enable brep-startup.service + +# Prepare directories for the package submission service. +# +run mkdir submit-data +run mkdir submit-temp +run setfacl -m g:www-data:rwx submit-data +run setfacl -m g:www-data:rwx submit-temp + +# Make the Apache2 user owned directories fully accessible by the 'brep' user +# (which the submit-pub submission handler will run as). +# +run setfacl -dm g:brep:rwx submit-data +run setfacl -dm g:brep:rwx submit-temp + +# Add the Apache2 user to sudoers, so the submission handler can re-execute +# itself as the 'brep' user. +# +run sudo tee -a /etc/sudoers.d/www-data >/dev/null <<EOF +www-data ALL=(ALL) NOPASSWD:ALL +EOF + +run sudo chmod 0440 /etc/sudoers.d/www-data + +# Setup the Apache2 module. +# +run sudo mkdir -p /var/www/brep/log/ + +run sudo cp "$scr_dir/brep-apache2.conf" /etc/apache2/sites-available/000-brep.conf +run sudo cp "$scr_dir/brep-logrotate" /etc/logrotate.d/brep + +run sudo a2dissite --purge -q 000-default +run sudo a2ensite -q 000-brep + +run sudo systemctl restart apache2 +run sudo systemctl status apache2 >/dev/null + +# Make sure the Apache2 service depends on PostgreSQL and +# brep-startup.service, so that they are started in proper order. +# +run sudo mkdir -p /etc/systemd/system/apache2.service.d/ +run sudo tee /etc/systemd/system/apache2.service.d/postgresql.conf >/dev/null <<EOF +[Unit] +Requires=postgresql.service +After=postgresql.service +EOF + +run sudo tee /etc/systemd/system/apache2.service.d/brep-startup.conf >/dev/null <<EOF +[Unit] +Requires=brep-startup.service +After=brep-startup.service +EOF + +run sudo mkdir -p /etc/systemd/system/postgresql.service.d/ +run sudo tee /etc/systemd/system/postgresql.service.d/apache2.conf >/dev/null <<EOF +[Unit] +Wants=apache2.service +EOF + +run sudo systemctl daemon-reload + +# Verify that Apache2 is stopped after PostgreSQL is stopped. +# +run sudo systemctl stop postgresql + +ec="0" +run sudo systemctl status apache2 >/dev/null || ec="$?" + +if [ "$ec" -ne 3 ]; then + error "exit code 3 (unit is not active) is expected instead of $ec" +fi + +# Verify that Apache2 is started after PostgreSQL is started. +# +run sudo systemctl start postgresql + +run sleep 3 +run sudo systemctl status apache2 >/dev/null + +# Setup periodic loader execution. +# +run sudo cp "$scr_dir/brep-load.service" /etc/systemd/system/ +run sudo cp "$scr_dir/brep-load.timer" /etc/systemd/system/ + +run sudo systemctl start brep-load.service # Make sure there are no issues. + +run sudo systemctl start brep-load.timer +run sudo systemctl status brep-load.timer >/dev/null +run sudo systemctl enable brep-load.timer +run sudo systemctl status brep-load.timer >/dev/null + +# Cleanup the installation environment-specific traces, if requested. +# +if [ "$clean" ]; then + + # Stop the relevant services. + # + run sudo systemctl stop brep-load.timer + run sudo systemctl stop apache2 + + # Remove the host name/IP from the configuration. + # + run cp "$scr_dir/brep-module.conf" config/ # Adjusted by brep-startup. + run rm config/loadtab # Recreated by brep-startup. + + # Finally, stop networking and cleanup the DHCP lease information. + # + # Note that after networking is stopped, sudo prints the 'unable to resolve + # host' diagnostics while trying to obtain the host IP. Thus, we execute the + # last two commands via a single sudo call. + # + run sudo bash -c "systemctl stop networking && rm -rf /var/lib/dhcp/*.leases" +fi diff --git a/etc/private/install/brep-load.service b/etc/private/install/brep-load.service new file mode 100644 index 0000000..272a124 --- /dev/null +++ b/etc/private/install/brep-load.service @@ -0,0 +1,11 @@ +[Unit] +Description=brep repository loader service + +[Service] +Type=oneshot +User=brep +Group=brep +ExecStart=/home/brep/install/bin/brep-load /home/brep/config/loadtab + +[Install] +WantedBy=default.target diff --git a/etc/private/install/brep-load.timer b/etc/private/install/brep-load.timer new file mode 100644 index 0000000..1bf78c4 --- /dev/null +++ b/etc/private/install/brep-load.timer @@ -0,0 +1,33 @@ +[Unit] +Description=brep repository loader timer +RefuseManualStart=no +RefuseManualStop=no + +# Note that due to brep-startup service's oneshot type, this unit won't be +# started until the brep-startup process exits successfully. +# +# Also note that if brep-startup fails and is restarted manually, similar to +# services, the timer is not started automatically. Instead, it has to be +# started manually with `systemctl start brep-load.timer`. +# +Requires=brep-startup.service +After=brep-startup.service + +[Timer] +Unit=brep-load.service + +# Don't keep track of the timer across reboots. +# +Persistent=false + +# Start the timer for the first time. +# +OnBootSec=1 + +# Then wait 4-5 seconds until the next run. +# +OnUnitInactiveSec=4 +AccuracySec=1 + +[Install] +WantedBy=timers.target diff --git a/etc/private/install/brep-logrotate b/etc/private/install/brep-logrotate new file mode 100644 index 0000000..67c7c90 --- /dev/null +++ b/etc/private/install/brep-logrotate @@ -0,0 +1,20 @@ +/var/www/brep/log/*.log { + weekly + missingok + rotate 4 + compress + delaycompress + notifempty + create 640 root adm + sharedscripts + postrotate + if /etc/init.d/apache2 status > /dev/null ; then \ + /etc/init.d/apache2 reload > /dev/null; \ + fi; + endscript + prerotate + if [ -d /etc/logrotate.d/httpd-prerotate ]; then \ + run-parts /etc/logrotate.d/httpd-prerotate; \ + fi; \ + endscript +} diff --git a/etc/private/install/brep-module.conf b/etc/private/install/brep-module.conf new file mode 100644 index 0000000..bfaa8f6 --- /dev/null +++ b/etc/private/install/brep-module.conf @@ -0,0 +1,532 @@ +# Configuration file for the brep module (note: this is not an apache2 .conf +# file but it can be converted to one by prefixing all the options with +# brep-). See brep(1) for detailed description of each configuration option. +# Commented out options indicate their default values. +# +# Besides being parsed by the brep module, this file may also be parsed by +# brep utilities that are normally only interested in the subset of the +# options. To simplify skipping of unrecognized, this file must always have an +# option name and its value on the same line. +# + +# Package search page title. It is placed inside XHTML5 <title> element. +# +# search-title Packages + + +# Package search page description. If specified, it is displayed before the +# search form on the first page only. The value is treated as an XHTML5 +# fragment. +# +# search-description "" + + +# Web page logo. It is displayed in the page header aligned to the left edge. +# The value is treated as an XHTML5 fragment. +# +# logo "" + + +# Web page menu. Each entry is displayed in the page header in the order +# specified and aligned to the right edge. A link target that starts with '/' +# or contains ':' is used as is. Otherwise, it is prefixed with the repository +# web interface root. +# +menu Packages= +# menu Builds=?builds +# menu Configs=?build-configs +menu Submit=?submit +# menu CI=?ci +menu About=?about + + +# Number of packages per page. +# +# search-page-entries 20 + + +# Number of pages in navigation (pager). +# +# search-pages 5 + + +# Number of package description characters to display in brief pages. +# +# package-description 500 + + +# Number of package changes characters to display in brief pages. +# +# package-changes 5000 + + +# The package database connection configuration. By default, brep will try to +# connect to the local instance of PostgreSQL with the operating system- +# default mechanism (Unix-domain socket, etc) and use operating system +# (login) user name and the database called 'brep_package'. If the role name +# is not empty then the login user will be switched (with SET ROLE) to this +# user prior to executing any statements. If not specified, then 'brep' is +# used. See brep(1) for details. +# +# package-db-user +# package-db-role brep +# package-db-password +# package-db-name brep_package +# package-db-host +# package-db-port + + +# The maximum number of concurrent package database connections per web server +# process. If 0, then no limitation is applied. +# +# package-db-max-connections 5 + + +# The maximum number of times to retry package database transactions in the +# face of recoverable failures (deadlock, loss of connection, etc). +# +# package-db-retry 10 + + +# Build configuration file. If not specified (default), then the package +# building functionality will be disabled. If specified, then the build +# database must be configured (see next). Note: must be an absolute path. +# +# build-config + + +# Number of build configurations per page. +# +# build-config-page-entries 20 + + +# Number of pages in navigation (pager). +# +# build-config-pages 5 + + +# Directory containing build bot agent public keys. If specified, then brep +# will perform agent authentication and will reject build results from +# unauthenticated ones. If not specified, then build results are accepted from +# all agents (which will be a security risk if the brep instance is publicly +# accessible). +# +# The directory is expected to contain one PEM-encoded public key per file with +# the .pem extension. All other files and subdirectories are ignored. The brep +# instance needs to be restarted after adding new key files for the changes to +# take effect. +# +# build-bot-agent-keys + + +# Regular expressions in the /<regex>/<replacement>/ form for transforming the +# interactive build login information, for example, into the actual command +# that can be used by the user. The regular expressions are matched against +# the "<agent> <interactive-login>" string containing the respective task +# request manifest values. The first matching expression is used for the +# transformation. If no expression matches, then the task request is +# considered invalid, unless no expressions are specified. Repeat this option +# to specify multiple expressions. +# +# build-interactive-login + + +# Order in which packages are considered for build. The valid values are +# 'stable' and 'random'. If not specified, then 'stable' is assumed. Note that +# interactive builds are always preferred. +# +#build-package-order stable + + +# Number of builds per page. +# +# build-page-entries 20 + + +# Number of pages in navigation (pager). +# +# build-pages 5 + + +# Time to wait before considering a package for a forced rebuild. Must be +# specified in seconds. Default is 10 minutes. +# +# build-forced-rebuild-timeout 600 + + +# Time to wait before considering a package for a soft rebuild (only to be +# performed if the build environment or any of the package dependencies have +# changed). Must be specified in seconds. The special zero value disables soft +# rebuilds. Default is 24 hours. +# +# build-soft-rebuild-timeout 86400 + + +# Alternative package soft rebuild timeout to use instead of the soft rebuild +# timeout (see the build-soft-rebuild-timeout option for details) during the +# specified time interval. Must be specified in seconds. Default is the time +# interval length plus (build-soft-rebuild-timeout - 24h) if soft rebuild +# timeout is greater than 24 hours (thus the rebuild is only triggered within +# the last 24 hours of the build-soft-rebuild-timeout expiration). +# +# The alternative rebuild timeout can be used to "pull" the rebuild window to +# the specified time of day, for example, to optimize load and/or power +# consumption of the build infrastructure (off-work hours, solar, off-peak +# electricity tariffs, etc). A shorter than the time interval rebuild timeout +# can also be used to force continuous rebuilds, for example, to shake out +# flaky tests. Note also that if the alternative rebuild timeout is greater +# than the normal rebuild timeout, then this will result in slower rebuilds +# during the alternative time interval. In this case, if the build +# infrastructure is monitored for delayed package builds, then the alternative +# rebuild timeout should only be made slightly greater than the normal timeout +# (see brep-monitor(1) for details). +# +# The time interval boundaries must be specified as times of day (in the local +# timezone) in the <hours>:<minutes> form. If the stop time is less than the +# start time then the interval extends through midnight. The start and stop +# times must both be either specified or absent. If unspecified, then no +# alternative rebuild timeout will be used. +# +# build-alt-soft-rebuild-timeout +# build-alt-soft-rebuild-start +# build-alt-soft-rebuild-stop + + +# Time to wait before considering a package for a hard rebuild (to be +# performed unconditionally). Must be specified in seconds. The special zero +# value disables hard rebuilds. Default is 7 days. +# +# build-hard-rebuild-timeout 604800 + + +# Alternative package hard rebuild timeout. The semantics is the same as for +# the build-alt-soft-rebuild-* options but for the build-hard-rebuild-timeout +# option. +# +# build-alt-hard-rebuild-timeout +# build-alt-hard-rebuild-start +# build-alt-hard-rebuild-stop + + +# Time to wait before assuming the 'queued' notifications are delivered for +# package CI requests submitted via third-party services (GitHub, etc). During +# this time a package is not considered for a build. Must be specified in +# seconds. Default is 30 seconds. +# +# build-queued-timeout 30 + + +# The maximum size of the build task request manifest accepted. Note that the +# HTTP POST request body is cached to retry database transactions in the face +# of recoverable failures (deadlock, loss of connection, etc). Default is +# 100K. +# +# build-task-request-max-size 102400 + + +# Time to wait before considering the expected task result lost. Must be +# specified in seconds. Default is 3 hours. +# +# build-result-timeout 10800 + + +# The maximum size of the build result manifest accepted. Note that the HTTP +# POST request body is cached to retry database transactions in the face of +# recoverable failures (deadlock, loss of connection, etc). Default is 10M. +# +# build-result-request-max-size 10485760 + + +# Enable or disable package build notification emails in the <name>=<mode> +# form. The valid <mode> values are 'none', 'latest', and 'all'. If 'all' is +# specified for a toolchain name, then emails are sent according to the +# build-*email package manifest values when all versions of a package are +# built with this toolchain. If 'latest' is specified, then for this toolchain +# name the emails are only sent for the latest version of a package. If 'none' +# is specified, then no emails are sent for this toolchain name. By default +# the 'latest' mode is assumed. Repeat this option to enable/disable emails +# for multiple toolchains. +# +# build-toolchain-email <toolchain-name>=latest|none|all + + +# The build database connection configuration. By default, brep will try to +# connect to the local instance of PostgreSQL with the operating system-default +# mechanism (Unix-domain socket, etc) and use operating system (login) user +# name and the database called 'brep_build'. If the role name is not empty +# then the login user will be switched (with SET ROLE) to this user prior +# to executing any statements. If not specified, then 'brep' is used. See +# brep(1) for details. +# +# build-db-user +# build-db-role brep +# build-db-password +# build-db-name brep_build +# build-db-host +# build-db-port + + +# The maximum number of concurrent build database connections per web server +# process. If 0, then no limitation is applied. +# +# build-db-max-connections 5 + + +# The maximum number of times to retry build database transactions in the +# face of recoverable failures (deadlock, loss of connection, etc). +# +# build-db-retry 10 + + +# The root directory where the uploaded binary distribution packages are +# saved to under the following directory hierarchy: +# +# [<tenant>/]<distribution>/<os-release>/<project>/<package>/<version>/<package-config> +# +# The package configuration directory symlinks that match these paths are +# mapped to web URLs based on the bindist-url value and displayed on the +# package version details page. If this option is specified, then bindist-url +# must be specified as well." +# +# bindist-root + + +# The root URL of the directory specified with the bindist-root option. This +# option must be specified if bindist-root is specified. +# +# bindist-url + + +# The openssl program to be used for crypto operations. You can also specify +# additional options that should be passed to the openssl program with +# openssl-option. If the openssl program is not explicitly specified, then brep +# will use openssl by default. +# +# openssl openssl + + +# Additional option to be passed to the openssl program (see openssl for +# details). Repeat this option to specify multiple openssl options. +# +# openssl-option + + +# Environment variable to be set (<name>=<value>) or unset (just <name>) for +# the openssl program (see openssl for details). Repeat this option to specify +# multiple openssl variables. Note that unspecified variables are inherited +# from the web server process. +# +# You need to at least set the RANDFILE environment variable to change the +# default location of the openssl program seed file and maybe also the +# OPENSSL_CONF variable if you would like to use a custom openssl configuration +# file. +# +# openssl-envvar RANDFILE=/home/brep/www-data-openssl.rnd +# openssl-envvar OPENSSL_CONF=/home/brep/www-data-openssl.cnf +# +# To create www-data-openssl.rnd with suitable permissions, run (as user brep): +# +# $ touch www-data-openssl.rnd +# $ setfacl -b -m g:www-data:rw www-data-openssl.rnd +# + + +# The directory to save final submission data to. If unspecified, the package +# submission functionality will be disabled. If specified, then submit-temp +# must be specified as well. +# +# Note that the directory path must be absolute and the directory itself must +# exist and have read, write, and execute permissions granted to the user that +# runs the web server. +# +submit-data /home/brep/submit-data + + +# The directory to save temporary submission data to. Must be specified if the +# package submission functionality is enabled. +# +# Note that this directory must be on the same filesystem and satisfy the same +# requirements as submit-data. It is also the user's responsibility to clean +# it up after an unclean web server shutdown. +# +submit-temp /home/brep/submit-temp + + +# The maximum size of the submission data accepted. Note that currently the +# entire submission request is read into memory. Default is 10M. +# +# 100M. +# +submit-max-size 104857600 + + +# The package submission form fragment. If specified, then its contents are +# treated as an XHTML5 fragment that is inserted into the <body> element of +# the submission page. If unspecified, then no submission page will be +# displayed. Note that the file path must be absolute. +# +submit-form /home/brep/install/share/brep/www/submit.xhtml + + +# The package submission email. If specified, the submission request and +# result manifests will be sent to this address. +# +# submit-email + + +# The handler program to be executed on package submission. The handler is +# executed as part of the HTTP request and is passed additional arguments that +# can be specified with submit-handler-argument followed by the absolute path +# to the submission directory. Note that the program path must be absolute. +# +submit-handler /home/brep/install/bin/brep-submit-pub + + +# Additional arguments to be passed to the submission handler program (see +# submit-handler for details). Repeat this option to specify multiple +# arguments. +# +submit-handler-argument --user +submit-handler-argument brep +submit-handler-argument --result-url +submit-handler-argument http://unknown +submit-handler-argument /home/brep/install/bin/brep-load +submit-handler-argument --db-name=brep_submit_package +submit-handler-argument /var/brep/bpkg/pkg + + +# The handler program timeout in seconds. If specified and the handler does +# not exit in the alloted time, then it is killed and its termination is +# treated as abnormal. +# +submit-handler-timeout 120 + + +# The directory to save CI request data to. If unspecified, the package CI +# functionality will be disabled. +# +# Note that the directory path must be absolute and the directory itself must +# exist and have read, write, and execute permissions granted to the user that +# runs the web server. +# +# ci-data + + +# The package CI form fragment. If specified, then its contents are treated as +# an XHTML5 fragment that is inserted into the <body> element of the CI page. +# If unspecified, then no CI page will be displayed. Note that the file path +# must be absolute. +# +# ci-form + + +# The package CI email. If specified, the CI request and result manifests will +# be sent to this address. +# +# ci-email + + +# The handler program to be executed on CI request. The handler is executed as +# part of the HTTP request and is passed additional arguments that can be +# specified with ci-handler-argument followed by the absolute path to the CI +# request directory. Note that the program path must be absolute. +# +# ci-handler + + +# Additional arguments to be passed to the CI handler program (see ci-handler +# for details). Repeat this option to specify multiple arguments. +# +# ci-handler-argument + + +# The CI handler program timeout in seconds. If specified and the handler does +# not exit in the allotted time, then it is killed and its termination is +# treated as abnormal. +# +# ci-handler-timeout + + +# The directory to save upload data to for the specified upload type. If +# unspecified, the build artifacts upload functionality will be disabled for +# this type. +# +# Note that the directory path must be absolute and the directory itself must +# exist and have read, write, and execute permissions granted to the user that +# runs the web server. +# +# upload-data <type>=<dir> + + +# The maximum size of the upload data accepted for the specified upload type. +# Note that currently the entire upload request is read into memory. The +# default is 10M. +# +# upload-max-size <type>=10485760 + + +# The build artifacts upload email. If specified, the upload request and +# result manifests will be sent to this address. +# +# upload-email <type>=<email> + + +# The handler program to be executed on build artifacts upload of the +# specified type. The handler is executed as part of the HTTP request and is +# passed additional arguments that can be specified with +# upload-handler-argument followed by the absolute path to the upload +# directory (upload-data). Note that the program path must be absolute. +# +# upload-handler <type>=<path> + + +# Additional arguments to be passed to the upload handler program for the +# specified upload type (see upload-handler for details). Repeat this option +# to specify multiple arguments. +# +# upload-handler-argument <type>=<arg> + + +# The upload handler program timeout in seconds for the specified upload type. +# If specified and the handler does not exit in the allotted time, then it is +# killed and its termination is treated as abnormal. +# +# upload-handler-timeout <type>=<seconds> + + +# Disable upload of the specified type for the specified toolchain name. +# Repeat this option to disable uploads for multiple toolchains. +# +# upload-toolchain-exclude <type>=<name> + + +# Disable upload of the specified type for packages from the repository with +# the specified canonical name. Repeat this option to disable uploads for +# multiple repositories. +# +# upload-repository-exclude <type>=<name> + + +# The default view to display for the global repository root. The value is one +# of the supported services (packages, builds, submit, ci, etc). Default is +# packages. +# +# root-global-view packages + + +# The default view to display for the tenant repository root. The value is one +# of the supported services (packages, builds, submit, ci, etc). Default is +# packages. +# +# root-tenant-view packages + + +# Name to call the tenant values on web pages. If not specified, then 'tenant' +# is used. +# +# tenant-name tenant + + +# Trace verbosity. Disabled by default. +# +# verbosity 0 diff --git a/etc/private/install/brep-startup b/etc/private/install/brep-startup new file mode 100755 index 0000000..780a2c0 --- /dev/null +++ b/etc/private/install/brep-startup @@ -0,0 +1,88 @@ +#! /usr/bin/env bash + +# file : etc/private/install/brep-startup +# license : MIT; see accompanying LICENSE file + +# (Re-)initialize the brep private instance, normally on the machine startup. +# +# Specifically: +# +# - Create the pkg repository and symlink to it, unless already exists. +# +# - Migrate the brep databases as a sanity check. +# +# - Adjust the brep module configuration file using the current host name/IP. +# +# - Generate the loadtab using the current host name/IP and run the loader. +# +trap "{ exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "error: $*"; exit 1; } + +# Create the pkg repository, if required. +# +d=/var/brep/bpkg + +if [ ! -L "$d/pkg" ]; then + rd="$(date "+pkg-%Y%m%d-%H%M%S-%N")" + + mkdir -p "$d/$rd/1" + ln -s "$rd" "$d/pkg" +fi + +r="$d/pkg/1" + +if [ ! -f "$r/repositories.manifest" ]; then + cat <<EOF >"$r/repositories.manifest" +: 1 +#summary: Private repository +#description: \\ +#This is a private repository. +#And this description can contain multiple lines. +#\\ +#email: admin@example.org + +#: +#role: prerequisite +#location: https://pkg.cppget.org/1/stable +#trust: ... +EOF +fi + +if [ ! -f "$r/packages.manifest" ]; then + bpkg rep-create -q "$r" +fi + +# Migrate the databases. +# +"$HOME/install/bin/brep-migrate" package +"$HOME/install/bin/brep-migrate" build +"$HOME/install/bin/brep-migrate" -n brep_submit_package package + +# Deduce the machine host name. +# +h="$(hostname -f)" +if [ "$h" == "localhost" ]; then + h="$(hostname -I | sed 's/ *$//')" # Strip the potential trailing space(s). +fi + +if [ -z "$h" ]; then + error "unable to obtain host name or IP address" +fi + +# Adjust the submission result URL host name in the brep module configuration +# file. +# +sed --in-place -re \ +"\$!N;s%^\s*(submit-handler-argument\s+--result-url\s*\\n)\ +\s*(submit-handler-argument\s+https?://)[^/]*(.*)\$%\1\2$h\3%;P;D" \ +"$HOME/config/brep-module.conf" + +# (Re-)generate the loadtab file and reload the repository. +# +f="$HOME/config/loadtab" + +echo "http://$h/1 private cache:$r" >"$f" +"$HOME/install/bin/brep-load" "$f" diff --git a/etc/private/install/brep-startup.service b/etc/private/install/brep-startup.service new file mode 100644 index 0000000..a3dc546 --- /dev/null +++ b/etc/private/install/brep-startup.service @@ -0,0 +1,17 @@ +[Unit] +Description=brep instance initialization service + +Wants=network-online.target +After=network-online.target + +Requires=postgresql.service +After=postgresql.service + +[Service] +Type=oneshot +User=brep +Group=brep +ExecStart=/home/brep/bin/brep-startup + +[Install] +WantedBy=default.target diff --git a/etc/private/install/vm-gen-service b/etc/private/install/vm-gen-service new file mode 100755 index 0000000..ae49a49 --- /dev/null +++ b/etc/private/install/vm-gen-service @@ -0,0 +1,207 @@ +#! /usr/bin/env bash + +# Generate systemd .service file for QEMU/KVM virtual machines. +# +# Normally the machines are run from a dedicated user account with its home +# directory containing all the relevant files (management scripts, images, +# configurations, and sockets). However, this can be overriden with the +# following options +# +# --home <dir> +# The virtual machines "home" directory. If unspecified, the user's home +# directory is assumed. +# +# --bin <dir> +# The virtual machines management scripts directory. If unspecified, +# <home>/bin is assumed. If specified as relative, then assumed relative +# to <home>. +# +# --etc <dir> +# The virtual machines configuration files directory. If unspecified, +# <home>/etc is assumed. If specified as relative, then assumed relative +# to <home>. +# +# --var <dir> +# The virtual machines image files directory. If unspecified, <home>/var is +# assumed. If specified as relative, then assumed relative to <home>. +# +# --run <dir> +# The virtual machines sockets directory. If unspecified, <home>/run is +# assumed. If specified as relative, then assumed relative to <home>. +# +# If <user> is unspecified, the current user is assumed. If <group> is +# unspecified, the user's primary group is assumed. +# +usage="usage: $0 [<options>] [<user>] [<group>]" + +owd="$(pwd)" +trap "{ cd '$owd'; exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +home= +bin= +etc= +var= +run= + +while [ "$#" -gt 0 ]; do + case "$1" in + --home) + shift + home="$1" + shift + ;; + --bin) + shift + bin="$1" + shift + ;; + --etc) + shift + etc="$1" + shift + ;; + --var) + shift + var="$1" + shift + ;; + --run) + shift + run="$1" + shift + ;; + *) + break + ;; + esac +done + +user="$1" + +if [ -z "$user" ]; then + user="$(id -un)" +fi + +group="$2" + +if [ -z "$group" ]; then + group="$(id -un "$user")" +fi + +if [ -z "$home" ]; then + home="$(eval echo ~$user)" +fi + +function complete_dir () # <default> <home> <dir> +{ + local r + if [ -z "$3" ]; then + r="$2/$1" + elif [ "${3:0:1}" != "/" ]; then + r="$2/$3" + else + r="$3" + fi + echo "$(realpath --no-symlinks --canonicalize-missing "$r")" +} + +bin="$(complete_dir bin "$home" "$bin")" +etc="$(complete_dir etc "$home" "$etc")" +var="$(complete_dir var "$home" "$var")" +run="$(complete_dir run "$home" "$run")" + +name="vm-$user" +file="$name@.service" + +# Thinks that must be \-escaped: +# +# - $ (including in comments) +# - \ (e.g., in line continuations) +# +cat <<EOF >"$file" +# $file -- QEMU/KVM machine service template for systemd +# +# user: $user +# group: $group +# bin: $bin +# etc: $etc +# var: $var +# run: $run +# +# To install: +# +# sudo cp $file /etc/systemd/system/ +# sudo chmod 644 /etc/systemd/system/$file +# +# cp ... $var/<machine>.img +# nano $etc/<machine>.conf # Specify RAM, CPU, TAP, MAC, etc. +# +# sudo systemctl start $name@<machine> +# sudo systemctl status $name@<machine> +# login-machine $run/<machine>-con.sock +# sudo systemctl stop $name@<machine> +# +# sudo systemctl enable $name@<machine> + +[Unit] +Description=QEMU/KVM virtual machine %I + +Wants=network-online.target +#After=network-online.target +After=multi-user.target + +[Service] +User=$user +Group=$user +UMask=0007 +WorkingDirectory=~ + +Environment=CPU=1 +Environment=RAM=2G + +# These MUST be specific in EnvironmentFile! +# +#Environment=TAP= +#Environment=MAC= + +# Note that using variable expansion in EnvironmentFile does not work (at +# least not with systemd 229). +# +EnvironmentFile=$etc/%i.conf + +# Note that the first word of ExecStart cannot contain variable expansions. +# +ExecStart=$bin/vm-start \\ + --cpu \${CPU} \\ + --ram \${RAM} \\ + --tap \${TAP} \\ + --mac \${MAC} \\ + --pid $run/%i.pid \\ + --monitor $run/%i-mon.sock \\ + --console $run/%i-con.sock \\ + $var/%i.img + +ExecStop=$bin/vm-stop $run/%i.pid $run/%i-mon.sock + +# This makes sure systemd waits for the ExecStart command to exit rather +# than killing it as soon as ExecStop exits (this is necessary since our +# vm-stop may exit just before vm-start). +# +KillMode=none +TimeoutStopSec=60 + +[Install] +WantedBy=multi-user.target +EOF + +info "generated $file for" +info " user: $user" +info " group: $group" +info " bin: $bin" +info " etc: $etc" +info " var: $var" +info " run: $run" diff --git a/etc/private/systemd-networkd/10-br0.netdev b/etc/private/systemd-networkd/10-br0.netdev new file mode 100644 index 0000000..6431ba8 --- /dev/null +++ b/etc/private/systemd-networkd/10-br0.netdev @@ -0,0 +1,8 @@ +# Create a bridge network device. +# +# Use ethernet interface's MAC address as bridge MAC. + +[NetDev] +Name=br0 +Kind=bridge +MACAddress=02:11:11:11:11:11 diff --git a/etc/private/systemd-networkd/10-tap0.netdev b/etc/private/systemd-networkd/10-tap0.netdev new file mode 100644 index 0000000..3989bd8 --- /dev/null +++ b/etc/private/systemd-networkd/10-tap0.netdev @@ -0,0 +1,12 @@ +# Create a tap network device. +# +# Set user/group to the user/group that will be using the tap +# (e.g., the user that will run the VM that will use this tap). + +[NetDev] +Name=tap0 +Kind=tap + +[Tap] +#User= +#Group= diff --git a/etc/private/systemd-networkd/20-br0-eth0.network b/etc/private/systemd-networkd/20-br0-eth0.network new file mode 100644 index 0000000..c57736f --- /dev/null +++ b/etc/private/systemd-networkd/20-br0-eth0.network @@ -0,0 +1,12 @@ +# Add the ethernet interface to the bridge. +# +# Change eth0 to your ethernet interface name. + +[Match] +Name=eth0 + +[Network] +Bridge=br0 + +[Link] +RequiredForOnline=no diff --git a/etc/private/systemd-networkd/20-br0-tap0.network b/etc/private/systemd-networkd/20-br0-tap0.network new file mode 100644 index 0000000..1c2c746 --- /dev/null +++ b/etc/private/systemd-networkd/20-br0-tap0.network @@ -0,0 +1,16 @@ +# Add the tap interface to the bridge. +# +# Note: do not assign MAC address to the tap interface, it's not the same +# thing as the interface inside the VM (which is what we want to assign the +# MAC address to). +# + +[Match] +Name=tap0 + +[Network] +Bridge=br0 +#ConfigureWithoutCarrier=yes + +[Link] +RequiredForOnline=no diff --git a/etc/private/systemd-networkd/30-br0-dhcp.network b/etc/private/systemd-networkd/30-br0-dhcp.network new file mode 100644 index 0000000..211d870 --- /dev/null +++ b/etc/private/systemd-networkd/30-br0-dhcp.network @@ -0,0 +1,17 @@ +# Configure the bridge with IPv4 DHCP. + +[Match] +Name=br0 + +[Network] +DHCP=ipv4 +IPForward=yes + +[DHCPv4] +#UseHostname=yes + +#SendHostname=yes +#Hostname=example.lan + +[Link] +RequiredForOnline=yes diff --git a/etc/private/systemd-networkd/README b/etc/private/systemd-networkd/README new file mode 100644 index 0000000..48bb7cd --- /dev/null +++ b/etc/private/systemd-networkd/README @@ -0,0 +1,106 @@ +This directory contains sample configuration files for setting up a bridge +(br0) and a permanent tap interface (tap0) using systemd's networkd network +manager. The tap interface can be used, for example, to run a virtual machine +that appears as a real machine on the host's Ethernet network. + +Assumptions: + + - The host uses Ethernet for networking. + + - The host uses IPv4 DHCP for network configuration. + +Note: only perform the following steps over a physical login to the host since +the configuration involves bringing the host's networking down. + +Note: commands that start with the `#` prompt must be executed as root. + +1. Switch to systemd-networkd for network configuration. + +Overall, the goal of this step is to disable the currently used network +manager and enable systemd-networkd. First check if systemd-networkd is +already used: + +# systemctl status systemd-networkd + +If it's enabled and running, skip to step 2. Otherwise, identify the currently +used network manager. The possible options depend on the distribution used so +consult the relevant documentation for details. One common option is the GNOME +network manager: + +# systemctl status NetworkManager + +If it's enabled and running, stop and disable: + +# systemctl stop NetworkManager +# systemctl disable NetworkManager + +For Debian-based distributions a common approach is to define the network +configuration in the /etc/network/interfaces file. To disable this method, +perform the following steps: + +# systemctl stop networking +# mv /etc/network/interfaces /etc/network/interfaces.disabled + +Once the current network manager is disabled, proceed to step 2. + + +2. Configure bridged networking using systemd-networkd. + +Copy configuration files found in this directory to /etc/systemd/network/ (see +the comment at the beginning of each file for its purpose): + +# cp *.netdev *.network /etc/systemd/network/ + +Note: if you are already using systemd-networkd, then you may already have +some configuration in /etc/systemd/network/. If the existing configuration +conflicts with this setup (for example, you already have a configuration for +the Ethernet interface), then you will need to remove the relevant files. + +Then adjust the following to match your setup: + + - Ethernet interface name if not eth0: 20-br0-eth0.network (both name and + content) + + Use the following command to list all network interfaces: + + # ip link show + + - Bridge MAC address: 10-br0.netdev + + Use your Ethernet interface's address as your bridge address, which + you can obtain with: + + # ip link show eth0 + + - Tap user/group: 10-tap0.netdev + + For example, set to the user/group that will run the VM that will use this + tap interface. + + +3. Test and enable networking using systemd-networkd. + +Once the configuration is complete, start/restart systemd-networkd and verify +networking is configured correctly. + +# systemctl restart systemd-networkd +# systemctl status systemd-networkd +# ip addr show br0 +# ip addr show tap0 + +Specifically, verify that: + + - The br0 MAC address is correct. + + - The br0 interface is assigned (via DHCP) an IP address and, if a fixed + IP is used, it's what's expected. + + - Try to ping example.org to confirm the overall network (routing, DNS) + is functional. + +If everything looks good, enable systemd-networkd: + +# systemctl enable systemd-networkd + +You may also want to reboot the host and performs the above verifications +one more time. diff --git a/etc/private/vm-gen-macaddress b/etc/private/vm-gen-macaddress new file mode 100755 index 0000000..c13a993 --- /dev/null +++ b/etc/private/vm-gen-macaddress @@ -0,0 +1,60 @@ +#! /usr/bin/env bash + +# Generate a locally administered MAC address (LAA) number <num> based on the +# specified universally administered address <mac> (UAA, for example, an +# address corresponding to the host's physical Ethernet interface). +# +# Specifically, the resulting address is formed by combining the +# LAA-conforming first octet with the subsequent five octets from <mac>: +# +# x[26ae]:xx:xx:xx:xx:xx +# +# The first octet is derived from <num> as follows: +# +# 0-15 : 02-f2 +# 16-31 : 06-f6 +# 32-47 : 0a-fa +# 48-63 : 0e-fe +# +# For example, <num> can correspond to the interface number, such as tap0, for +# which the resulting MAC address will be used. +# +usage="usage: $0 <mac> <num>" + +owd="$(pwd)" +trap "{ cd '$owd'; exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +if [ -z "$1" ]; then + error "$usage" +fi + +o='[0-9a-fA-F]' +mac="$(sed -nr -e "s/^$o$o:($o$o:$o$o:$o$o:$o$o:$o$o)$/\1/p" <<<"$1")" + +if [ -z "$mac" ]; then + error "invalid MAC address '$1'" +fi + +if [ -z "$2" ]; then + error "$usage" +fi + +num="$2" + +if (( num < 0 || num > 63 )); then + error "number '$num' is out of 0-63 range" +fi + +if (( num < 16 )); then + printf "%x2:%s\n" $(( num )) "$mac" +elif (( num < 32 )); then + printf "%x6:%s\n" $(( num - 16 )) "$mac" +elif (( num < 48 )); then + printf "%xa:%s\n" $(( num - 32 )) "$mac" +else + printf "%xe:%s\n" $(( num - 48 )) "$mac" +fi diff --git a/etc/private/vm-login b/etc/private/vm-login new file mode 100755 index 0000000..28e8864 --- /dev/null +++ b/etc/private/vm-login @@ -0,0 +1,33 @@ +#! /usr/bin/env bash + +# Get virtual machine console (using screen). +# +# Note: use Ctrl-a k to exit screen (or Ctrl-a a k if running inside screen). +# +usage="usage: $0 <console-socket>" + +trap "{ exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +con="$1" + +if [ -z "$con" ]; then + error "missing console socket" +fi + +pty="$(dirname "$con")/$(basename -s .sock "$con").pty" + +socat "UNIX-CONNECT:$con" "PTY,link=$pty" & +pid="$!" + +# Hack around terminal permission issue when running under `su - <user>`. +# +script -q -c "screen $pty" /dev/null + +# Note: socat may have already terminated (e.g., VM was shut down). +# +kill "$pid" 2>/dev/null || true +wait diff --git a/etc/private/vm-start b/etc/private/vm-start new file mode 100755 index 0000000..41c4247 --- /dev/null +++ b/etc/private/vm-start @@ -0,0 +1,98 @@ +#! /usr/bin/env bash + +# file : etc/private/vm-start +# license : MIT; see accompanying LICENSE file + +# Start the brep virtual machine (VM) for installing or running the previously +# installed brep private instance. Must be executed on the host as brep user. +# +# Share with the VM the brep state directory as the 9p filesystem with the +# passthrough security model enabled. This directory is expected to be owned +# by the brep user and either contain the pkg repository maintained by the +# brep instance or be empty, in which case the empty repository will be +# automatically initialized. +# +# Note that you can signal to the VM to regenerate the repository on startup +# (e.g., after package removal) by removing the packages.manifest file from +# the repository. +# +# Options: +# +# --state <dir> +# +# State directory to share with the VM. If unspecified, $HOME/state is +# assumed. +# +# --install <dir> +# +# Also share with the VM the install directory (contains the brep private +# instance installation script and auxiliary files). +# +# Note that this script wraps the generic vm-start-base script and passes +# through any arguments that follows these options to that script. +# +usage="usage: $0 [<options>] [<base-options>] <machine-img> [<extra-qemu-options>]" + +trap "{ exit 1; }" ERR +set -o errtrace # Trap ERR in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +install= +state="$HOME/state" + +while [ "$#" -gt 0 ]; do + case "$1" in + --install) + shift + install="${1%/}" + shift + ;; + --state) + shift + state="${1%/}" + shift + ;; + *) + break # The end of options is encountered. + ;; + esac +done + +if [ "$#" -eq 0 ]; then + error "missing machine image" +fi + +# Verify the state directory existence. +# +if [ ! -d "$state" ]; then + error "state directory '$state' does not exist or is not a directory" +fi + +# Compute the start and QEMU options. +# +start_ops=() +qemu_ops=(\ + -fsdev "local,id=state,path=$state,security_model=passthrough" \ + -device "virtio-9p-pci,fsdev=state,mount_tag=state") + +if [ -n "$install" ]; then + + # Verify the toolchain install script existence in the install directory. + # + if [ ! -f "$(echo "$install"/build2-install-*.sh)" ]; then + error "missing toolchain installation script in '$install' directory" + fi + + start_ops+=(--stdio) + qemu_ops+=(\ + -fsdev "local,id=install,path=$install,security_model=passthrough" \ + -device "virtio-9p-pci,fsdev=install,mount_tag=install") +fi + +# Finally, forward execution to the base script. +# +scr_dir="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" + +exec "$scr_dir/vm-start-base" "${start_ops[@]}" "$@" "${qemu_ops[@]}" diff --git a/etc/private/vm-start-base b/etc/private/vm-start-base new file mode 100755 index 0000000..4a81661 --- /dev/null +++ b/etc/private/vm-start-base @@ -0,0 +1,206 @@ +#! /usr/bin/env bash + +# Start a QEMU/KVM virtual machine. +# +# --cpu <num> +# CPU hardware threads to allocate to the VM, 1 by default. +# +# --ram <num> +# RAM to allocate to the VM, 2G by default (can be specified with G, +# M suffixes). +# +# --tap <tap> +# Existing tap interface to use instead of creating a new one. +# +# --mac <addr> +# MAC address to use for the machine. +# +# --pid <path> +# PID file path, /tmp/vm-<tap>.pid if unspecified. +# +# --monitor <path> +# Monitor UNIX socket path, /tmp/vm-<tap>-mon.sock if unspecified. +# +# --console <path> +# Console UNIX socket path, /tmp/vm-<tap>-con.sock if unspecified. +# +# --stdio +# Connect both console and monitor to stdio (multiplexed). This disables +# the creation of the monitor and console sockets. +# +# --stdio-monior +# Connect only monitor to stdio. This disables the creation of the monitor +# socket. +# +usage="usage: $0 [<options>] <vm-img> [<extra-qemu-options>]" + +trap "{ exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +qemu=(qemu-system-x86_64 -enable-kvm) + +# The bridge is only used if we are cretaing the tap. +# +br=br0 + +cpu=1 +ram=2G +tap= +mac="de:ad:be:ef:b8:da" +pid= +mon= +con= +stdio= +stdio_monitor= + +while [ "$#" -gt 0 ]; do + case "$1" in + --cpu) + shift + cpu="$1" + shift + ;; + --ram) + shift + ram="$1" + shift + ;; + --tap) + shift + tap="$1" + shift + ;; + --mac) + shift + mac="$1" + shift + ;; + --pid) + shift + pid="$1" + shift + ;; + --monitor) + shift + mon="$1" + shift + ;; + --console) + shift + con="$1" + shift + ;; + --stdio) + stdio=true + stdio_monitor= + shift + ;; + --stdio-monitor) + stdio= + stdio_monitor=true + shift + ;; + *) + break + ;; + esac +done + +img="$1" +shift + +if [ -z "$img" ]; then + error "missing virtual machine image" +fi + +if [ ! -f "$img" ]; then + error "virtual machine image '$img' does not exist" +fi + +# Open the reading file descriptor and lock the machine image. Fail if unable +# to lock. +# +# Note that the file descriptor is automatically closed on the script exit and +# the lock is released. +# +exec {lfd}<"$img" + +if ! flock -n "$lfd"; then + error "virtual machine image is already in use" +fi + +del_tap= +if [ -z "$tap" ]; then + tap=tap9 + sudo ip tuntap delete "$tap" mode tap || true + sudo ip tuntap add "$tap" mode tap user "$(whoami)" + sudo ip link set "$tap" up + #sleep 0.5s + sudo ip link set "$tap" master "$br" + del_tap=true +fi + +if [ -z "$pid" ]; then + pid="/tmp/vm-$tap.pid" +fi +echo "$$" >"$pid" + +if [ -z "$mon" ]; then + mon="/tmp/vm-$tap-mon.sock" +fi + +if [ -z "$con" ]; then + con="/tmp/vm-$tap-con.sock" +fi + +ops=(\ + -m "$ram" \ + -cpu host -smp "$cpu,sockets=1,cores=$cpu,threads=1" \ + \ + -netdev "tap,id=net0,ifname=$tap,script=no" \ + -device "virtio-net-pci,netdev=net0,mac=$mac" \ + \ + -drive "if=none,id=disk0,file=$img,format=raw" \ + -device "virtio-blk-pci,scsi=off,drive=disk0" \ + \ + -nographic \ +) + +# Console/monitor options. +# +if [ "$stdio" ]; then + # Multiplex the monitor and serial console onto stdio. In particular, this + # makes sure Ctrl-c is passed to the guest (rather than termination the QEMU + # process). To switch between monitor and console, Ctrl-a,c (to terminate + # QEMU, type quit in the monitor). + # + ops+=(-serial mon:stdio) +else + # Monitor. + # + if [ "$stdio_monitor" ]; then + ops+=(-chardev stdio,id=mon) + else + ops+=(-chardev "socket,id=mon,path=$mon,server,nowait") + fi + + ops+=(-mon chardev=mon,mode=readline) + + # Console. + # + ops+=(-chardev "socket,id=con,path=$con,server,nowait" \ + -serial chardev:con) +fi + +"${qemu[@]}" "${ops[@]}" -boot c "$@" + +if [ "$pid" -o "$mon" -o "$con" ]; then + rm -f "$pid" "$mon" "$con" +fi + +if [ "$del_tap" ]; then + sudo ip tuntap delete "$tap" mode tap +fi diff --git a/etc/private/vm-stop b/etc/private/vm-stop new file mode 100755 index 0000000..cf64dee --- /dev/null +++ b/etc/private/vm-stop @@ -0,0 +1,37 @@ +#! /usr/bin/env bash + +# Stop virtual machine started with vm-start. +# +usage="usage: $0 <pid-file> <monitor-socket>" + +trap "{ exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + + +if [ -z "$1" -o ! -f "$1" ]; then + error "missing or invalid PID file" +fi + +pid="$(sed -nr -e 's/([0-9]+)/\1/p' "$1")" + +if [ -z "$pid" ]; then + error "PID file $1 does not contain valid PID" +fi + +if [ -z "$2" -o ! -S "$2" ]; then + error "missing or invalid monitor socket" +fi + +mon="$2" + +echo system_powerdown | socat - "UNIX-CONNECT:$mon" >/dev/null + +# An alternative way to implement this would be to connect a pipe to the +# monitor socket and wait for it to be closed. +# +while [ -e "/proc/$pid" ]; do + sleep 0.2 +done diff --git a/etc/proxy-apache2.conf b/etc/proxy-apache2.conf new file mode 100644 index 0000000..fc7cfea --- /dev/null +++ b/etc/proxy-apache2.conf @@ -0,0 +1,144 @@ +# Paste the following fragment into the <VirtualHost> section intended for +# proxying HTTP(S) requests and caching the responses. See INSTALL-PROXY for +# details. +# +# List of modules used: +# +# rewrite +# headers +# ssl +# proxy +# proxy_http +# cache +# cache_disk +# + + # Enable the rewrite rules functionality. + # + <IfModule !rewrite_module> + Error "rewrite_module is not enabled" + </IfModule> + + RewriteEngine on + RewriteOptions AllowAnyURI + + # Make sure that the HTTP header management functionality is enabled. + # + <IfModule !headers_module> + Error "headers_module is not enabled" + </IfModule> + + # Enable the HTTP proxy. + # + <IfModule !proxy_module> + Error "proxy_module is not enabled" + </IfModule> + + <IfModule !proxy_http_module> + Error "proxy_http_module is not enabled" + </IfModule> + + ProxyRequests On + + # Enable SSL/TLS API usage for querying HTTPS URLs. + # + <IfModule !ssl_module> + Error "ssl_module is not enabled" + </IfModule> + + SSLProxyEngine on + + # Optional: prevent non-authorized proxy usage, for example: + # + # <Proxy *> + # Require ip 10.5 + # </Proxy> + + # Accept only the HTTP GET method and respond with the 403 HTTP status + # code (Forbidden) for other methods. + # + RewriteCond %{REQUEST_METHOD} !GET + RewriteRule .* - [F] + + # Optional: restrict the URL set allowed for proxying, for example: + # + # RewriteCond %{HTTP_HOST} !(.+\.)?example.org + # RewriteRule .* - [F] + + # Convert the http scheme to https for URLs being proxied. + # + # To prevent the conversion we can exclude certain hosts. For example: + # + # RewriteCond %{HTTP_HOST} !(.+\.)?example.org [OR] + # RewriteCond %{HTTP_HOST} !(.+\.)?example.net + # + # Or check for a custom header value. Note that this header should not + # be forwarded to the origin server. For example: + # + # RewriteCond %{HTTP:X-Preserve-HTTP} !(1|on|true) [NC] + # RequestHeader unset X-Preserve-HTTP + # + RewriteRule ^proxy:http://(.*)$ "https://$1" [P] + + # Enable the disk storage-based cache. + # + <IfModule !cache_module> + Error "cache_module is not enabled" + </IfModule> + + <IfModule !cache_disk_module> + Error "cache_disk_module is not enabled" + </IfModule> + + CacheEnable disk "http://" + + # Specify the cache root directory and make sure it is writable by the + # user under which Apache2 is running. + # + # Note that if there are no other proxies enabled for the WEB server, + # you can probably specify (you still have to specify it) the default + # cache directory (/var/cache/apache2/mod_cache_disk for Debian/Ubuntu + # and /var/cache/httpd/proxy for Fedora/RHEL). + # + CacheRoot + + # Cache entry maximum size (in bytes). + # + CacheMaxFileSize 100000000 + + # Prevent duplicate caching of responses for the same simultaneously + # proxied URL. Specify an appropriate per-URL lock timeout (in + # seconds) to avoid stalled downloads from keeping the entries + # uncached. + # + CacheLock on + CacheLockMaxAge 600 + + # Always validate an existing cache entry by querying the origin + # server. + # + # We do this by injecting the request header which always declares the + # existing cache entry as potentially stale (ignoring Expire response + # header and Cache-Control header's max-age field) which should also + # be propagated through all the upstream proxies forcing them to + # validate the resource freshness. + # + # Note that this relies on both the proxy and origin servers correctly + # supporting conditional requests based on entity tags (ETag HTTP + # response and If-None-Match HTTP request headers) or less accurate + # entity modification times (Last-Modified HTTP response and + # If-Modified-Since HTTP request headers), which is normally the case + # if both are running Apache. A proxy normally caches the ETag and/or + # Last-Modified response header values alongside the cached entity and + # adds If-None-Match and/or If-Modified-Since headers respectively to + # the entity validation request. An origin server normally checks if + # any of the ETag or Last-Modified headers changed for the entity and + # responds with its full content, if that's the case, or with the 304 + # HTTP status code (Not Modified) otherwise (see the Apache Caching + # Guide for details). + # + # Also note that to observe the injected header the cache handler + # should not be configured as a quick handler. + # + RequestHeader set Cache-Control max-age=0 + CacheQuickHandler off diff --git a/etc/systemd/brep-clean.service b/etc/systemd/brep-clean.service index 739a54a..d2e5630 100644 --- a/etc/systemd/brep-clean.service +++ b/etc/systemd/brep-clean.service @@ -1,5 +1,5 @@ [Unit] -Description=brep build database cleaner service +Description=brep build database and artifacts cleaner service [Service] Type=oneshot @@ -7,9 +7,12 @@ Type=oneshot #Group=brep # Run both tenants and builds cleaners if CI request functionality is enabled. +# Also run outdated build artifacts cleaners if build artifacts upload +# functionality is enabled. # #ExecStart=/home/brep/install/bin/brep-clean tenants 240 ExecStart=/home/brep/install/bin/brep-clean builds /home/brep/config/buildtab +#ExecStart=/home/brep/install/bin/brep-upload-bindist-clean /var/bindist 2880 [Install] WantedBy=default.target diff --git a/etc/systemd/brep-clean.timer b/etc/systemd/brep-clean.timer index f4c587e..8e1e6e7 100644 --- a/etc/systemd/brep-clean.timer +++ b/etc/systemd/brep-clean.timer @@ -10,9 +10,9 @@ Unit=brep-clean.service # Persistent=false -# Wait 20 seconds until the first run. +# Wait 30 seconds until the first run. # -OnBootSec=20 +OnBootSec=30 # Then wait 5 minutes until the next run. # diff --git a/etc/systemd/brep-monitor.service b/etc/systemd/brep-monitor.service new file mode 100644 index 0000000..0a5c25e --- /dev/null +++ b/etc/systemd/brep-monitor.service @@ -0,0 +1,14 @@ +[Unit] +Description=brep infrastructure monitor service + +[Service] +Type=oneshot +#User=brep +#Group=brep + +# Replace the public toolchain name with a real list of toolchains. +# +ExecStart=/home/brep/install/bin/brep-monitor --report-timeout 86400 --clean /home/brep/config/brep-module.conf public + +[Install] +WantedBy=default.target diff --git a/etc/systemd/brep-monitor.timer b/etc/systemd/brep-monitor.timer new file mode 100644 index 0000000..f5f5a64 --- /dev/null +++ b/etc/systemd/brep-monitor.timer @@ -0,0 +1,23 @@ +[Unit] +Description=brep infrastructure monitor timer +RefuseManualStart=no +RefuseManualStop=no + +[Timer] +Unit=brep-monitor.service + +# Don't keep track of the timer across reboots. +# +Persistent=false + +# Wait 40 seconds until the first run. +# +OnBootSec=40 + +# Then wait 1 hour until the next run. +# +OnUnitInactiveSec=1h + + +[Install] +WantedBy=timers.target |