diff options
Diffstat (limited to 'doc')
-rw-r--r-- | doc/buildfile | 10 | ||||
-rwxr-xr-x | doc/cli.sh | 38 | ||||
-rw-r--r-- | doc/manual.cli | 236 | ||||
m--------- | doc/style | 0 |
4 files changed, 261 insertions, 23 deletions
diff --git a/doc/buildfile b/doc/buildfile index 4b2305e..f0a9387 100644 --- a/doc/buildfile +++ b/doc/buildfile @@ -1,11 +1,11 @@ # file : doc/buildfile -# copyright : Copyright (c) 2014-2019 Code Synthesis Ltd # license : MIT; see accompanying LICENSE file -cmds = \ -brep-clean \ -brep-load \ -brep-migrate +cmds = \ +brep-clean \ +brep-load \ +brep-migrate \ +brep-monitor ./: {man1 xhtml}{$cmds} \ css{common pre-box man} \ @@ -1,7 +1,6 @@ #! /usr/bin/env bash -version=0.13.0-a.0.z -date="$(date +"%B %Y")" +version=0.17.0-a.0.z trap 'exit 1' ERR set -o errtrace # Trap in functions. @@ -9,6 +8,9 @@ set -o errtrace # Trap in functions. function info () { echo "$*" 1>&2; } function error () { info "$*"; exit 1; } +date="$(date +"%B %Y")" +copyright="$(sed -n -re 's%^Copyright \(c\) (.+) \(see the AUTHORS and LEGAL files\)\.$%\1%p' ../LICENSE)" + while [ $# -gt 0 ]; do case $1 in --clean) @@ -36,18 +38,31 @@ function compile () shift done - cli -I .. -v project="brep" -v version="$version" -v date="$date" \ ---include-base-last "${o[@]}" --generate-html --html-prologue-file \ -man-prologue.xhtml --html-epilogue-file man-epilogue.xhtml --html-suffix .xhtml \ + cli -I .. \ +-v project="brep" \ +-v version="$version" \ +-v date="$date" \ +-v copyright="$copyright" \ +--include-base-last "${o[@]}" \ +--generate-html --html-suffix .xhtml \ +--html-prologue-file man-prologue.xhtml \ +--html-epilogue-file man-epilogue.xhtml \ --link-regex '%bpkg([-.].+)%../../bpkg/doc/bpkg$1%' \ --link-regex '%brep(#.+)?%build2-repository-interface-manual.xhtml$1%' \ ../$n.cli - cli -I .. -v project="brep" -v version="$version" -v date="$date" \ ---include-base-last "${o[@]}" --generate-man --man-prologue-file \ -man-prologue.1 --man-epilogue-file man-epilogue.1 --man-suffix .1 \ + cli -I .. \ +-v project="brep" \ +-v version="$version" \ +-v date="$date" \ +-v copyright="$copyright" \ +--include-base-last "${o[@]}" \ +--generate-man --man-suffix .1 \ +--man-prologue-file man-prologue.1 \ +--man-epilogue-file man-epilogue.1 \ --link-regex '%bpkg(#.+)?%$1%' \ --link-regex '%brep(#.+)?%$1%' \ +--link-regex '%bbot(#.+)?%$1%' \ ../$n.cli } @@ -57,7 +72,7 @@ o="--output-prefix brep-" # #compile "brep" $o --output-prefix "" -pages="clean/clean load/load migrate/migrate" +pages="clean/clean load/load migrate/migrate monitor/monitor" for p in $pages; do compile $p $o @@ -79,13 +94,16 @@ function xhtml_to_ps () # <from> <to> [<html2ps-options>] cli -I .. \ -v version="$(echo "$version" | sed -e 's/^\([^.]*\.[^.]*\).*/\1/')" \ -v date="$date" \ +-v copyright="$copyright" \ --generate-html --html-suffix .xhtml \ --html-prologue-file doc-prologue.xhtml \ --html-epilogue-file doc-epilogue.xhtml \ --link-regex '%b([-.].+)%../../build2/doc/b$1%' \ --link-regex '%bpkg([-.].+)%../../bpkg/doc/bpkg$1%' \ --link-regex '%bpkg(#.+)?%../../bpkg/doc/build2-package-manager-manual.xhtml$1%' \ ---output-prefix build2-repository-interface- manual.cli +--link-regex '%bbot(#.+)?%../../bbot/doc/build2-build-bot-manual.xhtml$1%' \ +--output-prefix build2-repository-interface- \ +manual.cli xhtml_to_ps build2-repository-interface-manual.xhtml build2-repository-interface-manual-a4.ps -f doc.html2ps:a4.html2ps ps2pdf14 -sPAPERSIZE=a4 -dOptimize=true -dEmbedAllFonts=true build2-repository-interface-manual-a4.ps build2-repository-interface-manual-a4.pdf diff --git a/doc/manual.cli b/doc/manual.cli index 61ef1f8..2b96393 100644 --- a/doc/manual.cli +++ b/doc/manual.cli @@ -1,5 +1,4 @@ // file : doc/manual.cli -// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd // license : MIT; see accompanying LICENSE file "\name=build2-repository-interface-manual" @@ -16,7 +15,8 @@ This document describes \c{brep}, the \c{build2} package repository web interface. For the command line interface of \c{brep} utilities refer to the -\l{brep-load(1)}, \l{brep-clean(1)}, and \l{brep-migrate(1)} man pages. +\l{brep-load(1)}, \l{brep-clean(1)}, \l{brep-migrate(1)}, and +\l{brep-monitor(1)} man pages. \h1#submit|Package Submission| @@ -51,8 +51,8 @@ binary mode.| \li|Verify other parameters are valid manifest name/value pairs. -The value can only contain printable ASCII characters as well as tab -(\c{\\t}), carriage return (\c{\\r}), and line feed (\c{\\n}).| +The value can only contain UTF-8 encoded Unicode graphic characters as well as +tab (\c{\\t}), carriage return (\c{\\r}), and line feed (\c{\\n}).| \li|Check for a duplicate submission. @@ -121,7 +121,6 @@ reference: <abbrev-checksum> | - \li|Send the submission email. If \c{submit-email} is configured, send an email to this address containing @@ -228,8 +227,8 @@ upload.| \li|Verify other parameters are valid manifest name/value pairs. -The value can only contain printable ASCII characters as well as tab -(\c{\\t}), carriage return (\c{\\r}), and line feed (\c{\\n}).| +The value can only contain UTF-8 encoded Unicode graphic characters as well as +tab (\c{\\t}), carriage return (\c{\\r}), and line feed (\c{\\n}).| \li|Generate CI request id and create request directory. @@ -306,6 +305,15 @@ Check violations that are explicitly mentioned above are always reported with the CI result manifest. Other errors (for example, internal server errors) might be reported with unformatted text, including HTML. +If the CI request contains the \c{interactive} parameter, then the CI service +provides the execution environment login information for each test and stops +them at the specified breakpoint. + +Pre-defined breakpoint ids are \c{error} and \c{warning}. The breakpoint id is +included into the CI request manifest and the CI service must at least handle +\c{error} but may recognize additional ids (build phase/command identifiers, +etc). + If the CI request contains the \c{simulate} parameter, then the CI service simulates the specified outcome of the CI process without actually performing any externally visible actions (e.g., testing the package, publishing the @@ -328,16 +336,28 @@ corresponding to the custom request parameters. id: <request-id> repository: <url> [package]: <name>[/<version>] -timestamp: <date-time> +[interactive]: <breakpoint> [simulate]: <outcome> +timestamp: <date-time> [client-ip]: <string> [user-agent]: <string> +[service-id]: <string> +[service-type]: <string> +[service-data]: <string> \ The \c{package} value can be repeated multiple times. The \c{timestamp} value is in the ISO-8601 \c{<YYYY>-<MM>-<DD>T<hh>:<mm>:<ss>Z} form (always UTC). Note also that \c{client-ip} can be IPv4 or IPv6. +Note that some CI service implementations may serve as backends for +third-party services. The latter may initiate CI tasks, providing all the +required information via some custom protocol, and expect the CI service to +notify it about the progress. In this case the third-party service type as +well as optionally the third-party id and custom state data can be +communicated to the underlying CI handler program via the respective +\c{service-*} manifest values. + \h#ci-overrides-manifest|CI Overrides Manifest| @@ -349,8 +369,30 @@ being applied. Currently, only the following value groups can be overridden: \ build-email build-{warning,error}-email builds build-{include,exclude} +*-builds *-build-{include,exclude} +*-build-config \ +For the package configuration-specific build constraint overrides the +corresponding configuration must exist in the package manifest. In contrast, +the package configuration override (\cb{*-build-config}) adds a new +configuration if it doesn't exist and updates the arguments of the existing +configuration otherwise. In the former case, all the potential build +constraint overrides for such a newly added configuration must follow the +corresponding \cb{*-build-config} override. + +Note that the build constraints group values (both common and build package +configuration-specific) are overridden hierarchically so that the +\c{[\b{*-}]\b{build-}{\b{include},\b{exclude}\}} overrides don't affect the +respective \c{[\b{*-}]\b{builds}} values. + +Note also that the common and build package configuration-specific build +constraints group value overrides are mutually exclusive. If the common build +constraints are overridden, then all the configuration-specific constraints +are removed. Otherwise, if any configuration-specific constraints are +overridden, then for the remaining configurations the build constraints are +reset to \cb{builds:\ none}. + See \l{bpkg#manifest-package Package Manifest} for details on these values. @@ -368,4 +410,182 @@ message: <string> [reference]: <string> \ + +\h1#upload|Build Artifacts Upload| + +The build artifacts upload functionality allows uploading archives of files +generated as a byproduct of the package builds. Such archives as well as +additional, repository-specific information can optionally be uploaded by the +automated build bots via the HTTP \c{POST} method using the +\c{multipart/form-data} content type (see the \l{bbot \c{bbot} documentation} +for details). The implementation in \c{brep} only handles uploading as well as +basic actions and verification (build session resolution, agent +authentication, checksum verification) expecting the rest of the upload logic +to be handled by a separate entity according to the repository policy. Such an +entity can be notified by \c{brep} about a new upload as an invocation of the +\i{handler program} (as part of the HTTP request) and/or via email. It could +also be a separate process that monitors the upload data directory. + +For each upload request \c{brep} performs the following steps. + +\ol| + +\li|Determine upload type. + +The upload type must be passed via the \c{upload} parameter in the query +component of the request URL.| + +\li|Verify upload size limit. + +The upload form-data payload size must not exceed \c{upload-max-size} specific +for this upload type.| + +\li|Verify the required \c{session}, \c{instance}, \c{archive}, and +\c{sha256sum} parameters are present. If \c{brep} is configured to perform +agent authentication, then verify that the \c{challenge} parameter is also +present. See the \l{bbot#arch-result-req Result Request Manifest} for +semantics of the \c{session} and \c{challenge} parameters. + +The \c{archive} parameter must be the build artifacts archive upload while +\c{sha256sum} must be its 64 characters SHA256 checksum calculated in the +binary mode.| + +\li|Verify other parameters are valid manifest name/value pairs. + +The value can only contain UTF-8 encoded Unicode graphic characters as well as +tab (\c{\\t}), carriage return (\c{\\r}), and line feed (\c{\\n}).| + +\li|Resolve the session. + +Resolve the \c{session} parameter value to the actual package build +information.| + +\li| Authenticate the build bot agent. + +Use the \c{challenge} parameter value and the resolved package build +information to authenticate the agent, if configured to do so.| + +\li|Generate upload request id and create request directory. + +For each upload request a unique id (UUID) is generated and a request +subdirectory is created in the \c{upload-data} directory with this id as its +name.| + +\li|Save the upload archive into the request directory and verify its +checksum. + +The archive is saved using the submitted name, and its checksum is calculated +and compared to the submitted checksum.| + +\li|Save the upload request manifest into the request directory. + +The upload request manifest is saved as \c{request.manifest} into the request +subdirectory next to the archive.| + +\li|Invoke the upload handler program. + +If \c{upload-handler} is configured, invoke the handler program passing to it +additional arguments specified with \c{upload-handler-argument} (if any) +followed by the absolute path to the upload request directory. + +The handler program is expected to write the upload result manifest to +\c{stdout} and terminate with the zero exit status. A non-zero exit status is +treated as an internal error. The handler program's \c{stderr} is logged. + +Note that the handler program should report temporary server errors (service +overload, network connectivity loss, etc.) via the upload result manifest +status values in the [500-599] range (HTTP server error) rather than via a +non-zero exit status. + +The handler program assumes ownership of the upload request directory and can +move/remove it. If after the handler program terminates the request directory +still exists, then it is handled by \c{brep} depending on the handler process +exit status and the upload result manifest status value. If the process has +terminated abnormally or with a non-zero exit status or the result manifest +status is in the [500-599] range (HTTP server error), then the directory is +saved for troubleshooting by appending the \c{.fail} extension to its name. +Otherwise, if the status is in the [400-499] range (HTTP client error), then +the directory is removed. If the directory is left in place by the handler or +is saved for troubleshooting, then the upload result manifest is saved as +\c{result.manifest} into this directory, next to the request manifest. + +If \c{upload-handler-timeout} is configured and the handler program does not +exit in the allotted time, then it is killed and its termination is treated as +abnormal. + +If the handler program is not specified, then the following upload result +manifest is implied: + +\ +status: 200 +message: <upload-type> upload is queued +reference: <request-id> +\ + +| + +\li|Send the upload email. + +If \c{upload-email} is configured, send an email to this address containing +the upload request manifest and the upload result manifest.| + +\li|Respond to the client. + +Respond to the client with the upload result manifest and its \c{status} value +as the HTTP status code.| + +| + +Check violations (max size, etc) that are explicitly mentioned above are +always reported with the upload result manifest. Other errors (for example, +internal server errors) might be reported with unformatted text, including +HTML. + + +\h#upload-request-manifest|Upload Request Manifest| + +The upload request manifest starts with the below values and in that order +optionally followed by additional values in the unspecified order +corresponding to the custom request parameters. + +\ +id: <request-id> +session: <session-id> +instance: <name> +archive: <name> +sha256sum: <sum> +timestamp: <date-time> + +name: <name> +version: <version> +project: <name> +target-config: <name> +package-config: <name> +target: <target-triplet> +[tenant]: <tenant-id> +toolchain-name: <name> +toolchain-version: <standard-version> +repository-name: <canonical-name> +machine-name: <name> +machine-summary: <text> +\ + +The \c{timestamp} value is in the ISO-8601 +\c{<YYYY>-<MM>-<DD>T<hh>:<mm>:<ss>Z} form (always UTC). + + +\h#upload-result-manifest|Upload Result Manifest| + +The upload result manifest starts with the below values and in that order +optionally followed by additional values if returned by the handler program. +If the upload request is successful, then the \c{reference} value must be +present and contain a string that can be used to identify this request (for +example, the upload request id). + +\ +status: <http-code> +message: <string> +[reference]: <string> +\ + " diff --git a/doc/style b/doc/style -Subproject 7e75bb936cf5b2bd2fa3344e13b6c486c8ecc8a +Subproject b72eb624d13b1628e27e9f6c0b3c80853e8e015 |