From 6c5fad831329c6949a858a956fea117c4736f08d Mon Sep 17 00:00:00 2001
From: Karen Arutyunov <karen@codesynthesis.com>
Date: Tue, 2 Nov 2021 20:46:23 +0300
Subject: Document worker scripts

---
 doc/manual.cli | 609 +++++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 550 insertions(+), 59 deletions(-)

diff --git a/doc/manual.cli b/doc/manual.cli
index 38b52e0..86b90ba 100644
--- a/doc/manual.cli
+++ b/doc/manual.cli
@@ -917,14 +917,38 @@ modules (\c{<env-modules>}) and the list of configuration options and variables
 The re-executed \c{bbot} worker then proceeds to test the package from the
 repository by executing the following commands, collectively called a
 \i{worker script}. Each command has a unique \i{step id} that can be used as a
-breakpoint as well as a prefix in the \c{<config-args>},
+breakpoint and normally as a prefix in the \c{<config-args>},
 \c{<env-config-args>}, and \c{<env-modules>} values as discussed in
-\l{#arch-controller Controller Logic}. Some step ids have fallback step ids
-(listed in parenthesis) which are used in the absence of the primary step id
-values. The \c{<>}-values are from the task manifest and the environment.
+\l{#arch-controller Controller Logic}. The \c{<>}-values are from the task
+manifest and the environment though some are assigned by the worker during the
+script execution (configuration directories, UUIDs, etc).
+
+Some prefix step ids have fallback step ids which are used in the absence of
+the primary step id values. If the prefix step id differs from the breakpoint
+step id and/or has the fallback step ids, then they are listed in parenthesis:
+the prefix id before the colon and the fallback ids after it.
+
+Some commands have no configuration or environment options or variables. Such
+commands have only breakpoint step ids associated, which are listed in square
+brackets.
+
+Note that the worker script varies for different primary package types. The
+\c{bbot} worker classifies the primary package based on the configuration type
+in which it is built: \c{module} (build system module packages), \c{host}
+(packages such as source code generators, marked with the \c{requires: host}
+manifest value; see \l{bpkg#manifest-package Package Manifest} for details),
+and \c{target} (all other packages).
+
+Note also that the \c{*.configure.build} step configures potentially multiple
+packages (primary package, tests, etc) in potentially multiple configurations
+by always using the \c{bpkg.global.configure.build} prefix step id for global
+(as opposed to package-specific) \l{bpkg-pkg-build(1)} options. The
+\c{bpkg.global.configure.build} prefix id has no fallback ids.
+
+Worker script for \c{target} packages:
 
 \
-# bpkg.create
+# bpkg.create (bpkg.target.create : b.create, bpkg.create)
 #
 bpkg -V create <env-modules> <env-config-args> <config-args>
 
@@ -936,9 +960,13 @@ bpkg -v add <repository-url>
 #
 bpkg -v fetch --trust <repository-fp>
 
-# bpkg.configure.build
+# bpkg.configure.build (
+#   bpkg.global.configure.build,
+#   (bpkg.target.configure.build : b.configure, bpkg.configure.build))
 #
-bpkg -v build --yes --configure-only <package-name>/<package-version>
+bpkg -v build --configure-only <env-config-args> <config-args> \\
+     <package-name>/<package-version> \\
+     [<test-package-name>[ <test-version-constraint>]...]
 
 # bpkg.update
 #
@@ -952,20 +980,15 @@ bpkg -v update <package-name>
   bpkg -v test <package-name>
 }
 
-# For each package referred to by the tests, examples, or benchmarks
-# package manifest values not excluded by the bbot controller:
+# For each (runtime) tests, examples, or benchmarks package referred
+# to by the task manifest:
 #
 {
-  # bpkg.test-separate.configure.build (bpkg.configure.build)
-  #
-  bpkg -v build --yes --configure-only \\
-       '<package-name> [<version-constraint>]'
-
-  # bpkg.test-separate.update (bpkg.update)
+  # bpkg.test-separate.update ( : bpkg.update)
   #
   bpkg -v update <package-name>
 
-  # bpkg.test-separate.test (bpkg.test)
+  # bpkg.test-separate.test ( : bpkg.test)
   #
   bpkg -v test <package-name>
 }
@@ -981,51 +1004,61 @@ bpkg -v update <package-name>
   # operation:
   #
   {
-    # b.test-installed.create
+    # b.test-installed.create ( : b.create)
     #
     b -V create <env-modules> <env-config-args> <config-args>
 
-    # b.test-installed.configure
+    # For each test subproject:
     #
-    b -v configure
+    {
+      # b.test-installed.configure ( : b.configure)
+      #
+      b -v configure
+    }
 
     # b.test-installed.test
     #
     b -v test
   }
 
-  # If any of the tests, examples, or benchmarks package manifest
-  # values are not excluded by the bbot controller:
+  # If task manifest refers to any (runtime) tests, examples, or
+  # benchmarks packages.
   #
   {
-    # bpkg.test-installed.create (bpkg.create)
+    # bpkg.test-separate-installed.create (
+    #   bpkg.test-separate-installed.create_for_target :
+    #     bpkg.test-separate-installed.create)
     #
     bpkg -V create <env-modules> <env-config-args> <config-args>
 
-    # bpkg.test-installed.configure.add (bpkg.configure.add)
+    # bpkg.test-separate-installed.configure.add (
+    #  : bpkg.configure.add)
     #
     bpkg -v add <repository-url>
 
-    # bpkg.test-installed.configure.fetch (bpkg.configure.fetch)
+    # bpkg.test-separate-installed.configure.fetch (
+    #   : bpkg.configure.fetch)
     #
     bpkg -v fetch --trust <repository-fp>
 
-    # For each package referred to by the tests, examples, or
-    # benchmarks package manifest values not excluded by the
-    # bbot controller:
+    # bpkg.test-separate-installed.configure.build (
+    #   bpkg.global.configure.build,
+    #   (bpkg.test-separate-installed.configure.build_for_target :
+    #      bpkg.test-separate-installed.configure.build))
     #
-    {
-      # bpkg.test-separate-installed.configure.build (
-      #   bpkg.configure.build)
-      #
-      bpkg -v build --yes --configure-only \\
-           '<package-name> [<version-constraint>]'
+    bpkg -v build --configure-only <env-config-args> <config-args> \\
+         <test-package-name>[ <test-version-constraint>]... \\
+         ?sys:<package-name>
 
-      # bpkg.test-separate-installed.update (bpkg.update)
+    # For each (runtime) tests, examples, or benchmarks package
+    # referred to by the task manifest:
+    #
+    {
+      # bpkg.test-separate-installed.update ( : bpkg.update)
       #
       bpkg -v update <package-name>
 
-      # bpkg.test-separate-installed.test (bpkg.test)
+      # bpkg.test-separate-installed.test ( : bpkg.test)
       #
       bpkg -v test <package-name>
     }
@@ -1041,47 +1074,505 @@ bpkg -v update <package-name>
 # This step id can only be used as a breakpoint.
 \
 
-For details on configuring and testing installation refer to
-\l{#arch-controller Controller Logic}.
+Worker script for \c{host} packages:
+
+\
+# If configuration is self-hosted:
+#
+{
+  # bpkg.create (bpkg.host.create : b.create, bpkg.create)
+  #
+  bpkg -V create --type host -d <host-conf> \\
+       <env-modules> <env-config-args> <config-args>
+}
+#
+# Otherwise:
+#
+{
+  # [bpkg.create]
+  #
+  b -V create(<host-conf>, cc) config.config.load=~host
+  bpkg -v create --existing --type host -d <host-conf>
+}
+
+# bpkg.configure.add
+#
+bpkg -v add -d <host-conf> <repository-url>
+
+# bpkg.configure.fetch
+#
+bpkg -v fetch -d <host-conf> --trust <repository-fp>
+
+# If configuration is self-hosted and config.install.root is specified:
+#
+{
+  # bpkg.create (bpkg.target.create : b.create, bpkg.create)
+  #
+  bpkg -V create -d <install-conf> \\
+       <env-modules> <env-config-args> <config-args>
+
+  # [bpkg.link]
+  #
+  bpkg -v link -d <install-conf> <host-conf>
+
+  # bpkg.configure.add
+  #
+  bpkg -v add -d <install-conf> <repository-url>
+
+  # bpkg.configure.fetch
+  #
+  bpkg -v fetch -d <install-conf> --trust <repository-fp>
+}
+
+# If task manifest refers to any build-time tests, examples, or
+# benchmarks packages:
+#
+{
+  # bpkg.create (bpkg.target.create : b.create, bpkg.create)
+  #
+  bpkg -V create -d <target-conf> \\
+       <env-modules> <env-config-args> <config-args>
+
+  # [bpkg.create]
+  #
+  b -V create(<module-conf>, cc) config.config.load=~build2
+  bpkg -v create --existing --type build2 -d <module-conf>
+
+  # [bpkg.link]
+  #
+  bpkg -v link -d <target-conf> <host-conf>
+  bpkg -v link -d <target-conf> <module-conf>
+  bpkg -v link -d <host-conf>   <module-conf>
+
+  # If configuration is self-hosted and config.install.root is
+  # specified:
+  #
+  {
+    # [bpkg.link]
+    #
+    bpkg -v link -d <install-conf> <module-conf>
+  }
+
+  # bpkg.configure.add
+  #
+  bpkg -v add -d <target-conf> <repository-url>
+
+  # bpkg.configure.fetch
+  #
+  bpkg -v fetch -d <target-conf> --trust <repository-fp>
+}
+
+# bpkg.configure.build (bpkg.global.configure.build)
+#
+# Notes:
+#
+# - Some parts may be omitted.
+#
+# - Parts related to different configurations have different prefix
+#   step ids:
+#
+#   bpkg.host.configure.build   for <host-uuid>
+#   bpkg.target.configure.build for <install-uuid>
+#   bpkg.target.configure.build for <target-uuid>
+#
+# - All parts have the same fallback step ids: b.configure and
+#   bpkg.configure.build.
+#
+bpkg -v build --configure-only <env-config-args> <config-args> \\
+\\
+{ --config-uuid <host-uuid> <env-config-args> <config-args> }+ \\
+<package-name>/<package-version> \\
+\\
+{ --config-uuid <install-uuid> <env-config-args> <config-args> }+ \\
+<package-name>/<package-version> \\
+\\
+{ --config-uuid <host-uuid> <env-config-args> <config-args> }+ \\
+{ <runtime-test-package-name>[ test-version-constraint>]... } \\
+\\
+{ --config-uuid <target-uuid> <env-config-args> <config-args> }+ \\
+{ <buildtime-test-package-name>[ test-version-constraint>]... }
+
+# bpkg.update
+#
+bpkg -v update -d <host-conf> <package-name>
+
+# If the test operation is supported by the package:
+#
+{
+  # bpkg.test
+  #
+  bpkg -v test -d <host-conf> <package-name>
+}
+
+# If configuration is self-hosted, then for each runtime tests,
+# examples, or benchmarks package referred to by the task manifest:
+#
+{
+  # bpkg.test-separate.update ( : bpkg.update)
+  #
+  bpkg -v update -d <host-conf> <package-name>
+
+  # bpkg.test-separate.test ( : bpkg.test)
+  #
+  bpkg -v test -d <host-conf> <package-name>
+}
+
+# For each build-time tests, examples, or benchmarks package referred
+# to by the task manifest:
+#
+{
+  # bpkg.test-separate.update ( : bpkg.update)
+  #
+  bpkg -v update -d <target-conf> <package-name>
+
+  # bpkg.test-separate.test ( : bpkg.test)
+  #
+  bpkg -v test -d <target-conf> <package-name>
+}
+
+# If configuration is self-hosted and config.install.root is specified:
+#
+{
+  # bpkg.install
+  #
+  bpkg -v install -d <install-conf> <package-name>
+
+  # If the package contains subprojects that support the test
+  # operation:
+  #
+  {
+    # b.test-installed.create ( : b.create)
+    #
+    b -V create <env-modules> <env-config-args> <config-args>
+
+    # For each test subproject:
+    #
+    {
+      # b.test-installed.configure ( : b.configure)
+      #
+      b -v configure
+    }
+
+    # b.test-installed.test
+    #
+    b -v test
+  }
+
+  # If task manifest refers to any tests, examples, or benchmarks
+  # packages:
+  #
+  {
+    # bpkg.test-separate-installed.create (
+    #   bpkg.test-separate-installed.create_for_host :
+    #     bpkg.test-separate-installed.create)
+    #
+    bpkg -V create --type host -d <host-conf> \\
+         <env-modules> <env-config-args> <config-args>
+
+    # If task manifest refers to any runtime tests, examples, or
+    # benchmarks packages:
+    #
+    {
+      # bpkg.test-separate-installed.configure.add (
+      #   : bpkg.configure.add)
+      #
+      bpkg -v add -d <host-conf> <repository-url>
+
+      # bpkg.test-separate-installed.configure.fetch (
+      #   : bpkg.configure.fetch)
+      #
+      bpkg -v fetch -d <host-conf> --trust <repository-fp>
+    }
+
+    # If task manifest refers to any build-time tests, examples, or
+    # benchmarks packages:
+    #
+    {
+      # bpkg.test-separate-installed.create (
+      #   bpkg.test-separate-installed.create_for_host :
+      #     bpkg.test-separate-installed.create)
+      #
+      bpkg -V create -d <target-conf> \\
+           <env-modules> <env-config-args> <config-args>
+
+      # [bpkg.test-separate-installed.create]
+      #
+      b -V create(<module-conf>, cc) config.config.load=~build2
+      bpkg -v create --existing --type build2 -d <module-conf>
+
+      # [bpkg.test-separate-installed.link]
+      #
+      bpkg -v link -d <target-conf> <host-conf>
+      bpkg -v link -d <target-conf> <module-conf>
+      bpkg -v link -d <host-conf>   <module-conf>
+
+      # bpkg.test-separate-installed.configure.add (
+      #   : bpkg.configure.add)
+      #
+      bpkg -v add -d <target-conf> <repository-url>
+
+      # bpkg.test-separate-installed.configure.fetch (
+      #   : bpkg.configure.fetch)
+      #
+      bpkg -v fetch -d <target-conf> --trust <repository-fp>
+    }
+
+    # bpkg.test-separate-installed.configure.build (
+    #   bpkg.global.configure.build,
+    #   (bpkg.test-separate-installed.configure.build_for_host :
+    #      bpkg.test-separate-installed.configure.build))
+    #
+    # Note that any of the runtime or build-time tests related parts
+    # (but not both) may be omitted.
+    #
+    bpkg -v build --configure-only <env-config-args> <config-args> \\
+    \\
+    { --config-name <host-conf> }+ \\
+    { <runtime-test-package-name>[ <test-version-constraint>]... } \\
+    \\
+    { --config-name <target-conf> }+ \\
+    <buildtime-test-package-name>[ <test-version-constraint>]... \\
+    \\
+    ?sys:<package-name>
+
+    # For each tests, examples, or benchmarks package referred
+    # to by the task manifest:
+    #
+    {
+      # bpkg.test-separate-installed.update ( : bpkg.update)
+      #
+      bpkg -v update <package-name>
+
+      # bpkg.test-separate-installed.test ( : bpkg.test)
+      #
+      bpkg -v test <package-name>
+    }
+  }
+
+  # bpkg.uninstall
+  #
+  bpkg -v uninstall -d <install-conf> <package-name>
+}
+
+# end
+#
+# This step id can only be used as a breakpoint.
+\
 
-If the package is a build system module, then it is built and tested (using
-the bundled tests) in a separate configuration that mimics the one used to
-build \c{build2} itself. Note that the configuration and environment options
-and variables are not passed to commands that may affect this configuration.
-Such commands, therefore, have associated step ids that can only be used
-as breakpoints (listed in square brackets):
+Worker script for \c{module} packages:
 
 \
-# [bpkg.module.create]
+# If configuration is self-hosted:
+#
+{
+  # bpkg.create (bpkg.module.create)
+  #
+  b -V create(<module-conf>, <env-modules>) config.config.load=~build2 \\
+     <env-config-args> <config-args>
+  bpkg -v create --existing --type build2 -d <module-conf>
+}
 #
-b -V create config.config.load=~build2
-bpkg -v create --existing
+# Otherwise:
+#
+{
+  # [bpkg.create]
+  #
+  b -V create(<module-conf>, cc) config.config.load=~build2
+  bpkg -v create --existing --type build2 -d <module-conf>
+}
 
-# bpkg.module.configure.add (bpkg.configure.add)
+# bpkg.configure.add
 #
-bpkg -v add <repository-url>
+bpkg -v add -d <module-conf> <repository-url>
 
-# bpkg.module.configure.fetch (bpkg.configure.fetch)
+# bpkg.configure.fetch
 #
-bpkg -v fetch --trust <repository-fp>
+bpkg -v fetch -d <module-conf> --trust <repository-fp>
 
-# [bpkg.module.configure.build]
+# If configuration is self-hosted and config.install.root is specified:
 #
-bpkg -v build --yes --configure-only <package-name>/<package-version>
+{
+  # bpkg.create (bpkg.module.create)
+  #
+  b -V create(<install-conf>, <env-modules>) \\
+     config.config.load=~build2 <env-config-args> <config-args>
+  bpkg -v create --existing --type build2 -d <install-conf>
+
+  # bpkg.configure.add
+  #
+  bpkg -v add -d <install-conf> <repository-url>
 
-# [bpkg.module.update]
+  # bpkg.configure.fetch
+  #
+  bpkg -v fetch -d <install-conf> --trust <repository-fp>
+}
+
+# If task manifest refers to any (build-time) tests, examples, or
+# benchmarks packages:
 #
-bpkg -v update <package-name>
+{
+  # bpkg.create (bpkg.target.create : b.create, bpkg.create)
+  #
+  bpkg -V create -d <target-conf> \\
+       <env-modules> <env-config-args> <config-args>
+
+  # [bpkg.create]
+  #
+  b -V create(<host-conf>, cc) config.config.load=~host
+  bpkg -v create --existing --type host -d <host-conf>
+
+  # [bpkg.link]
+  #
+  bpkg -v link -d <target-conf> <host-conf>
+  bpkg -v link -d <target-conf> <module-conf>
+  bpkg -v link -d <host-conf>   <module-conf>
+
+  # bpkg.configure.add
+  #
+  bpkg -v add -d <target-conf> <repository-url>
+
+  # bpkg.configure.fetch
+  #
+  bpkg -v fetch -d <target-conf> --trust <repository-fp>
+}
+
+# bpkg.configure.build (bpkg.global.configure.build)
+#
+# Notes:
+#
+# - Some parts may be omitted.
+#
+# - Parts related to different configurations have different prefix
+#   step ids:
+#
+#   bpkg.module.configure.build for <module-uuid>
+#   bpkg.target.configure.build for <install-uuid>
+#   bpkg.target.configure.build for <target-uuid>
+#
+# - All parts have the same fallback step ids: b.configure and
+#   bpkg.configure.build.
+#
+bpkg -v build --configure-only <env-config-args> <config-args> \\
+\\
+{ --config-uuid <module-uuid> <env-config-args> <config-args> }+ \\
+<package-name>/<package-version> \\
+\\
+{ --config-uuid <install-uuid> <env-config-args> <config-args> }+ \\
+<package-name>/<package-version> \\
+\\
+{ --config-uuid <target-uuid> <env-config-args> <config-args> }+ \\
+{ <buildtime-test-package-name>[ test-version-constraint>]... }
+
+# bpkg.update
+#
+bpkg -v update -d <module-conf> <package-name>
 
 # If the test operation is supported by the package:
 #
 {
-  # [bpkg.module.test]
+  # bpkg.test
   #
-  bpkg -v test <package-name>
+  bpkg -v test -d <module-conf> <package-name>
+}
+
+# For each (build-time) tests, examples, or benchmarks package referred
+# to by the task manifest:
+#
+{
+  # bpkg.test-separate.update ( : bpkg.update)
+  #
+  bpkg -v update -d <target-conf> <package-name>
+
+  # bpkg.test-separate.test ( : bpkg.test)
+  #
+  bpkg -v test -d <target-conf> <package-name>
+}
+
+# If configuration is self-hosted and config.install.root is specified:
+#
+{
+  # bpkg.install
+  #
+  bpkg -v install -d <install-conf> <package-name>
+
+  # If task manifest refers to any (build-time) tests, examples, or
+  # benchmarks packages:
+  #
+  {
+    # [bpkg.test-separate-installed.create]
+    #
+    b -V create(<module-conf>, cc) config.config.load=~build2
+    bpkg -v create --existing --type build2 -d <module-conf>
+
+    # bpkg.test-separate-installed.create (
+    #   bpkg.test-separate-installed.create_for_module :
+    #     bpkg.test-separate-installed.create)
+    #
+    bpkg -V create -d <target-conf> \\
+         <env-modules> <env-config-args> <config-args>
+
+    # bpkg.test-separate-installed.create (
+    #   bpkg.test-separate-installed.create_for_module :
+    #     bpkg.test-separate-installed.create)
+    #
+    bpkg -V create --type host -d <host-conf> \\
+         <env-modules> <env-config-args> <config-args>
+
+    # [bpkg.test-separate-installed.link]
+    #
+    bpkg -v link -d <target-conf> <host-conf>
+    bpkg -v link -d <target-conf> <module-conf>
+    bpkg -v link -d <host-conf>   <module-conf>
+
+    # bpkg.test-separate-installed.configure.add (
+    #   : bpkg.configure.add)
+    #
+    bpkg -v add -d <target-conf> <repository-url>
+
+    # bpkg.test-separate-installed.configure.fetch (
+    #   : bpkg.configure.fetch)
+    #
+    bpkg -v fetch -d <target-conf> --trust <repository-fp>
+
+    # bpkg.test-separate-installed.configure.build (
+    #   bpkg.global.configure.build,
+    #   (bpkg.test-separate-installed.configure.build_for_module :
+    #      bpkg.test-separate-installed.configure.build))
+    #
+    bpkg -v build --configure-only <env-config-args> <config-args> \\
+    \\
+    { --config-name <target-conf> }+ \\
+    <buildtime-test-package-name>[ <test-version-constraint>]... \\
+    \\
+    ?sys:<package-name>
+
+    # For each (build-time) tests, examples, or benchmarks package
+    # referred to by the task manifest:
+    #
+    {
+      # bpkg.test-separate-installed.update ( : bpkg.update)
+      #
+      bpkg -v update -d <target-conf> <package-name>
+
+      # bpkg.test-separate-installed.test ( : bpkg.test)
+      #
+      bpkg -v test -d <target-conf> <package-name>
+    }
+  }
+
+  # bpkg.uninstall
+  #
+  bpkg -v uninstall -d <install-conf> <package-name>
 }
+
+# end
+#
+# This step id can only be used as a breakpoint.
 \
 
+For details on configuring and testing installation refer to
+\l{#arch-controller Controller Logic}.
+
 If a primary or test package comes from a version control-based repository,
 then its \c{dist} meta-operation is also tested as a part of the
 \c{bpkg[.*].configure.build} steps by re-distributing the source directory in
@@ -1173,8 +1664,7 @@ manifest.
 Values in the \c{<config-arg>} list can be opionally prefixed with the \i{step
 id} or a leading portion thereof to restrict it to a specific step, operation,
 phase, or tool in the \i{worker script} (see \l{#arch-worker Worker
-Logic}). Unprefixed values only apply to the \c{bpkg.create},
-\c{b.test-installed.create}, and \c{bpkg.test-installed.create} steps. Note
+Logic}). Unprefixed values only apply to the \c{*.create[_for_*]} steps. Note
 that options with values can only be specified using the single argument
 notation. For example:
 
@@ -1271,5 +1761,6 @@ linux*-gcc* linux-gcc-prvinstall x86_64-linux-gnu \"all default gcc\" config.ins
 
 Note also that while building and running tests against the installation the
 worker makes the \c{bin} subdirectory of \c{config.install.root} the first
-entry in the \c{PATH} environment variable.
+entry in the \c{PATH} environment variable, except for build system modules
+which supposedly don't install any executables.
 "
-- 
cgit v1.1