0.3.0 release updates0.3.0

7 files changed, 258 insertions, 87 deletions

diff --git a/dist b/dist
index ad13a1f..56e24da 100755
--- a/dist
+++ b/dist
@@ -71,15 +71,15 @@ function dist()
 
 for t in $tools; do
   f=`dist $t`
-  mkdir -p cppget.org/repository/1/alpha/$t
-  cp $f cppget.org/repository/1/alpha/$t/
+  mkdir -p cppget.org/repository/1/queue/$t
+  cp $f cppget.org/repository/1/queue/$t/
 done
 
 dist build2-toolchain
 
 # Regenerate repository manifests.
 #
-cppget.org/update cppget.org/repository/1/
+cppget.org/update cppget.org/repository/1/queue
 
 cd $owd
 info "distribution in build2-$v/"
diff --git a/install b/install
index fe23261..6ee6896 100755
--- a/install
+++ b/install
@@ -107,8 +107,6 @@ while [ $# -gt 0 ]; do
   esac
 done
 
-echo "'$bpkgflags'"
-
 if [ -z "$tca" ]; then
   error $usage
 fi
@@ -164,12 +162,13 @@ tar xfz $tca
 #
 cd $tcb/build2
 ./bootstrap --cxx $cxx --cxxflags "$cxxflags"
-./build2/b-boot                         \
-  config.cxx=$cxx                       \
-  config.cxx.coptions="$cxxflags"       \
-  config.bin.rpath=$ins-boot/lib update
 cd ..
 
+./build2/build2/b-boot                      \
+  "!config.cxx=$cxx"                        \
+  "!config.cxx.coptions=$cxxflags"          \
+  "!config.bin.rpath=$ins-boot/lib" build2/
+
 ./build2/build2/b                                \
   config.cxx=$cxx                                \
   config.cxx.coptions="$cxxflags"                \
@@ -180,7 +179,7 @@ cd ..
   config.install.root.sudo=$sudo                 \
   configure
 
-./build2/build2/b update
+./build2/build2/b
 ./build2/build2/b install
 
 function show () # <dir> <prog>...
diff --git a/release.txt b/release.txt
index 6f9aec8..cb941fd 100644
--- a/release.txt
+++ b/release.txt
@@ -1,62 +1,62 @@
 TODO
 ====
 
-@@ Need to test upgradability via bpkg (create config with previous version
-   of toolchain, upgrade).
-
-@@ Looks like we need a public staging host.
 @@ Local Clang test is broken and disabled, using FreeBSD for Clang testing
 
-
 Notes
-====
+=====
 
-- Be very careful with submodules if fixing anything, remember to update
+* Be very careful with submodules if fixing anything, remember to update
   build2-toolchain.
 
 
 Setup
 =====
 
-- Make symlinks to development b, bpkg in /usr/local/bin/, used as the latest
+* Make symlinks to development b, bpkg in /usr/local/bin/, used as the latest
   toolchain.
 
-- Boot cppget1, freebsd VMs, make sure data/time is correct.
-
-- Set passwordless sudo, logins to freebsd, cppget1, rbook (Mac OS)
+* Set passwordless sudo, logins to freebsd, cppget1, rbook (Mac OS)
 
   For Mac OS:
     echo "boris ALL=(ALL) NOPASSWD:ALL" >>/etc/sudoers
-    #echo "192.168.111.103 cppget1 pkg.cppget1" >>/etc/hosts
-    #echo "192.168.0.19 protem" >>/etc/hosts
-    curl --resolve 'pkg.cppget1:8080:192.168.0.19' http://pkg.cppget1:8080/1/
-    iptables -t nat -A PREROUTING -d 192.168.0.19 -p tcp --dport 8080 -j DNAT --to-destination 192.168.111.103:80
 
-- Copy prerequisites into build2-X.Y/, generate sha256 sums in BINARY (-b)
+- Boot cppget1, freebsd VMs, make sure data/time is correct.
+
++ Copy prerequisites into build2-X.Y/, generate sha256 sums in BINARY (-b)
 
-- Build new prerequisites (if any) on local, freebsd (install to /usr/local).
++ Build new prerequisites (if any) on local, freebsd (install to /usr/local).
 
++ If new preprequisites/version, need to update manifests.
 
 Prelude
 =======
 
-- Review '@@' items [note: etc, private excluded]
++ Remove etc/ from modules in etc/git/modules.
+
++ Review '@@' items [note: etc, private excluded]
 
   etc/review | less -R
 
-- Increment version to final.
++ Need to make sure Intro is still accurate wrt output, etc. Use -s switch
+  to etc/intro to compare.
 
-  * Need to update versions/dates for man generation.
++ Increment version to final.
 
-- Close schema versions, review schema changlog difference from previous
-  release (tag) for any data migration that may be required (@@ Would also
-  need to test this).
+  * Need to update versions/dates for man generation (cli.sh in build2,
+    bpkg, brep, and built2-toolchain).
 
-- Need to update submodules? Remember to push first.
+  * Review manifests for any other changes (new entries/files, etc).
 
-- Need to regenerate ODB files, CLI documentation, in build2-toolchin!
++ Close schema versions, review schema changlog difference from previous
+  release (tag) for any data migration that may be required (@@ would also
+  need to test this; see upgrade test).
 
-- Update everything:
++ Need to update submodules? Remember to push first.
+
++ Need to regenerate ODB files, CLI documentation, in build2-toolchin!
+
++ Update everything:
 
   b libbutl/ build2/ libbpkg/ bpkg/ brep/ build2-toolchain-default/
 
@@ -66,20 +66,18 @@ Prelude
 
   * Manually in build2/
 
-  * Manually in bpkg/; test valgrind, publish, test remote with -v, FreeBSD
-    tools:
+  * Manually in bpkg/; test (valgrind/asan), publish, test remote with -v,
+    FreeBSD tools:
 
     ./test.sh -v --remote --fetch fetch --fetch-option --no-verify-peer \
       --tar bsdtar --sha256 sha256-freebsd
 
-  * Update hello repository:
+  * Update cppget repository (existing packages, not publishing yet):
 
-    cd hello
-    for d in hello libhello libprint libformat repository; do \
-      git -C $d status; done
-    ./release -a
+    @@ What is the purpose of this step? It will mess up upgrade test
+    below.
 
-  * Update cppget repository (existing packages, not publishing yet):
+    -t /home/boris/work/build2/etc1/install/0.2.0 -e queue
 
     cd cppget.org
     git -C repository status
@@ -87,8 +85,9 @@ Prelude
 
     This might not work in the early stage (backwards-incompatible)
 
-    ./test -c cxx -c config.cxx.loptions=-L/usr/local/lib \
-      -c config.cxx.poptions="-I/usr/include/apache2 -I/usr/include/apr-1.0" \
+    ./test -c cxx \
+    -c config.cxx.loptions=-L/usr/local/lib \
+    -c config.cxx.poptions="-I/usr/include/apache2 -I/usr/include/apr-1.0" \
       repository/1/
 
   * Test dev integration of brep/ on hello and cppget repos
@@ -106,49 +105,93 @@ Prelude
     brep/load/brep-load cppget.org/brep-loader-home.conf
     sudo /etc/init.d/apache2 restart
 
+    If need to drop the entire database, one way to do it is by executing:
+
+    DROP OWNED BY <user>;
+
 Procession
 ==========
 
-- Generate distribution (use -t for toolchain-only, without brep)
+- Generate distribution (use -t for toolchain-only, without brep). The
+  result is placed into the queue.
 
   etc/dist 2>&1 | tee dist.log
 
-- Stage packages to cppget1 host
+- @@ Would be nice to test packages for missing files, etc., before upgrade.
 
-  etc/stage
+- Test upgrade from previous version (replace Y in 0.Y.0).
+
+  etc/test-upgrade -c cxx \
+    -c config.cxx.loptions=-L/usr/local/lib \
+    -c config.cxx.poptions="-I/usr/include/apache2 -I/usr/include/apr-1.0" \
+    /home/boris/work/build2/etc1/install/0.Y.0
+
+- Update and test hello repository
+
+  cd hello
+  for d in hello libhello libprint libformat repository; do \
+    git -C $d status; done
+  ./release -a
+
+- Test Intro steps on the local hello repository
+
+  etc/intro `pwd`/hello/repository/1/
+
+- Publish hello repository, commit but don't push.
+
+  [We need to publish now because etc/test uses it.]
 
-- Determine the earliest supported toolchain (see requires: in manifests),
-  update etc/test
+  @@ Ideally we would want to stage it.
 
-- Publish hello repository [@@ Ideally we would want to stage it. Perhaps
-  on cppget1?]
+  ./release -p
 
-- Test Intro steps:
+- Test Intro steps on the remote hello repository
 
   etc/intro https://build2.org/pkg/1/hello/
 
+- Stage queue to cppget.org
+
+  etc/stage --dry-run
+  etc/stage
+
+- Regenerate other cppget.org sections with the new toolchain
+
+  cd cppget.org
+  git -C repository status
+  ./update -e queue repository/1/
+
+- Determine the earliest supported backwards-compatible toolchain (see
+  requires: in manifests), update etc/test
+
 - Test
 
   etc/test 2>&1 | tee test.log
   grep -i warning test.log
 
-- Upgrade brep on cppget1 (using pkg.cppget1), verify works
+- Upgrade brep on build2.org (via package/queue upgrade if possible),
+  verify works.
 
-- Publish to production (build2.org/cppget.org)
+- Move packages from queue to alpha, regenerate repositories.
+
+  ./update repository/1/
+
+- Publish to production (build2.org/cppget.org).
 
   etc/publish
 
 - Test production and save a copy of toolchain in etc1/install/X.Y.Z
 
   etc/install -t -i "etc1/install/`cat build2-toolchain/version`" \
-    https://download.build2.org/0.Y/build2-toolchain-0.2.0.tar.gz \
+    https://download.build2.org/0.Y/build2-toolchain-0.Y.0.tar.gz \
     https://pkg.cppget.org/1/alpha
 
-etc/install -t -i "etc1/install/`cat build2-toolchain/version`" \
-    https://download.build2.org/0.2/build2-toolchain-0.2.0.tar.gz \
-    https://pkg.cppget.org/1/alpha
+- Upgrade brep on cppget.org (via package/alpha upgrade if possible),
+  verify works.
 
-- Upgrade brep on cppget.org (using pkg.cppget.org), verify works
+  Note that here we also have queue:
+
+  install/bin/brep-migrate -n brep_queue
+  install/bin/brep-load -n brep_queue config/brep-queue-load.conf
 
 - Tag (review tag_modules)
 
@@ -160,7 +203,32 @@ etc/install -t -i "etc1/install/`cat build2-toolchain/version`" \
 Release
 =======
 
-cat `ls -1 *.sha256`
+- Update style/ submodule in private/
+
+- Write release notes, use placeholder for announcement URL. Add link to
+  the doc page.
+
+- Any new FAQ entries or any other updates (main page, etc)?
+
+- Update download page.
+
+- Regenerate (./cli.sh) all hosts.
+
+- Test local, publish, test production:
+
+  cd private/
+  ./publish --dry-run
+  ./publish
+
+- Write and send announcements
+
+  Remember upgrade instructions.
+
+  cat `ls -1 *.sha256`
+
+- Patch the announcement URL in release notes, re-publish.
+
+- Announce on reddit and other places (doc/ann).
 
 Postlude
 ========
@@ -170,4 +238,6 @@ Postlude
 - Increment and open schema versions. Maybe not. Maybe we should only do that
   when there is a model change.
 
-- Increment project versions to alpha1
+- Increment project versions to alpha1, open master for business
+
+- Commit changes to etc/, add it back to etc/git/modules.
diff --git a/stage b/stage
index 98e5aac..15d061d 100755
--- a/stage
+++ b/stage
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 
-# Stage build2 to cppget1 host.
+# Stage packages in queue to cppget.org/queue.cppget.org.
 #
 # Usage: publish [<rsync-options>]
 #
@@ -8,4 +8,12 @@ usage="$0 [<rsync-options>]"
 
 trap 'exit 1' ERR
 
-cppget.org/publish cppget.org/repository/1/ cppget1:/var/bpkg/1/ $*
+cppget.org/publish \
+  cppget.org/repository/1/queue/ \
+  cppget.org:/var/bpkg/1/queue/ \
+  $*
+
+cppget.org/publish \
+  cppget.org/repository/1/queue/ \
+  queue.cppget.org:/var/bpkg/1/queue/ \
+  $*
diff --git a/test b/test
index 73dd613..8c41daa 100755
--- a/test
+++ b/test
@@ -24,23 +24,24 @@ fi
 
 src=build2-$v
 
+#if false; then
+
 # Update the development build.
 #
 info "making sure everythings is up to date..."
 b build2/ bpkg/
 
-# Test repository with the earliest and latest (development build via
+
+# Test queue with the earliest and latest (as development build via
 # /usr/local links) toolchains.
 #
-# etc1/install/0.1.0
-#
-for t in /usr/local; do
-  cppget.org/test -t $t -n \
+for t in etc1/install/0.2.0 /usr/local; do
+  cppget.org/test -t $t \
 -c cxx \
 -c config.cxx.coptions="-W -Wall -Wno-unknown-pragmas" \
 -c config.cxx.poptions="-I/usr/include/apache2 -I/usr/include/apr-1.0" \
 -c config.cxx.loptions=-L/usr/local/lib \
-cppget.org/repository/1/
+cppget.org/repository/1/queue/
 done
 
 # Test the build2-toolchain INSTALL procedure (plus brep) with the earliest
@@ -54,7 +55,7 @@ for c in g++-4.8 g++-4.9 g++-5; do
 --cxx $c \
 --cppflags "-I/usr/include/apache2 -I/usr/include/apr-1.0" \
 $src/build2-toolchain-$tcv.tar.gz \
-http://pkg.cppget1/1/alpha
+https://pkg.cppget.org/1/queue
 done
 
 # This doesn't work since libodb, etc are using libstdc++.
@@ -66,14 +67,14 @@ for c in clang++-3.5 clang++-3.6; do
 --cppflags "-I/usr/include/apache2 -I/usr/include/apr-1.0" \
 --cxxflags --stdlib=libc++ \
 $src/build2-toolchain-$tcv.tar.gz \
-http://pkg.cppget1/1/alpha
+https://pkg.cppget.org/1/queue
 done
 fi
 
 # Test installation requiring sudo.
 #
 etc/install -t -i /opt/build2 -s $src/build2-toolchain-$tcv.tar.gz \
-http://pkg.cppget1/1/alpha
+https://pkg.cppget.org/1/queue
 
 # Test on FreeBSD with Clang 3.4 (default) and Clang 3.7.
 #
@@ -83,7 +84,7 @@ for c in clang++ clang++37; do
   ssh freebsd cd /tmp ';' ./install \
 --cxx $c \
 --cppflags '"-I/usr/local/include/apr-1 -I/usr/local/include/apache24"' \
-build2-toolchain-$tcv.tar.gz http://pkg.cppget1/1/alpha
+build2-toolchain-$tcv.tar.gz https://pkg.cppget.org/1/queue
 done
 
 # Also run the Intro steps based on the toolchain we just built.
@@ -98,13 +99,13 @@ ssh freebsd cd /tmp ';' ./intro \
 -h /tmp/hello2                  \
 https://build2.org/pkg/1/hello/
 
+
 # Test on Mac OS (no brep).
 #
 scp etc/install $src/build2-toolchain-$tcv.tar.gz rbook:/tmp/
 
-ssh rbook cd /tmp ';' ./install -t --cxx clang++ --bpkgflags \
-"'--fetch-option --resolve --fetch-option pkg.cppget1:8080:192.168.0.19'" \
-build2-toolchain-$tcv.tar.gz http://pkg.cppget1:8080/1/alpha
+ssh rbook cd /tmp ';' ./install -t --cxx clang++ \
+  build2-toolchain-$tcv.tar.gz https://pkg.cppget.org/1/queue
 
 # Again, run the Intro steps based on the toolchain we just built.
 #
diff --git a/test-upgrade b/test-upgrade
new file mode 100755
index 0000000..29705da
--- /dev/null
+++ b/test-upgrade
@@ -0,0 +1,93 @@
+#! /usr/bin/env bash
+
+# Test build2 toolchain (and brep) upgrade. The old packages come from
+# cppget.org/alpha and the new ones from cppget.org/queue. Run from build2/
+# root.
+#
+# Usage: test-upgrade [options] <old-toolchain>
+#
+# -c <option>|<module>|<var>
+#    Specify additional options, modules, or configuration variables to pass
+#    to the bpkg create command. For example:
+#
+usage="usage: $0 -t <old-toolchain>"
+
+owd=`pwd`
+trap "{ cd $owd; exit 1; }" ERR
+set -o errtrace # Trap in functions.
+
+function info () { echo "$*" 1>&2; }
+function error () { info "$*"; exit 1; }
+
+cfg=/tmp/upgrade-cfg
+ins=/tmp/upgrade-install
+toolchain=
+co=() # Use a bash array to handle arguments with spaces (say "-Ifoo -Ibar").
+
+while [ $# -gt 0 ]; do
+  case $1 in
+    -c)
+      shift
+      co=("${co[@]}" "$1")
+      shift
+      ;;
+    *)
+      toolchain=${1%/}
+      shift
+      break
+      ;;
+  esac
+done
+
+if [ -z "$toolchain" ]; then
+  error $usage
+fi
+
+ob="$toolchain/bin/b"
+if [ ! -x $ob ]; then
+  error "$ob does not exist or is not executable"
+fi
+obo="--build $ob"
+
+obpkg="$toolchain/bin/bpkg"
+if [ ! -x $obpkg ]; then
+  error "$obpkg does not exist or is not executable"
+fi
+
+# First create the configuration and build old packages using the old
+# toolchain.
+#
+$obpkg $obo create -d $cfg --wipe "${co[@]}"
+$obpkg add -d $cfg cppget.org/repository/1/alpha
+$obpkg fetch -d $cfg
+$obpkg $obo build -d $cfg --yes build2 bpkg brep
+
+# Now upgrade to new packages but still using the old toolchain.
+#
+$obpkg add -d $cfg cppget.org/repository/1/queue
+$obpkg fetch -d $cfg
+$obpkg $obo build -d $cfg --yes build2 bpkg brep
+
+# Install the new packages still using the old toolchain.
+#
+$obpkg $obo install -d $cfg config.install.root=$ins \
+  config.bin.rpath=$ins/lib build2 bpkg brep
+
+$ins/bin/b --version 1>&2
+$ins/bin/bpkg --version 1>&2
+$ins/bin/brep-load --version 1>&2
+
+# Check the use of the new toolchain on the old configuration.
+#
+nb="$ins/bin/b"
+nbo="--build $nb"
+nbpkg="$ins/bin/bpkg"
+
+$nbpkg $nbo clean -d $cfg build2 bpkg brep
+$nbpkg $nbo build -d $cfg --yes build2 bpkg brep
+$nbpkg $nbo install -d $cfg config.install.root=$ins \
+  config.bin.rpath=$ins/lib build2 bpkg brep
+
+$ins/bin/b --version 1>&2
+$ins/bin/bpkg --version 1>&2
+$ins/bin/brep-load --version 1>&2
diff --git a/version b/version
index 042eada..e99e175 100755
--- a/version
+++ b/version
@@ -14,31 +14,31 @@ usage="usage: $0 [<module>...]"
 #   currently, max can only be 1.2.3- (#if-conditions will need adjustment)
 #
 
-build2_min=0.2.0 # No max, always >=.
+build2_min=0.3.0 # No max, always >=.
 
-libbutl=0.3.0-a1
+libbutl=0.4.0-a1
 libbutl_min=$libbutl
 libbutl_max=$libbutl
-#libbutl_max=0.4.0-
+#libbutl_max=0.5.0-              # Comment if pre-release.
 libbutl_build2=$build2_min
 libbutl_hxx=butl/version
 
-build2=0.3.0-a1
+build2=0.4.0-a1
 build2_libbutl_min=$libbutl_min
 build2_libbutl_max=$libbutl_max
 build2_build2=$build2_min
 build2_hxx=build2/version
 
-libbpkg=0.3.0-a1
+libbpkg=0.4.0-a1
 libbpkg_min=$libbpkg
 libbpkg_max=$libbpkg
-#libbpkg_max=0.4.0-
+#libbpkg_max=0.5.0-             # Comment if pre-release.
 libbpkg_libbutl_min=$libbutl_min
 libbpkg_libbutl_max=$libbutl_max
 libbpkg_build2=$build2_min
 libbpkg_hxx=bpkg/version
 
-bpkg=0.3.0-a1
+bpkg=0.4.0-a1
 bpkg_libbutl_min=$libbutl_min
 bpkg_libbutl_max=$libbutl_max
 bpkg_libbpkg_min=$libbpkg_min
@@ -46,10 +46,10 @@ bpkg_libbpkg_max=$libbpkg_max
 bpkg_build2=$build2_min
 bpkg_hxx=bpkg/bpkg-version
 
-build2_toolchain=0.3.0-a1
+build2_toolchain=0.4.0-a1
 build2_toolchain_build2=$build2_min
 
-brep=0.3.0-a1
+brep=0.4.0-a1
 brep_libbutl_min=$libbutl_min
 brep_libbutl_max=$libbutl_max
 brep_libbpkg_min=$libbpkg_min