diff options
| -rw-r--r-- | git/modules | 15 | ||||
| -rwxr-xr-x | publish | 21 | ||||
| -rw-r--r-- | release.txt | 402 | ||||
| -rwxr-xr-x | stage | 167 | ||||
| -rwxr-xr-x | stage-pkg | 122 | ||||
| -rwxr-xr-x | tag | 45 | ||||
| -rwxr-xr-x | version | 329 |
7 files changed, 441 insertions, 660 deletions
diff --git a/git/modules b/git/modules index 39ac99a..b512991 100644 --- a/git/modules +++ b/git/modules @@ -1,6 +1,5 @@ modules=" \ git \ -private \ change \ libbutl \ build2 \ @@ -9,15 +8,17 @@ bpkg \ brep \ libbbot \ bbot \ -buildos" +buildos \ +msvc-linux" # During release it is useful to exclude etc/ and/or build2-toolchain so # can do mass commits (like version updates). # -modules="$modules build2-toolchain" modules="$modules etc" +modules="$modules private" +modules="$modules build2-toolchain" -# We don't tag etc/ and private/. +# We don't tag git/, etc/, and private/. # tag_modules=" \ libbutl \ @@ -28,9 +29,10 @@ brep \ libbbot \ bbot \ buildos \ +msvc-linux \ build2-toolchain" -# build2-toolchain has to be done manually. +# Submodule update in build2-toolchain has to be done manually. # sub_modules=" \ private \ @@ -41,4 +43,5 @@ bpkg \ brep \ libbbot \ bbot \ -buildos" +buildos \ +msvc-linux" @@ -2,7 +2,7 @@ # Publish build2 to build2.org/cppget.org (and brep.cppget.org). # -# The distribution is taken from staging/0/ and packages from +# The distribution is taken from cppget.org/0/ and packages from # cppget.org/repository/. # # Usage: publish [<rsync-options>] @@ -16,24 +16,21 @@ set -o errtrace # Trap in functions. function info () { echo "$*" 1>&2; } function error () { info "$*"; exit 1; } -v="$(cat build2-toolchain/version)" -vm="$(echo $v | sed -e 's/^\(.*\)\.\(.*\)\..*$/\1.\2/')" # X.Y +v="$(sed -n -re 's/^version: ([^.]+\.[^.]+\.[^-]+(-[ab]\.[^.+]+)?).*$/\1/p' build2-toolchain/manifest)" -if [ -z "$v" -o -z "$vm" ]; then - error "unable to extract version from `cat build2-toolchain/version`" -fi - -d="staging/0/$v" - -if [ ! -d "$d" ]; then - error "distribution directory $d does not exist" +if [ -z "$v" ]; then + error "unable to extract version build2-toolchain/manifest" fi function sync () { info "build2.org:" + + rsync -v -rlpt -c --copy-unsafe-links --prune-empty-dirs --delete-after \ +"${@}" "cppget.org/0/$v/" "build2.org:/var/www/download.build2.org/public/$v/" + rsync -v -rlpt -c --copy-unsafe-links --prune-empty-dirs --delete-after \ -"${@}" "$d/" "build2.org:/var/www/download.build2.org/public/$vm/" +"${@}" "cppget.org/0/toolchain.sha256" "build2.org:/var/www/download.build2.org/public/" info "cppget.org:" etc/rep-publish cppget.org/repository/1/ cppget.org:/var/bpkg/1/ "${@}" diff --git a/release.txt b/release.txt index ad06109..5bdead3 100644 --- a/release.txt +++ b/release.txt @@ -1,363 +1,257 @@ -TODO -==== +@@ No upgrade testing -@@ No upgrade testing (see old test-upgrade script). +- Replace 0.5.0 and 0.5 in this document with the new version. -@@ Not saving in previous toolchains (see old install script). +- Remove etc/ and private/ from modules in etc/git/modules to reduce + noise during stat. Also, review for any new modules. -@@ Using FreeBSD for Clang/libc++ testing, both 3.4 (earliest we support) - and 3.7. +- Review '@@' items [private/ excluded, at least look for @@ TMP] -Setup -===== - -* Make symlinks to development b, bpkg in /usr/local/bin/, used as the latest - toolchain. - -* Set passwordless sudo, logins to freebsd, cppget1, rbook (Mac OS) - - For Mac OS: - echo "boris ALL=(ALL) NOPASSWD:ALL" >>/etc/sudoers - -- Boot freebsd VM, make sure data/time is correct. - -Notes -===== - -* Be very careful with submodules if fixing anything, remember to update - build2-toolchain. - -* Be very carefult with regenerating documentation, ODB files, both in - individual projects and build2-toolchain. - -Stage -===== - -- To stage build2 - - ~ make sure build2-toolchain is ready (submodules, doc/odb regen) - ~ update BUILD2_REPO URL in build2-toolchain/build* scripts if needed - - To stage to stage.build2.org: - - ~ to exclude baseutils/mingw, add -p - - etc/stage + etc/review | less -R - To stage to cppget.org/queue: +- Identify packages that will be released (see etc/stage). They should all be + staged and built successfully, including submodule-updated build2-toolchain + and baseutils/mingw. Review stageing/0/ and staging/1/ for anything stray. - ~ note: assuming brep running on queue can handle it +- Update and test local builds: - etc/stage -q + b test: build2/ bpkg/ brep/ bbot/ libbutl/ libbpkg/ libbbot/ + b msvc-linux/ -- To stage hello +- Stage hello projects and test: cd hello - ./stat + ./foreach-git pull + ./foreach-git stat ./stage - To run intro on staged: - - etc/intro -s https://build2.org/pkg/1/stage - -Test : - - + Local with cli disables - - + Local with Intel icc - ~ fails with ICE on build2 - - + Local with gcc 4.8 - - + Local brep build: - - bpkg create -d brep cc config.cc.poptions="-I$(apxs -q includedir)" - cd brep - bpkg add https://stage.build2.org/1 - bpkg fetch - bpkg build brep ?sys:libapr1 ?sys:libpq - -Cross: - - + Test VC cross-build - - + Test MinGW cross-build (build2 build fails on GC 4.8.2, using ODB cross) - - -Bootstrap (with upgrade procedures): - - + CentOS GCC 4.8 - - + FreeBSD 10 Clang 3.4 (clang++) - - + FreeBSD 10 Clang 3.7 (clang++37) with -W -Wall (mod build.sh) + brep: - - bpkg create -d brep cc config.cxx=clang++37 \ - config.cc.poptions="-I$(apxs -q includedir)" - cd brep - bpkg add https://stage.build2.org/1 - bpkg fetch - bpkg build brep ?sys:libapr1 ?sys:libpq - - + Windows7-64 VC14u2 - - + WindowsXP-32 MinGW (build2-mingw) - - + Windows MSYS2 - - + Mac OS Xcode/Clang (and g++ alias) - -Upgrade brep on stage - -Prelude -======= - -+ Remove etc/, build2-toolchain from modules in etc/git/modules. - -+ Review '@@' items [note: etc, private excluded] - - etc/review | less -R - -+ Make sure Intro is still accurate wrt output, etc. Use -s -p switches to - etc/intro to compare (after running hello/stage): - + cd .. etc/intro -s `pwd`/hello/repository/1/ + etc/intro -s https://build2.org/pkg/1/stage -+ If new preprequisites/version, need to update manifests. See - private/build2-projects.txt to review. - -+ Increment version to final. +- Update NEWS files in all project that will have non-pre-release version. - ~ update and use etc/version script +- Review documentation for (1) sample output changes and (2) still being + relevant/making sense: - ~ update versions/dates for man generation (cli.sh in build2, bpkg, brep, - and built2-toolchain) + - Testscript manual (for 1, see hello/hello-testscript). -+ Update BUILD2_REPO URL in build2-toolchain/build* scripts to alpha, also - grep docs for 'stage.' in case got copied. + - Install guide for (for 1 & 2). -+ Close schema versions, review schema changlog difference from previous - release (tag) for any data migration that may be required (@@ would also - need to test this; see upgrade test). + - Introduction for (for 1 & 2). Use intro script to get output: -+ Update NEWS files. + etc/intro -s -p https://build2.org/pkg/1/stage -+ Need to update submodules? (remember to push first) +- Change to final versions for all packages being released, some by hand, + some with version scripts (see etc/stage for list + msvc-linux). -+ Need to regenerate ODB files, CLI documentation, in build2-toolchin? + - Close schema versions. Review schema changlog difference from previous + release (tag) for any data migration that may be required (would also + need to test this). -Procession -========== + - Push and update all submodules in buil2-toolchain -+ Stage final packages (remove -p if need baseutils/mingw) + - Regenerate all odb & docs (in both packages and inside build2-toolchain) - etc/stage -p + - Bootstrap then update and test local -- Upgrade brep on lists.build2, lists.cppget from stage + @@ Need to manually clean old the project builds because they + pick up old version.hxx (during dep extraction). - ~ assumming it can handle old repositories (should normally be the case) + cd build2 + b clean + git cout build2/version.hxx build2/b-options.?xx + ./bootstrap.sh g++-6 - ~ backup config if changed + cd .. + b-boot '{clean update}(libbutl/ build2/)' - cd private/ - scp -r lists.build2.org:/home/brep/config lists.build2.org/home/brep/ - scp -r lists.cppget.org:/home/brep/config lists.cppget.org/home/brep/ + b clean: bpkg/ brep/ bbot/ libbpkg/ libbbot/ msvc-linux/ - ~ note: on cppget we have queue: + Then the local test step above. - install/bin/brep-migrate -n brep_queue - install/bin/brep-load -n brep_queue config/brep-queue-load.conf + - Cleanup staging repo: -+ Queue final packages + rm -r staging/0/* + rm -r staging/repository/1/*/ - etc/stage -p -q + - Restage (step 1 above, with -b): -+ Smoke-test any last-minute changes + etc/stage -b -+ Move packages from queue to alpha, cleanup obsolete, regenerate. + - Check /0/ and /1/ for anything stray - cd cppget.org - git -C repository status - ./update +- Upgrade brep on cppget.org (queue and main repo) using stage -+ Publish to production (build2.org/cppget.org) +- Queue packages: - etc/publish + - Change BUILD2_REPO in build2_toolchain build script to queue -+ Smoke-test + - Cleanup queue repo: -+ Publish hello repository + rm -rf cppget.org/0/* + rm -rf cppget.org/repository/1/queue/*/ - cd hello - ./stage -p + - Queue - Test intro steps: + etc/stage -q -b - etc/intro -s https://build2.org/pkg/1/hello + - check /0/ and /1/ for anything stray -- Commit and push git repositories: + - Make sure bot builds are successful -+ ~ cppget.org (commit individually and push) +- Upgrade brep on build2.org (hello) using queue - ~ hello project and respository (add & commit individual packages) +- Publish and test hello cd hello ./foreach-git stat + ./stage -p - ./foreach-git -x libhello-1.0 tag -a build2-X.Y -m "Tag for build2 version X.Y" - git -C libhello-1.0 tag -a 1.0/build2-X.Y -m "Tag for build2 version X.Y" - ./foreach-git push --tags + cd .. + etc/intro -s https://build2.org/pkg/1/hello -+ Tag (review tag_modules) and push. + - Drop build database (package versions do not change) - etc/tag - ./push.sh + - Make sure bot builds are successful -- Tag and increment version for each published package (see - private/build2-projects.txt) +- Publish and test toolchain -============================================================================= + - Change BUILD2_REPO in build2_toolchain build script to public -@@ When copying to production need to make sure we don't override existing - versions of prerequisite packages that haven't changed (but which we still - stage/queue). + - Regenerate build2-toolchain package in cppget.org/0/ -============================================================================= + etc/stage -p -@@ Move up. + - Move packages (manually for now): -- Update everything: + - old/replaced/FTB packages either to legacy or delete - b libbutl/ build2/ libbpkg/ bpkg/ brep/ build2-toolchain-default/ + - from queue to appropriate repositories (don't override existing) -- Run tests for each project: + - Regenerate the repository - * b 'test(libbutl/ libbpkg/)' + cd cppget.org + git -C repository add . + git -C repository status + ./update - * Manually in build2/ + - Disable the queue toolchain on bbots (moved packages from queue/), + delete distribution - * Manually in bpkg/; test (valgrind/asan), publish, test remote with -v, - FreeBSD tools: + echo disabled | ssh build2.org tee >/dev/null \ + /var/www/download.build2.org/public/queue/toolchain.sha256 - ./test.sh -v --remote --fetch fetch --fetch-option --no-verify-peer \ - --tar bsdtar --sha256 sha256-freebsd + ssh build2.org rm -r /var/www/download.build2.org/public/queue/*/ - * Test dev integration of brep/ on hello and cppget repos + - Publish everything to build2.org/cppget.org - First check if Apache2 configuration needs updating (INSTALL-DEV). Then - from build2/ work root: + etc/publish - # This will have broken links since we are not running it as /pkg/hello. - # - brep/migrate/brep-migrate --recreate - brep/load/brep-load hello/brep-loader-home.conf - sudo /etc/init.d/apache2 restart + - Make sure bot builds are successful, drop hello build database again. - brep/migrate/brep-migrate --recreate - brep/load/brep-load cppget.org/brep-loader-home.conf - sudo /etc/init.d/apache2 restart +- Release - If need to drop the entire database, one way to do it is by executing: + - Update submodules (style/) in private/. - DROP OWNED BY <user>; + - Write release notes, use placeholder for announcement URL. Add link from + the doc page. -Procession -========== + - Any new FAQ entries or any other updates (main page, etc)? Any new + documentation to link from the doc page? -X Generate distribution (use -t for toolchain-only, without brep). The - result is placed into the queue. + - Update download page. - etc/dist 2>&1 | tee dist.log + cat `ls -1 *.sha256` -X Test upgrade from previous version (replace Y in 0.Y.0). + - Regenerate documentation (./cli.sh) for all hosts. - etc/test-upgrade -c cxx \ - -c config.cxx.loptions=-L/usr/local/lib \ - -c config.cxx.poptions="-I/usr/include/apache2 -I/usr/include/apr-1.0" \ - /home/boris/work/build2/etc1/install/0.Y.0 + - Test locally, publish, test production -X Update and test hello repository + cd private/ + ./publish --dry-run + ./publish - cd hello - ./foreach-git status - ./stage -p + - Write and send the announcement to mailing lists. -X Test Intro steps on the local hello repository + Remember upgrade instructions. - etc/intro `pwd`/hello/repository/1/ + cat `ls -1 *.sha256` -X Stage queue to cppget.org + - Patch the announcement URL in release notes, re-publish. - etc/stage --dry-run - etc/stage + - Announce on reddit and other places (see doc/ann/). -- Determine the earliest supported backwards-compatible toolchain (see - requires: in manifests), update etc/test + - Commit and tag private/ -- Test + git tag -a 0.5.0 -m "Tag version 0.5.0" && git push --tags - etc/test 2>&1 | tee test.log - grep -i warning test.log +- Tag & Commit hello + + - commit hello/repository/ (see commit history for procedure) + + cd hello + ./foreach-git stat + ./foreach-git -x libhello-1.0 tag -a build2-0.5.0 -m "Tag for build2 version 0.5.0" + git -C libhello-1.0 tag -a 1.0/build2-0.5.0 -m "Tag for build2 version 0.5.0" + ./foreach-git push --tags -- Test production and save a copy of toolchain in etc1/install/X.Y.Z +- Tag & Commit - etc/install -t -i "etc1/install/`cat build2-toolchain/version`" \ - https://download.build2.org/0.Y/build2-toolchain-0.Y.0.tar.gz \ - https://pkg.cppget.org/1/alpha + - commit cppget.org/repository/ (see commit history for procedure) -- Upgrade brep on cppget.org (via package/alpha upgrade if possible), - verify works. + - Tag and push all released packages, some by hand, some with tag scripts + (see etc/stage for list). - Note that here we also have queue: + cd <proj> - install/bin/brep-migrate -n brep_queue - install/bin/brep-load -n brep_queue config/brep-queue-load.conf + v="$(sed -n -re 's/version: ([^ ]+)/\1/p' manifest)"; echo $v; read; \ + git tag -a $v -m "Tag version $v" && git push --tags - Backup configs if changed: + For build2 projects, review tag_modules in etc/git/modules, then: - cd private/ - scp -r lists.cppget.org:/home/brep/config lists.cppget.org/home/brep/ + ./tag.sh 0.5.0 + ./push.sh -Release -======= +- Increment versions and open master for business -+ Update style/ submodule in private/ + - Change to next development versions for all released packages, some by + hand, some with version scripts (see etc/stage for list + msvc-linux). -+ Write release notes, use placeholder for announcement URL. Add link from - the doc page. + cd <proj> -- Any new FAQ entries or any other updates (main page, etc)? + ~ edit manifest -+ Update download page. + v="$(sed -n -re 's/version: ([^ ]+)/\1/p' manifest)"; echo $v; read; \ + git ci -a -m "Bump version to $v, master is open for business" && git push -+ Regenerate (./cli.sh) all hosts. + - In build2-toolchain, update all submodules (including libodb*, etc), + change BUILD2_REPO to staging. -+ Test local, publish, test production: + - Regenerate odb & cli docs everywhere (packages and build2-toolchain). - cd private/ - ./publish --dry-run - ./publish + - Update/test local (see early steps of this list). -+ Write and send announcements + - Make symlinks for new version in baseutils (for both baseutils and mingw, + the idea is that we will start with those and maybe upgrade during + development). Update version file. - Remember upgrade instructions. + - Restage - cat `ls -1 *.sha256` + rm -r staging/0/* + rm -r staging/repository/1/*/ -+ Patch the announcement URL in release notes, re-publish. + etc/stage -b -- Announce on reddit and other places (doc/ann). + - Make sure bbot builds of the new development snapshot are successful. -Postlude -======== +- Commit etc/ -- Increment and open schema versions. Maybe not. Maybe we should only do that - when there is a model change. + - Restore changes to etc/git/modules -+ Increment project versions to alpha1, change BUILD2_REPO to staging, - open master for business. + - Change all '+ ' back to '- '. -- Commit changes to etc/, add it back to etc/git/modules, and tag: + - Commit and tag - git tag -a X.Y.Z -m "Tag version X.Y.Z" + git tag -a 0.5.0 -m "Tag version 0.5.0" && git push --tags @@ -2,11 +2,13 @@ # Stage or queue build2 packages and distributions. # -# -p -# Packages only, no baseutils/mingw. +# -b +# Packages baseutils/mingw. # -q -# Put packages into the queue instead of staging (distributions still go to -# staging). +# Put packages into the queue instead of staging. +# -p +# Regenerate build2-toolchain into queue's 0/ and update the checksums file +# but don't do anything else. # usage="usage: etc/stage" @@ -19,31 +21,34 @@ set -o errtrace # Trap in functions. function info () { echo "$*" 1>&2; } function error () { info "$*"; exit 1; } -v="$(cat build2-toolchain/version)" +v="$(sed -n -re 's/^version: ([^.]+\.[^.]+\.[^-]+(-[ab]\.[^.+]+)?).*$/\1/p' build2-toolchain/manifest)" vf="$(echo $v | sed -re 's/^([^.]*)\.([^.]*)\.([^-]*).*$/\1.\2.\3/')" # Final. if [ -z "$v" -o -z "$vf" ]; then error "unable to extract version from build2-toolchain/version" fi -base="y" - -repo_name="STAGE.BUILD2.ORG" -repo_dir="staging/repository/1" -repo_host1="stage.build2.org:/var/bpkg/1" -repo_host2= +mode= +base= +dist_src=staging/0 +dist_dst=stage.build2.org:/var/www/stage.build2.org/public/0 while [ $# -gt 0 ]; do case $1 in - -p) - base=n + -b) + base=true shift ;; -q) - repo_name="CPPGET.ORG/QUEUE" - repo_dir="cppget.org/repository/1/queue" - repo_host1="cppget.org:/var/bpkg/1/queue" - repo_host2="queue.cppget.org:/var/bpkg/1/queue" + mode=-q + dist_src=cppget.org/0 + dist_dst=build2.org:/var/www/download.build2.org/public/queue + shift + ;; + -p) + mode=-p + dist_src=cppget.org/0 + dist_dst=example.org/not-used shift ;; *) @@ -52,11 +57,11 @@ while [ $# -gt 0 ]; do esac done -mkdir -p "staging/0/$v" +mkdir -p "$dist_src/$v" # Generate baseutils if requested. # -if [ "$base" = "y" ]; then +if [ -n "$base" ]; then cd private/baseutils echo "$v" >version @@ -79,17 +84,17 @@ if [ "$base" = "y" ]; then cd ../.. - mv "/tmp/build2-baseutils-$v-i686-windows.zip" "staging/0/$v/" - mv "/tmp/build2-baseutils-$v-i686-windows.zip.sha256" "staging/0/$v/" + mv "/tmp/build2-baseutils-$v-i686-windows.zip" "$dist_src/$v/" + mv "/tmp/build2-baseutils-$v-i686-windows.zip.sha256" "$dist_src/$v/" - mv "/tmp/build2-baseutils-$v-x86_64-windows.zip" "staging/0/$v/" - mv "/tmp/build2-baseutils-$v-x86_64-windows.zip.sha256" "staging/0/$v/" + mv "/tmp/build2-baseutils-$v-x86_64-windows.zip" "$dist_src/$v/" + mv "/tmp/build2-baseutils-$v-x86_64-windows.zip.sha256" "$dist_src/$v/" - mv "/tmp/build2-mingw-$v-i686-windows.tar.xz" "staging/0/$v/" - mv "/tmp/build2-mingw-$v-i686-windows.tar.xz.sha256" "staging/0/$v/" + mv "/tmp/build2-mingw-$v-i686-windows.tar.xz" "$dist_src/$v/" + mv "/tmp/build2-mingw-$v-i686-windows.tar.xz.sha256" "$dist_src/$v/" - mv "/tmp/build2-mingw-$v-x86_64-windows.tar.xz" "staging/0/$v/" - mv "/tmp/build2-mingw-$v-x86_64-windows.tar.xz.sha256" "staging/0/$v/" + mv "/tmp/build2-mingw-$v-x86_64-windows.tar.xz" "$dist_src/$v/" + mv "/tmp/build2-mingw-$v-x86_64-windows.tar.xz.sha256" "$dist_src/$v/" fi mkdir -p /tmp/dist @@ -98,12 +103,20 @@ mkdir -p /tmp/dist # # @@ .sha256 sums: make sure they don't contain full paths! +# Cleanup old stuff. +# +rm -f "$owd/$dist_src/$v"/build2-toolchain-* + b "dist(build2-toolchain-default/)" \ - "config.dist.archives=$owd/staging/0/$v/tar.xz $owd/staging/0/$v/tar.gz" + "config.dist.archives=$owd/$dist_src/$v/tar.xz $owd/$dist_src/$v/tar.gz" + +cd "$owd/$dist_src/$v/" + +tv="$(echo build2-toolchain-*.tar.gz | sed -re 's/build2-toolchain-(.+).tar.gz/\1/')" + +sha256sum -b "build2-toolchain-$tv.tar.gz" >"build2-toolchain-$tv.tar.gz.sha256" +sha256sum -b "build2-toolchain-$tv.tar.xz" >"build2-toolchain-$tv.tar.xz.sha256" -cd "$owd/staging/0/$v/" -sha256sum -b "build2-toolchain-$v.tar.gz" >"build2-toolchain-$v.tar.gz.sha256" -sha256sum -b "build2-toolchain-$v.tar.xz" >"build2-toolchain-$v.tar.xz.sha256" cd "$owd" # Prepare the toolchain.sha256 checksums file. @@ -111,72 +124,52 @@ cd "$owd" function checksum_line () # <file> { local s - s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' "staging/0/$v/$1.sha256")" - echo "$s *$v/$1" >>"staging/0/toolchain.sha256" + s="$(sed -n -re 's/^([^ ]+) .+$/\1/p' "$dist_src/$v/$1.sha256")" + echo "$s *$v/$1" >>"$dist_src/toolchain.sha256" } -echo "# $v" >"staging/0/toolchain.sha256" -checksum_line "build2-toolchain-$v.tar.xz" # Must be first. +echo "# $tv" >"$dist_src/toolchain.sha256" +checksum_line "build2-toolchain-$tv.tar.xz" # Must be first. checksum_line "build2-baseutils-$v-x86_64-windows.zip" -#checksum_line "build2-mingw-$v-x86_64-windows.tar.xz" +checksum_line "build2-mingw-$v-x86_64-windows.tar.xz" #checksum_line "build2-baseutils-$v-i686-windows.zip" #checksum_line "build2-mingw-$v-i686-windows.tar.xz" -# Dist individual packages into the repository. -# -function dist() # <dir> <category> -{ - local b="$(basename $1)" - local o="$repo_dir/$2" - - # Clean up old packages. - # - mkdir -p "$o" - rm -f "$o/$b"-* - - b "dist($1-default/)" "config.dist.archives=$owd/$o/tar.gz" -} - -dist packaging/apr/libapr1 apr -dist packaging/postgresql/libpq postgresql - -dist packaging/sqlite/libsqlite3 sqlite -dist packaging/sqlite/sqlite3 sqlite - -dist ../studxml/libstudxml studxml - -dist ../odb/builds/default/libodb odb -dist ../odb/builds/default/libodb-sqlite odb -dist ../odb/builds/default/libodb-pgsql odb - -dist libbutl build2 -dist build2 build2 -dist libbpkg build2 -dist bpkg build2 -dist brep build2 - -dist libbbot build2 -dist bbot build2 - -# Regenerate the repository. -# -info "Insert $repo_name signing key and press Enter" -read -etc/rep-update "$repo_dir/" \ - --openssl-option -engine --openssl-option pkcs11 \ - --openssl-option -keyform --openssl-option engine \ - --key "label_SIGN key" +if [ "$mode" = "-p" ]; then + exit 0 +fi -# Sync repository and distribution. +# Sync the distribution. # # Note that we do the distribution first so that the toolchain changes are # visible before package changes. This is relied upon by the Build OS monitor. # +info "Press Enter to start distribution upload" +read rsync -v -rlpt -c --copy-unsafe-links --prune-empty-dirs --delete-after \ - $rsync_ops staging/0/ stage.build2.org:/var/www/stage.build2.org/public/0/ + $rsync_ops "$dist_src/" "$dist_dst/" -etc/rep-publish "$repo_dir/" "$repo_host1/" $rsync_ops - -if [ -n "$repo_host2" ]; then - etc/rep-publish "$repo_dir/" "$repo_host2/" $rsync_ops -fi +# Distribute packages, regenerate and sync the repository. +# +etc/stage-pkg -c $mode -d apr packaging/apr/libapr1 +etc/stage-pkg -c $mode -d postgresql packaging/postgresql/libpq + +etc/stage-pkg $mode -d sqlite \ + packaging/sqlite/libsqlite3 \ + packaging/sqlite/sqlite3 + +etc/stage-pkg $mode -d studxml ../studxml/libstudxml + +etc/stage-pkg $mode -d odb \ + ../odb/builds/default/libodb \ + ../odb/builds/default/libodb-sqlite \ + ../odb/builds/default/libodb-pgsql + +etc/stage-pkg -c $mode build2 \ + libbutl \ + build2 \ + libbpkg \ + bpkg \ + libbbot \ + brep \ + bbot diff --git a/stage-pkg b/stage-pkg new file mode 100755 index 0000000..02b3e96 --- /dev/null +++ b/stage-pkg @@ -0,0 +1,122 @@ +#! /usr/bin/env bash + +# Stage or queue one or more projects from the same group. +# +# -c +# Cleanup (remove) previous versions. +# +# -d +# Distribute only without regenerating or publishing the repository. +# +# -g +# Distribute and regenerating only without publishing the repository. +# +# -q +# Publish packages into the queue instead of staging. +# +usage="usage: etc/stage-pkg [<options>] <group> <dir>..." + +rsync_ops="--progress" + +owd=`pwd` +trap "{ cd $owd; exit 1; }" ERR +set -o errtrace # Trap in functions. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +repo_name="STAGE.BUILD2.ORG" +repo_dir="staging/repository/1" +repo_host1="stage.build2.org:/var/bpkg/1" +repo_host2= + +clean= +dist_only= +gen_only= +group= +dirs=() + +while [ $# -gt 0 ]; do + case $1 in + -c) + clean=true + shift + ;; + -d) + dist_only=true + shift + ;; + -g) + gen_only=true + shift + ;; + -q) + repo_name="CPPGET.ORG/QUEUE" + repo_dir="cppget.org/repository/1/queue" + repo_host1="cppget.org:/var/bpkg/1/queue" + repo_host2="queue.cppget.org:/var/bpkg/1/queue" + shift + ;; + *) + if [ -z "$group" ]; then + group="$1" + else + dirs+=("${1%/}") + fi + shift + ;; + esac +done + +if [ -z "$group" -o "${#dirs[@]}" -eq 0 ]; then + error "$usage" +fi + +mkdir -p /tmp/dist + +# Dist individual packages into the repository. +# +function dist() # <group> <dir> +{ + local o="$repo_dir/$1" + local b="$(basename $2)" + + mkdir -p "$o" + + # Clean up old packages. + # + if [ -n "$clean" ]; then + rm -f "$o/$b"-* + fi + + b "dist($2-default/)" "config.dist.archives=$owd/$o/tar.gz" +} + +for d in "${dirs[@]}"; do + dist "$group" "$d" +done + +if [ -n "$dist_only" ]; then + exit 0 +fi + +# Regenerate the repository. +# +info "Insert $repo_name signing key and press Enter" +read +etc/rep-update "$repo_dir/" \ + --openssl-option -engine --openssl-option pkcs11 \ + --openssl-option -keyform --openssl-option engine \ + --key "label_SIGN key" + +if [ -n "$gen_only" ]; then + exit 0 +fi + +# Sync repository. +# +etc/rep-publish "$repo_dir/" "$repo_host1/" $rsync_ops + +if [ -n "$repo_host2" ]; then + etc/rep-publish "$repo_dir/" "$repo_host2/" $rsync_ops +fi @@ -1,45 +0,0 @@ -#! /usr/bin/env bash - -# Tag build2 distribution. -# -# Usage: tag [-t] -# -# -t -# Toolchain only. -# -usage="usage: $0 [-t]" - -owd=`pwd` -trap "{ cd $owd; exit 1; }" ERR -set -o errtrace # Trap in functions. - -function info () { echo "$*" 1>&2; } -function error () { info "$*"; exit 1; } - -toolchain="libbutl build2 libbpkg bpkg build2-toolchain" -extras="brep" - -while [ $# -gt 0 ]; do - case $1 in - -t) - extra= - shift - ;; - *) - error "unexpected $1" - ;; - esac -done - -tools="$toolchain $extras" - -for t in $tools; do - v=`cat $t/version` - if [ -z "$v" ]; then - error "unable to extract version from $t/version" - fi - git -C $t tag -a $v -m "Tag version $v" - info "tagged $t $v" -done - -info "remember to push" @@ -2,294 +2,111 @@ # Manage build2 toolchain version. # -# Usage: version [<module>...] -# -usage="usage: $0 [<module>...]" - -# Notes: -# -# * min/max are [min max) -# if min == max, then '== min' is assumed -# if max is unspecified, then '>= min' is assumed -# currently, max can only be 1.2.3- (#if-conditions will need adjustment) +# Usage: etc/version # +usage="usage: etc/version" # No max, always >=. # -build2_min=0.4.0 -bpkg_min=0.4.0 - -libbutl=0.5.0-a1 -libbutl_min=$libbutl -libbutl_max=$libbutl -#libbutl_max=0.6.0- # Comment if pre-release. -libbutl_build2=$build2_min -libbutl_bpkg=$bpkg_min -libbutl_hxx=butl/version -libbutl_libver=y - -build2=0.5.0-a1 -build2_libbutl_min=$libbutl_min -build2_libbutl_max=$libbutl_max -build2_build2=$build2_min -build2_bpkg=$bpkg_min -build2_hxx=build2/version - -libbpkg=0.5.0-a1 -libbpkg_min=$libbpkg -libbpkg_max=$libbpkg -#libbpkg_max=0.6.0- # Comment if pre-release. -libbpkg_libbutl_min=$libbutl_min -libbpkg_libbutl_max=$libbutl_max -libbpkg_build2=$build2_min -libbpkg_bpkg=$bpkg_min -libbpkg_hxx=bpkg/version -libbpkg_libver=y - -bpkg=0.5.0-a1 -bpkg_libbutl_min=$libbutl_min -bpkg_libbutl_max=$libbutl_max -bpkg_libbpkg_min=$libbpkg_min -bpkg_libbpkg_max=$libbpkg_max -bpkg_build2=$build2_min -bpkg_bpkg=$bpkg_min -bpkg_hxx=bpkg/bpkg-version - -build2_toolchain=0.5.0-a1 -build2_toolchain_build2=$build2_min - -brep=0.5.0-a1 -brep_libbutl_min=$libbutl_min -brep_libbutl_max=$libbutl_max -brep_libbpkg_min=$libbpkg_min -brep_libbpkg_max=$libbpkg_max -brep_build2=$build2_min -brep_bpkg=$bpkg_min -brep_hxx=brep/version -brep_libver=y - -# -# -owd=`pwd` -trap "{ cd $owd; exit 1; }" ERR -set -o errtrace # Trap in functions. +build2_min=0.5.0- +bpkg_min=0.5.0- -function info () { echo "$*" 1>&2; } -function error () { info "$*"; exit 1; } +ver=0.6.0-a.0.z +mods=(libbutl libbpkg libbbot) -modules= +odb_ver=2.5.0-b.2.z +odb_mods=(libodb libodb-sqlite libodb-pgsql) -while [ $# -gt 0 ]; do - case $1 in - *) - modules="$modules ${1%/}" - shift - ;; - esac -done +xml_ver=1.1.0-b.1.z +xml_mods=(libstudxml) -if [ -z "$modules" ]; then - modules="libbutl build2 libbpkg bpkg build2-toolchain brep" -fi +all=(libbutl build2 libbpkg bpkg libbbot brep bbot msvc-linux build2-toolchain) -function str_major () { echo "$1" | sed -e 's/\([0-9]*\).*/\1/' -; } -function str_minor () { echo "$1" | sed -e 's/[^.]*\.\([0-9]*\).*/\1/' -; } -function str_patch () { echo "$1" | sed -e 's/[^.]*\.[^.]*\.\([0-9]*\).*/\1/' -; } -function str_prere () { echo "$1" | sed -n -e 's/[^.]*\.[^.]*\.[^-]*\(-[ab]*[0-9]*\)/\1/p' -; } - -# Convert string version representation (1.2.3-a1) to its numeric equivalent -# in the form AABBCCDD where: -# -# AA - major version number -# BB - minor version number -# CC - bugfix version number -# DD - alpha / beta (DD + 50) version number -# -# When DD is not 00, 1 is subtracted from AABBCC. For example: -# -# Version AABBCCDD -# 2.0.0 02000000 -# 2.1.0 02010000 -# 2.1.1 02010100 -# 2.2.0-a1 02019901 -# 3.0.0-b2 02999952 +# In manifests for alpha/beta we use an exact match. For snapshots -- snapshot +# range. Otherwise an X.Y.* range. # -# For a version in the 1.2.3- form, it returns (AABBCC-1)00, which is the -# highest possible version that is less than 1.2.3-. -# -function str_num () # <ver-str> +function constraint () # <ver> { - # Here we assume the version is valid. - # - local a="$(str_major "$1")" - local b="$(str_minor "$1")" - local c="$(str_patch "$1")" - local d="$(str_prere "$1")" + local mj mn pt ab pr r - local v=$(($a * 1000000 + $b * 10000 + $c * 100)) + mj="$(echo "$1" | sed -re 's/([^.]+)\..*/\1/')" + mn="$(echo "$1" | sed -re 's/[^.]+\.([^.]+)\..*/\1/')" + pt="$(echo "$1" | sed -re 's/[^.]+\.[^.]+\.([^.-]+).*/\1/')" + ab="$(echo "$1" | sed -n -re 's/[^.]+\.[^.]+\.[^.-]+-([ab]).*/\1/p')" + pr="$(echo "$1" | sed -n -re 's/[^.]+\.[^.]+\.[^.-]+-[ab]\.([^.+]+).*/\1/p')" - if [ "$d" ]; then - ((v -= 100)) - if [ $d != "-" ]; then - if [ ${d:1:1} = 'b' ]; then - ((v += 50)) - fi - d=`echo $d | sed -e 's/-[ab]\([0-9]*\)/\1/' -` - ((v += $d)) + if [ -n "$ab" ]; then + if [[ $1 == *.z ]]; then + r="[$mj.$mn.$pt-$ab.$pr.1 $mj.$mn.$pt-$ab.$(($pr+1)))" + else + r="== $1" fi + else + r="[$mj.$mn.0 $mj.$(($mn+1)).0-); $mj.$mn.X" fi - #info $1 -> $a $b $c $d -> $v - echo $v + echo "$r" } -function str_num_test () { if [ `str_num $1` != $2 ]; then error $1 != $2; fi } - -str_num_test 2.0.0 2000000 -str_num_test 2.1.0 2010000 -str_num_test 2.1.1 2010100 -str_num_test 2.2.0-a1 2019901 -str_num_test 3.0.0-b2 2999952 +dep="$(constraint "$ver")" +odb_dep="$(constraint "$odb_ver")" +xml_dep="$(constraint "$xml_ver")" -str_num_test 2.0.0- 1999900 -str_num_test 2.2.0- 2019900 - - -# In-place sed. -# -function ised () # <regex> <file> +function process () # { - local r=$1 - local f=$2 - local o=$f.orig - - mv $f $o - cp -p $o $f # Keep owner, permissions. - - if ! sed -e "$r" $o >$f; then - mv $o $f - return 1 - fi - - if cmp -s $o $f; then - mv $o $f - else - rm $o - fi -} - -# Modules which we may depend on. These are translated to 'depends:' in the -# manifest and preprocessor checks in the version header. Note that we can -# add external modules like libodb, etc., when the time comes. -# -depends="libbutl libbpkg" - -for m in $modules; do - v=`echo $m | sed -e 's/-/_/g'` - V=`echo $v | sed -e 's/\(.*\)/\U\1/'` - - s=${!v} - n=`str_num $s` - - - b=${v}_build2; b=${!b} - p=${v}_bpkg; p=${!p} - l=${v}_libver; l=${!l} - - # version - # - ised "s/^[0-9].*/$s/" $m/version - - # manifest - # - if [ -f $m/manifest ]; then - ised "s/^\(version:\) .*/\1 $s/" $m/manifest - ised "s/^\(depends: \* build2\) .*/\1 >= $b/" $m/manifest - ised "s/^\(depends: \* bpkg\) .*/\1 >= $p/" $m/manifest - fi - - # bootstrap.build - # - ised "s/^\(version =\) .*/\1 $s/" $m/build/bootstrap.build - ised "s/^\(using build@\).*/\1$b/" $m/build/bootstrap.build - - # Library abi_* versions. + # Manifest. # - if [ "$l" = "y" ]; then - a="$(str_major "$s")" - b="$(str_minor "$s")" - c="$(str_patch "$s")" - d="$(str_prere "$s")" + sed -i -re "s/^(version:).*/\1 $ver/" manifest + sed -i -re "s/^(depends: \* build2) .*/\1 >= $build2_min/" manifest + sed -i -re "s/^(depends: \* bpkg) .*/\1 >= $bpkg_min/" manifest - #info "$m $s -> $a $b $c $d" - - if [ -z "$d" ]; then - d="false" - else - d="true" - fi - - ised "s/^\(abi_major =\) .*/\1 $a/" $m/build/bootstrap.build - ised "s/^\(abi_minor =\) .*/\1 $b/" $m/build/bootstrap.build - ised "s/^\(abi_patch =\) .*/\1 $c/" $m/build/bootstrap.build - ised "s/^\(abi_prerelease =\) .*/\1 $d/" $m/build/bootstrap.build - fi + local m + for m in "${mods[@]}"; do + sed -i -re "s/^(depends: $m) .*/\1 $dep/" manifest + done - # Fix configuration version in build scripts. - # - if [ "$m" = "build2-toolchain" ]; then - a="$(str_major "$s")" - b="$(str_minor "$s")" - d="$(str_prere "$s")" + for m in "${odb_mods[@]}"; do + sed -i -re "s/^(depends: $m) .*/\1 $odb_dep/" manifest + done - ised 's/^\(cver=\).*/\1"'"$a.$b$d"'"/' $m/build.sh - ised 's/^\(set "cver=\).*/\1'"$a.$b$d"'"/' $m/build-msvc.bat - ised 's/^\(set "cver=\).*/\1'"$a.$b$d"'"/' $m/build-mingw.bat - fi + for m in "${xml_mods[@]}"; do + sed -i -re "s/^(depends: $m) .*/\1 $xml_dep/" manifest + done - # version header + # cli.sh # - h=${v}_hxx; h=${!h} - if [ "$h" ]; then - ised "s/^\(#define ${V}_VERSION\) .*/\1 $n/" $m/$h - ised "s/^\(#define ${V}_VERSION_STR\) .*/\1 \"$s\"/" $m/$h + if [ -f doc/cli.sh ]; then + sed -i -re "s/^version=.*/version=$ver/" doc/cli.sh fi +} - # Dependencies (manifest and version header). - # - for d in $depends; do - if [ $d = $m ]; then continue; fi - - dv=`echo $d | sed -e 's/-/_/g'` - DV=`echo $dv | sed -e 's/\(.*\)/\U\1/'` +for m in "${all[@]}"; do + cd "$m" + process + cd .. +done - dmin=${v}_${dv}_min; dmin=${!dmin} - dmax=${v}_${dv}_max; dmax=${!dmax} +# Configuration version. +# +mj="$(echo "$ver" | sed -re 's/([^.]+)\..*/\1/')" +mn="$(echo "$ver" | sed -re 's/[^.]+\.([^.]+)\..*/\1/')" +ab="$(echo "$ver" | sed -n -re 's/[^.]+\.[^.]+\.[^.-]+-([ab]).*/\1/p')" +pr="$(echo "$ver" | sed -n -re 's/[^.]+\.[^.]+\.[^.-]+-[ab]\.([^.+]+).*/\1/p')" - if [ -z "$dmin" ]; then continue; fi # No dependency. +cver="$mj.$mn" - dminn=`str_num $dmin` +if [ -n "$ab" ]; then + cver="$cver-$ab.$pr" +fi - # Figure out dependency contraints for the manifest and the header. - # - if [ -z "$dmax" ]; then - dcm=">= $dmin" - dch="${DV}_VERSION < $dminn" - elif [ "$dmin" = "$dmax" ]; then - dcm="== $dmin" - dch="${DV}_VERSION != $dminn" - else - dmaxn=`str_num $dmax` - dcm="[$dmin $dmax)" - dch="${DV}_VERSION < $dminn || ${DV}_VERSION > $dmaxn" - fi +cd build2-toolchain - if [ -f $m/manifest ]; then - ised "s/^\(depends: $d\) .*/\1 $dcm/" $m/manifest - fi +for m in build*.sh; do + sed -i -re "s/cver=\".+\"/cver=\"$cver\"/" $m +done - if [ "$h" ]; then - ised "s/^#if ${DV}_VERSION .*/#if $dch/" $m/$h - fi - done +for m in build*.bat; do + sed -i -re "s/\"cver=.+\"/\"cver=$cver\"/" $m done + +cd .. |