path: root/bootstrap

#! /usr/bin/env bash

# @@ Do we really want to require PDF doc generation? This will unlikely
#    work anywhere except Linux. Maybe we should only distribute .xhtml
#    and only generate PDFs for web publishing? Maybe decide when moving
#    to ad hoc rules?

# @@ Describe filesystem structure, created configurations, symlinks.

# --ssh --clean --extra --module rust --cxx g++-7 --cfg gcc7
# --no-clone --no-symlink --no-libs --no-modules --cxx g++-7 --cfg gcc7-tsan --coptions "-Wall -Wextra -Werror -g -fsanitize=thread"
# bdep init -d <proj> -A builds/gcc7 @gcc7

# Boostrap the build2 development environment.
#
# Note that this script runs git-update-index commands specified in the
# README-GIT files of cloned repositories.
#
# This script assumes the following steps have been performed:
#
# 1. The latest staged toolchain has been temporarily installed somewhere
#    other than /usr/local (e.g., /tmp/build2-install) and is in PATH.
#    Hint: use --local --no-modules when building the staged toolchain to
#    speed things up seeing that this is a throw-away installation:
#
#    sh build2-install-...-stage.sh --local --no-modules /tmp/build2-install
#
# 2. The CLI and ODB compilers have been built and are either installed (if
#    you don't plan to contribute to them) or symlinked in /usr/local/bin/.
#    For the latter case they would typically go to ~/work/{cli,odb/}, for
#    example, for CLI:
#
#    NOTE: clone using the git.codesynthesis.com:/var/scm/... SSH URL if you
#          have rw access.
#
#    mkdir -p ~/work/cli
#    cd ~/work/cli
#    git clone --recursive https://git.codesynthesis.com/cli/cli.git
#    cd cli
#    ... see README.md for instructions
#    ... use config.cc.coptions="-Wall -Wextra -Werror -g3 -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer"
#    b
#    sudo ln -s "$(pwd)/cli/cli/cli" /usr/local/bin/cli
#    which cli
#    cli --version
#
#    For ODB (no sanitizer use is currently possible due to the way GCC is
#    built):
#
#    mkdir -p ~/work/odb
#    cd ~/work/odb
#    git clone --recursive https://git.codesynthesis.com/odb/odb.git
#    cd odb
#    bdep init -C ../builds/gccN-asan @gccN-asan cc \
#      config.cxx=g++-N                             \
#      config.cc.coptions="-Wall -Wextra -Werror -g3"
#    b
#    sudo ln -s "$(pwd)/odb/odb"    /usr/local/bin/odb
#    sudo ln -s "$(pwd)/odb/odb.so" /usr/local/bin/odb.so
#    which odb
#    odb --version
#
# Once this is done, you should be able to:
#
# NOTE: clone using the git.build2.org:/var/scm/... SSH URL and pass the
#       --ssh bootstrap option if you have rw access.
#
# mkdir -p ~/work/build2
# cd ~/work/build2
# git clone --recursive https://git.build2.org/etc.git
# PATH="/tmp/build2-install/bin:$PATH" etc/bootstrap
#
# After a successful bootstrap you can remove the staged toolchain.
#
# Also a few notes and tips on the development process: Because the toolchain
# is used for its own development you will sometimes find yourself in a
# situation where one or more tools in the toolchain are no longer runnable or
# functioning correctly but to fix this you need to be able to run those
# tools. This can happen, for example, because of the linking errors or
# because the version (and therefore the name) of one of the shared libraries
# has changed as a result of updating another part of the toolchain (for
# instance, updating bpkg can trigger an update of libbpkg and this could make
# bdep, which also depends on libbpkg, no longer runnable).
#
# There are several mechanisms for recovering from such situations. If the
# build system is functional, it itself and the rest of the toolchain can
# always be update by first disabling the auto-synchronization hooks (which
# invoke bdep and bpkg). For example, the following commands should get the
# toolchain back into the fully-functional state:
#
# BDEP_SYNC=0 b build2/build2/ bpkg/bpkg/ bdep/bdep/
#             b build2/build2/ bpkg/bpkg/ bdep/bdep/
#
# If the build system itself is not functional, it can always be rebuilt using
# the bootstrapped build system (b-boot; built by this script). For example:
#
# BDEP_SYNC=0 b-boot build2/build2/
# BDEP_SYNC=0 b      build2/build2/
#
# Note that the bootstrap build system can only be used to update build2. It
# also makes sense to rebuild it from time to time to keep it reasonably up to
# date with master. Normally this is done when master is assumed to be in a
# reasonably good shape. See build2/INSTALL for details on the bootstrap build
# system.
#
# Options:
#
# --ssh
#    Use SSH URL for cloning (rw access).
#
# --extra
#    Initialize additional packages (bbot, brep, etc) that are not part of
#    the toolchain core (build2, bpkg, and bdep). Note that they are
#    configured but not updated. Also note that for brep we assume libapr1,
#    libapreq2, and libpq are available from the system (see brep/INSTALL
#    for details).
#
# --module <name>
#    Additional (to standard pre-installed) build system module to initialize.
#    Repeat this option to specify multiple modules. Note: <name> should be
#    without the libbuild2- prefix.
#
# --no-modules
#    Do not initialize standard pre-installed build system modules.
#
# --no-clone
#    Do not clone the repositories or build the bootstrap build system
#    (b-boot) assuming this has already been done. This option is primarily
#    useful for initializing additional configurations (e.g., Clang in
#    addition to GCC).
#
# --clean
#    Clean (or clone if does not exist) the source directories (remove .bdep/,
#    forwarded configurations) and the configuration directories. This is
#    primarily useful to re-initialized an already initialized setup.
#
# --no-libs
#    Do not create an extra configuration for libraries. Having such a
#    configuration allows you to test changes in libraries without rendering
#    the rest of the toolchain no longer runnable/functional.
#
# --no-symlink
#    Do not create executable symlinks in /usr/local/bin/.
#
# --cxx
# --cfg
#    C++ compiler to use and the corresponding build configuration name, for
#    example, g++-9/gcc9 or clang++-8/clang8; g++/gcc by default.
#
# --coptions
# --loptions
#    Compiler and linker options to use. By default a debug build with
#    ASAN/UBSAN and parallel gold linking is configured:
#
#    -Wall -Wextra -Werror -g -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer
#    -fuse-ld=gold -Wl,--threads,--thread-count,4
#
#    To configure TSAN, use the following coptions instead:
#
#    -Wall -Wextra -Werror -g -fsanitize=thread
#
owd="$(pwd)"
trap "{ cd '$owd'; exit 1; }" ERR
set -o errtrace # Trap in functions.

function info () { echo "$*" 1>&2; }
function error () { info "$*"; exit 1; }

# Run a command with tracing (similar to set -x).
#
# Note that this function will execute a command with arguments that contain
# spaces but it will not print them as quoted (and neither does set -x).
#
function run ()
{
  echo "+ $@" 1>&2
  "$@"
}

# Make sure the necessary tools are runnable.
#
cli  --version >/dev/null
odb  --version >/dev/null
b    --version >/dev/null
bpkg --version >/dev/null
bdep --version >/dev/null

url="https://git.build2.org"
mod_url="https://github.com/build2"
extra=
mod=(hello autoconf kconfig)
sym=true
clone=true
clean=
libs=true
cxx=g++
cfg=gcc
copt="-Wall -Wextra -Werror -g -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer"
lopt="-fuse-ld=gold -Wl,--threads,--thread-count,4"
popt=

while [ $# -gt 0 ]; do
  case $1 in
    --ssh)
      url="git.build2.org:/var/scm"
      mod_url="git@github.com:build2"
      shift
      ;;
    --extra)
      extra=true
      shift
      ;;
    --module)
      shift
      mod+=("$1")
      shift
      ;;
    --no-modules)
      mod=()
      shift
      ;;
    --no-symlink)
      sym=
      shift
      ;;
    --no-clone)
      clone=
      shift
      ;;
    --clean)
      clean=true
      shift
      ;;
    --no-libs)
      libs=
      shift
      ;;
    --cxx)
      shift
      cxx="$1"
      shift
      ;;
    --cfg)
      shift
      cfg="$1"
      shift
      ;;
    --coptions)
      shift
      copt="$1"
      shift
      ;;
    --loptions)
      shift
      lopt="$1"
      shift
      ;;
    *)
      error "unexpected $1"
      ;;
  esac
done

if [ "$clean" ]; then

  if [ "$sym" ]; then
    run sudo rm -f /usr/local/bin/b-boot
    run sudo rm -f /usr/local/bin/b
    run sudo rm -f /usr/local/bin/bpkg
    run sudo rm -f /usr/local/bin/bdep
  fi

  run rm -rf builds/

  for m in "${mod[@]}"; do
    run rm -rf "libbuild2-$m-build/"
  done

  run cd build2
  run make -f bootstrap.gmake CXX="$cxx" CXXFLAGS=-O3 clean
  run cd -

  def=true

else
  # Determine if this is the first/default configuration.
  #
  if test -d build2/.bdep; then
    def=
  else
    def=true
  fi
fi

if [ ! "$def" ]; then

  if [ "$libs" ]; then
    error "non-default configuration requires --no-libs"
  fi

  if [ "${#mod[@]}" -ne 0 ]; then
    error "non-default configuration requires --no-modules"
  fi
fi

if [ "$extra" ]; then
  popt="$popt -I$(apxs -q includedir)"
fi

# Clone or clean the repository.
#
function git_clone () # <url>
{
  local u d l

  u="$1"
  d="$(basename -s .git "$u")"

  # Clone it if doesn't exist.
  #
  if [ "$clean" -a -d "$d" ]; then

    run rm -rf "$d/.bdep/"

    for l in $(find "$d" -name out-root.build); do
      run rm -f "$l"
    done

  else
    run git clone --recursive "$u"
  fi

  # Run git-update-index commands from README-GIT.
  #
  if [ -f "$d/README-GIT" ]; then
    run cd "$d"
    l=
    while read l || [ -n "$l" ]; do
      info "+ $l"
      eval $l
    done < <(sed -rn -e 's/^(git update-index .+)/\1/p' README-GIT)
    run cd -
  fi

  # Generate documentation (currently and temporarily handled with a script).
  #
  if [ -f "$d/doc/cli.sh" ]; then
    run cd "$d/doc"
    run ./cli.sh
    run cd -
  fi
}

if [ "$clone" -o "$clean" ]; then
  git_clone "$url/libbutl.git"
  git_clone "$url/build2.git"
  git_clone "$url/libbpkg.git"
  git_clone "$url/bpkg.git"
  git_clone "$url/bdep.git"

  if [ "$extra" ]; then
    git_clone "$url/libbutl.bash.git"
    git_clone "$url/bpkg-util.git"
    git_clone "$url/bdep-util.git"
    git_clone "$url/libbbot.git"
    git_clone "$url/bbot.git"
    git_clone "$url/brep.git"
  fi

  for m in "${mod[@]}"; do
    git_clone "$mod_url/libbuild2-$m.git"
  done

  # Build the bootstrap build system (b-boot).
  #
  run cd build2
  run make -f bootstrap.gmake CXX="$cxx" CXXFLAGS=-O3 -j 8
  run make -f bootstrap.gmake CXX="$cxx" CXXFLAGS=-O3 cleano # Cleanup objs.
  run build2/b-boot --version
  run cd -
fi

run mkdir -p builds

# The -libs configuration.
#
# Note that by doing this first we automatically make sure this will be their
# default/forwarded configuration.
#
if [ "$libs" ]; then

  run bpkg create -d "builds/$cfg-libs" cc cli \
    config.cli=cli \
    config.cxx="$cxx" \
    config.cc.coptions="$copt" \
    config.cc.loptions="$lopt" \
    config.bin.lib=shared \
    config.install.root=/tmp/install config.dist.root=/tmp/dist

  run bdep init -d libbutl -A "builds/$cfg-libs" "@$cfg-libs"
  run bdep init -d libbpkg -A "builds/$cfg-libs" "@$cfg-libs"

  if [ "$extra" ]; then
    run bdep init -d libbutl.bash -A "builds/$cfg-libs" "@$cfg-libs"
    run bdep init -d libbbot      -A "builds/$cfg-libs" "@$cfg-libs"
  fi
fi

# Relative and absolute configuration directories.
#
rd="builds/$cfg"
ad="$owd/$rd"

# Build system configuration.
#
# Notes:
#
#   1. Using ?cli since this should be buildable by b-boot.
#
#   2. Only building shared libraries for speed of development.
#
run bpkg create -d "$rd" cc ?cli \
  config.cli=cli \
  config.cxx="$cxx" \
  config.cc.poptions="$popt" \
  config.cc.coptions="$copt" \
  config.cc.loptions="$lopt" \
  config.bin.lib=shared \
  config.install.root=/tmp/install config.dist.root=/tmp/dist

run bdep init -d libbutl -A "$rd" "@$cfg"
run bdep init -d build2  -A "$rd" "@$cfg"
run bdep init -d libbpkg -A "$rd" "@$cfg"
run bdep init -d bpkg    -A "$rd" "@$cfg"
run bdep init -d bdep    -A "$rd" "@$cfg"

if [ "$extra" ]; then
  run bdep init -d libbutl.bash -A "$rd" "@$cfg"
  run bdep init -d bpkg-util    -A "$rd" "@$cfg"
  run bdep init -d bdep-util    -A "$rd" "@$cfg"
  run bdep init -d libbbot      -A "$rd" "@$cfg"
  run bdep init -d bbot         -A "$rd" "@$cfg"
  run bdep init -d brep         -A "$rd" "@$cfg" \
    ?sys:libapr1 ?sys:libapreq2 ?sys:libpq
fi

# Generate database support (currently and temporarily handled with a script).
#
# Note: this has to be done after bdep-init since we need the libodb headers.
#       We also have to pre-update version headers.
#
if [ $def ]; then
  run bpkg update -d "$rd" libodb
  run b "$rd/libbutl/libbutl/hxx{version}"
  run b "$rd/libbpkg/libbpkg/hxx{version}"
  run b "$rd/bpkg/bpkg/hxx{version common-options}"
  run b "$rd/bdep/bdep/hxx{version common-options project-options}"

  run cd bpkg/bpkg
  run ./odb.sh
  run cd -

  run cd bdep/bdep
  run ./odb.sh
  run cd -

  if [ "$extra" ]; then
    run b "$rd/brep/web/xhtml/hxx{version}"
    run b "$rd/brep/libbrep/hxx{version}"

    run cd brep/libbrep
    run ./odb.sh
    run cd -
  fi
fi

if [ $def ]; then
  run b build2/build2/ bpkg/bpkg/ bdep/bdep/
  run build2/build2/b --version
  run bpkg/bpkg/bpkg --version
  run bdep/bdep/bdep --version
else
  run b "$rd/build2/build2/" "$rd/bpkg/bpkg/" "$rd/bdep/bdep/"
  run "$rd/build2/build2/b" --version
  run "$rd/bpkg/bpkg/bpkg" --version
  run "$rd/bdep/bdep/bdep" --version
fi

# Add symlinks.
#
if [ "$sym" ]; then
  # Note that we symlink the actual executable in the build configuration
  # rather than the backlink in the forwarded source to make the old binary
  # runnable in the face of compilation errors (which trigger the removal of
  # backlinks).
  #
  run sudo mkdir -p /usr/local/bin
  run sudo ln -s "$owd/build2/build2/b-boot" /usr/local/bin/b-boot
  run sudo ln -s "$ad/build2/build2/b"       /usr/local/bin/b
  run sudo ln -s "$ad/bpkg/bpkg/bpkg"        /usr/local/bin/bpkg
  run sudo ln -s "$ad/bdep/bdep/bdep"        /usr/local/bin/bdep

  run export PATH="/usr/local/bin:$PATH"
else
  run export PATH="$ad/build2/build2:$PATH"
  run export PATH="$ad/bpkg/bpkg:$PATH"
  run export PATH="$ad/bdep/bdep:$PATH"
fi

run which b bpkg bdep

# Re-run update using the bootstrapped toolchain (normally should be a
# noop).
#
if [ $def ]; then
  run b build2/build2/ bpkg/bpkg/ bdep/bdep/
else
  run b "$rd/build2/build2/" "$rd/bpkg/bpkg/" "$rd/bdep/bdep/"
fi

# Configure the modules.
#
for m in "${mod[@]}"; do
  d="libbuild2-$m"

  run cd "$d"

  run bdep init --empty
  run bdep config create @module "../$d-build/module/" --type build2 cc config.config.load=~build2
  run bdep config create @target "../$d-build/target/" cc config.cxx="$cxx"

  run bdep init @module -d "$d/"

  # See if this module requires bootstrap.
  #
  b=$(sed -rn -e 's/^\s*requires\s*:\s*bootstrap\s*$/true/p' "$d/manifest")

  if [ "$b" ]; then
    info "module $d requires bootstrapping"
    info "make sure ~/.build/b.option contains the following line and press Enter"
    info "!config.import.libbuild2_$m=$owd/$d-build/module/"
    read
  fi

  # Assume every package other than the module is a test.
  #
  # Note: sed regex is from the glue buildfile.
  #
  for t in $(sed -rn -e 's/^\s*location\s*:\s*(\S+)\s*$/\1/p' packages.manifest); do
    if [ "$d/" = "$t" ]; then
      continue
    fi

    run bdep init @target -d "$t"
  done

  run cd -
done