path: root/bootstrap

#! /usr/bin/env bash

# Bootstrap build2 Build OS.
#
# Assumptions/expectations:
#
# - host debootstrap/debian-archive-keyring matching release
#
# - /btrfs/<user> is a btrfs directory where the current user can create
#   snapshots
#
# - sudo is passwordless (used to run debootstrap, systemd-nspawn, etc)
#
# Options:
#
# --stage <num>
#   Jump straigh to stage <num> by clonning the previous stage snapshot.
#   Stages are:
#
#   1 - bootstrap phase 1
#   2 - bootstrap phase 2
#   3 - kernel build
#   4 - setup
#   5 - create footfs
#   6 - create kernel image and initrd
#
usage="usage: $0"

macaddr="de:ad:be:ef:b8:da" # @@ TMP mac address for testing.

id="$(id -un)"
btrfs=/btrfs
root="$btrfs/$id/buildos"

# Source distribution and packages. Base packages are installed on stage 1 via
# debootstrap. Extra packages are added on stage 4 via apt-get install. The
# idea is to be able to add extra packages without upgrading the base system.
# When we do upgrade the base system we normally move the extras to base.
#
# Notes:
#
#  - some packages (such as CPU microcode updates) are in non-free.
#  - systemd-container seems to be required by host systemd-nspawn.
#  - not installing linux-image-amd64 since building custom below
#
release="testing"
components="main,contrib,non-free"
mirror="http://deb.debian.org/debian/"
#mirror="https://deb.debian.org/debian/"

base_pkgs="locales,klibc-utils,sudo,systemd-container,udev"
base_pkgs+=",kmod,linux-base,firmware-linux-free,irqbalance"
base_pkgs+=",intel-microcode,amd64-microcode"
base_pkgs+=",pciutils,usbutils,dmidecode,cpuid"
base_pkgs+=",hdparm,btrfs-progs"
base_pkgs+=",lm-sensors,smartmontools"

base_pkgs+=",psmisc"
base_pkgs+=",net-tools,iproute2,iptables,isc-dhcp-client"
base_pkgs+=",ifupdown,bridge-utils,dnsmasq,ntp,postfix"
base_pkgs+=",iputils-ping,wget,curl,ca-certificates"
base_pkgs+=",openssh-client,openssh-server"
base_pkgs+=",tftp-hpa,tftpd-hpa"

base_pkgs+=",bzip2,xz-utils"
base_pkgs+=",less,nano,time"

base_pkgs+=",qemu-kvm,qemu-utils,socat"

base_pkgs+=",g++,make,pkg-config"

extra_pkgs=""

owd="$(pwd)"
trap "{ cd '$owd'; exit 1; }" ERR
set -o errtrace # Trap in functions.

function info () { echo "$*" 1>&2; }
function error () { info "$*"; exit 1; }

stage="1"
stage_max="6"

while [ "$#" -gt 0 ]; do
  case "$1" in
    --stage)
      shift
      stage="$1"
      shift
      ;;
    -*)
      error "unknown option: $1"
      ;;
    *)
      break
      ;;
  esac
done

if [ "$stage" -lt "1" -o "$stage" -gt "$stage_max" ]; then
  error "invalid stage number $stage"
fi

# Extract version.
#
version="$(sed -n -re 's/^version: ([0-9]+\.[0-9]+\.[0-9]+).*$/\1/p' ./manifest)"
version_id="$(sed -n -re 's/^([0-9]+\.[0-9]+)\.[0-9]+$/\1/p' <<<"$version")"
if [ -z "$version" -o -z "$version_id" ]; then
  error "unable to extract version from manifest"
fi

# Btrfs subvolume manipulation.
#
function subvol_delete () # <path>
{
  # The subvol show (just as list) needs root.
  #
  if sudo btrfs subvol show "$1" 1>/dev/null 2>&1; then
    btrfs property set -ts "$1" ro false
    btrfs subvol delete "$1"
  fi
}

function subvol_create () # <path>
{
  btrfs subvol create "$@"
}

function subvol_snapshot () # [-r] <src-path> <dst-path>
{
  btrfs subvol snapshot "$@"
}

# Clean up the working subvolume and all the snapshots starting from the
# requested stage. Also, if stage is not 1, then restore the working subvolume
# from the previous stage snapshot.
#
subvol_delete "$root"

for i in $(seq "$stage" "$stage_max"); do
  subvol_delete "$root-$i"
done

if [ "$stage" -gt "1" ]; then
  i="$(($stage - 1))"
  info "restoring working subvolume from stage $i snapshot"
  subvol_snapshot "$root-$i" "$root"
fi

# Spawn a systemd namespace container (systemd-nspawn)
#
function nspawn () # <root> <systemd-nspawn-args>
{
  local r="$1"
  shift

  sudo systemd-nspawn --register=no -D "$r" "$@"

  # systemd-nspawn may create the /var/lib/machines subvolume which prevents
  # the deletion of the containing submodule. So we clean it up.
  #
  if sudo btrfs subvol show "$r/var/lib/machines" 1>/dev/null 2>&1; then
    sudo btrfs subvol delete "$r/var/lib/machines"
  fi
}

# (Over)write or append to a file in the installation root, for example:
#
# write <<<'localhost' /etc/hostname
#
function write () # <path> [<root>]
{
  local r="$2"
  if [ -z "$r" ]; then
    r="$root"
  fi

  sudo tee "$r$1" >/dev/null
}

function append () # <path> [<root>]
{
  local r="$2"
  if [ -z "$r" ]; then
    r="$root"
  fi

  sudo tee -a "$r$1" >/dev/null
}

# Stage 1: debootstrap, phase 1.
#
if [ "$stage" -eq "1" ]; then

  subvol_create "$root"

  sudo debootstrap \
    --foreign \
    --arch=amd64 \
    --merged-usr \
    --variant=minbase \
    --components="$components" \
    --include="$base_pkgs" \
    "$release" "$root" "$mirror"

  # Post-phase 1 fixups.
  #

  # Set the initial hostname to 'localhost'. This value is detected and
  # overriden by /sbin/dhclient-script if the DHCP server sends host-name.
  #
  write <<<'localhost' /etc/hostname

  # Set timezone to UTC (picked up by tzdata package during stage 2).
  #
  write <<<'Etc/UTC' /etc/timezone

  subvol_snapshot -r "$root" "$root-1"
fi

# Stage 2: debootstrap, phase 2.
#
if [ "$stage" -le "2" ]; then

  # Create a bootstrap script that will finish the bootstrap from within the
  # installation via systemd-nspawn.
  #
  sudo mkdir "$root/bootstrap"

  write <<EOF /bootstrap/bootstrap
#!/bin/bash

trap "exit 1" ERR
set -x

# Hack around systemd bug#79306 (changes /etc/localtime) by removing it now
# and making readonly below.
#
rm /etc/localtime

# Both nspawn and debootstrap try to mount /proc /sys (Debian bug#840372).
#
mkdir /tmp/proc /tmp/sys
mount --move /proc /tmp/proc
mount --move /sys /tmp/sys

# Run second stage of debootstrap.
#
if ! /debootstrap/debootstrap --second-stage; then
  cat /debootstrap/debootstrap.log 1>&2
  bash
  exit 1
fi

rm -f /etc/localtime
cp /usr/share/zoneinfo/UTC /etc/localtime
chattr +i /etc/localtime

# Change /etc/os-release, /etc/issue, /etc/motd.
#
cat <<EOF1 >/etc/os-release
NAME="Build OS"
VERSION="$version"
ID=buildos
ID_LIKE=debian
PRETTY_NAME="Build OS $version (Based on Debian)"
VERSION_ID="$version_id"
HOME_URL="https://build2.org/"
SUPPORT_URL="https://lists.build2.org/"
BUG_REPORT_URL="https://lists.build2.org/"
EOF1

cat <<EOF1 >/etc/issue
Build OS $version (Based on Debian) \n \l

EOF1

cat <<EOF1 >/etc/motd

Welcome to Build OS $version (https://build2.org)!

EOF1

# Make root login passwordless (we disable SSH root login in init).
#
passwd -d root

# Enable IPv4 forwarding (used for private bridge NAT).
#
sed -i -e 's/^# *\(net.ipv4.ip_forward\).*/\1=1/' /etc/sysctl.conf

# Setup locale. We only support en_US.UTF-8.
#
sed -i -e 's/^# *\(en_US.UTF-8\)/\1/' /etc/locale.gen
locale-gen --purge

cat <<EOF1 >/etc/default/locale
LANG="en_US.UTF-8"
LANGUAGE="en_US:en"
EOF1

EOF
  sudo chmod u+x "$root/bootstrap/bootstrap"

  # Notes:
  #
  # - Failed to create directory .../sys/fs/selinux: Read-only file system is
  #   harmless and fixed upstream (systemd issue#3748).
  #
  nspawn "$root" /bootstrap/bootstrap

  subvol_snapshot -r "$root" "$root-2"
fi

# Stage 3: kernel build.
#
if [ "$stage" -le "3" ]; then

  # Create the setup service that will be used by both this stage and the
  # setup stage below. Note that we will do actual building (which requires
  # installing extra packages) in a snapshot on the side.
  #
  sudo mkdir -p "$root/bootstrap"

  # Note that when started via systemd-nspawn, we get /dev/console, not
  # /dev/tty0.
  #
  write <<EOF /usr/lib/systemd/system/buildos-setup.service
[Unit]
Description=build os setup
After=multi-user.target
Conflicts=console-getty.service

[Service]
Type=idle
TimeoutStartSec=infinity
RemainAfterExit=true
ExecStart=/bootstrap/setup
StandardInput=tty-force
StandardOutput=inherit
StandardError=inherit
TTYPath=/dev/console
TTYReset=yes
TTYVHangup=yes

[Install]
WantedBy=multi-user.target
EOF

  sudo mkdir -p "$root/etc/systemd/system/multi-user.target.wants"
  sudo ln -sf "$root/usr/lib/systemd/system/buildos-setup.service" \
    "$root/etc/systemd/system/multi-user.target.wants/buildos-setup.service"

  # Create the kernel build snapshot, write the script that does the build
  # from within the installation and boot it up via systemd-nspawn --boot.
  #
  # Add this line before shutdown if need to debug/check things.
  #
  # setsid /bin/bash -c "exec /bin/bash -i <>/dev/console 1>&0 2>&1"
  #
  subvol_delete "$root-3-kernel"
  subvol_snapshot "$root" "$root-3-kernel"

  write <<EOF /bootstrap/setup "$root-3-kernel"
#!/bin/bash

trap "exit 1" ERR
set -x

# This seems to be the simplest method of building the standard Debian
# kernel with adjusted configuration. Taken from the Debian Kernel Handbook.
#
apt-get update
apt-get install -y build-essential linux-source
apt-get install -y bison flex
apt-get install -y libelf-dev
apt-get install -y libssl-dev
cd /usr/src
tar xf linux-source-*
mv linux-source-*/ linux
xzcat linux-config-*/config.amd64_none_amd64.xz >linux/.config
cd linux

# Adjust configuration.
#
echo 'CONFIG_SYSTEM_TRUSTED_KEYS=""' >>.config
echo 'CONFIG_BUILD_SALT=""'          >>.config
echo 'CONFIG_MODULE_SIG=n'           >>.config

# Adjust kernel command line size limit.
#
sed -i -re 's/^(#define COMMAND_LINE_SIZE ).+\$/\1 4096/' arch/x86/include/asm/setup.h
make oldconfig
scripts/config --disable DEBUG_INFO

make clean
make deb-pkg LOCALVERSION=-buildos KDEB_PKGVERSION=1 -j 8

# Clean up and shutdown.
#
rm /bootstrap/setup
systemctl poweroff

EOF
  sudo chmod u+x "$root-3-kernel/bootstrap/setup"

  nspawn "$root-3-kernel" --boot

  # Copy the kernel over and install it.
  #
  sudo cp "$root-3-kernel/usr/src/linux-image-"*.deb "$root/usr/src/"

  write <<EOF /bootstrap/setup
#!/bin/bash

trap "exit 1" ERR
set -x

dpkg -i /usr/src/linux-image-*.deb
rm -rf /usr/src/*
cd /
ln -s boot/vmlinuz-* /vmlinuz

# Clean up and shutdown.
#
rm /bootstrap/setup
systemctl poweroff

EOF
  sudo chmod u+x "$root/bootstrap/setup"

  nspawn "$root" --boot

  subvol_snapshot -r "$root" "$root-3"
fi

# Stage 4: setup.
#
if [ "$stage" -le "4" ]; then

  # Write the setup script that will finish the setup (the service is already
  # there from stage 3).
  #
  write <<EOF /bootstrap/setup
#!/bin/bash

trap "exit 1" ERR
set -x

# Install extra packages.
#
apt-get update
for p in \$(sed -e 's/,/ /g' <<<"$extra_pkgs"); do
  apt-get install -y --no-install-recommends "\$p"
done

# We want the utility (smartctl) but not the daemon.
#
systemctl disable smartd

# Create the build user, /build home directory. Make a password-less sudo'er.
#
# Note that we force UID/GID to specific numbers to make sure they are
# consistent across builds.
#
addgroup --gid 2000 build
adduser --uid 2000 --gid 2000 --home /build --gecos "" --disabled-password build
adduser build kvm
echo "build ALL=(ALL) NOPASSWD:ALL" >/etc/sudoers.d/build
echo "Defaults:build !syslog" >>/etc/sudoers.d/build
chmod 0440 /etc/sudoers.d/build

# Clean up package cache.
#
apt-get clean

# Clean up /bootstrap.
#
rm /etc/systemd/system/multi-user.target.wants/buildos-setup.service
rm /usr/lib/systemd/system/buildos-setup.service
rm -r /bootstrap

# Shutdown.
#
systemctl poweroff

EOF
  sudo chmod u+x "$root/bootstrap/setup"

  nspawn "$root" --boot

  subvol_snapshot -r "$root" "$root-4"
fi

# Stage 5: generate rootfs.
#
if [ "$stage" -le "5" ]; then

  # Note that there is also initramfs image that is embedded into kernel. In
  # Debian it contains just /dev/console and /root/.
  #
  # Quite a few files/directories are only accessible by root (e.g., /root) so
  # we run under sudo.
  #
  root_dirs="build dev etc mnt root usr var"
  root_links="bin sbin lib lib32 lib64"

  info "generating buildos-rootfs.cpio.gz..."

  cd "$root"
  sudo find $root_dirs $root_links -print0 | \
    sudo cpio --null -o -H newc | \
    gzip -9 > "$owd/buildos-rootfs.cpio.gz"
  cd "$owd"

  subvol_snapshot -r "$root" "$root-5"
fi

# Stage 6: generate initrd.
#
if [ "$stage" -le "6" ]; then

  # Generate buildid and store it in /etc/os-release and in buildos-buildid.
  # These are used by the monitor to detect when it's time to reboot.
  #
  # Note that /etc/os-release is a symlink to /usr/lib/os-release.
  #
  buildid="$(uuidgen)"

  sudo tee -a "$root/etc/os-release" >/dev/null <<<"BUILD_ID=\"$buildid\""

  # Install init and buildos monitor.
  #
  sudo install -m 755 ./init "$root/"
  sudo install -m 755 ./buildos "$root/usr/sbin/"
  sudo install -m 644 ./buildos.service "$root/usr/lib/systemd/system/"
  sudo ln -sf "$root/usr/lib/systemd/system/buildos.service" \
    "$root/etc/systemd/system/multi-user.target.wants/buildos.service"

  info "generating buildos-init.cpio.gz..."

  cd "$root"
  sudo cpio -o -H newc <<EOF | \
    gzip -9 > "$owd/buildos-init.cpio.gz"
usr/lib/os-release
init
usr/sbin/buildos
usr/lib/systemd/system/buildos.service
etc/systemd/system/multi-user.target.wants/buildos.service
EOF
  cd "$owd"

  cat buildos-rootfs.cpio.gz buildos-init.cpio.gz >buildos-initrd

  # Copy the kernel image next to the initramfs for convenience.
  #
  cp "$root/vmlinuz" buildos-image
  echo "$buildid" >buildos-buildid

  subvol_snapshot -r "$root" "$root-6"
fi

exit 0

# Test.
#
if [ ! -e /tmp/buildos-state ]; then
  qemu-img create -f raw /tmp/buildos-state 20M
fi

if [ ! -e /tmp/buildos-machines ]; then
  qemu-img create -f raw /tmp/buildos-machines 100M
fi

# To test PXE boot, replace -kernel/-initrd/-append with '-boot n'.
#
sudo kvm \
  -m 16G \
  -cpu host -smp "sockets=1,cores=4,threads=2" \
  -device "e1000,netdev=net0,mac=$macaddr" \
  -netdev "tap,id=net0,script=./qemu-ifup" \
  -device "virtio-scsi-pci,id=scsi" \
  -device "scsi-hd,drive=disk1" \
  -drive  "if=none,id=disk1,file=/tmp/buildos-state,format=raw" \
  -device "scsi-hd,drive=disk2" \
  -drive  "if=none,id=disk2,file=/tmp/buildos-machines,format=raw" \
  -boot n

#  -kernel buildos-image -initrd buildos-initrd \
#  -append "buildos.smtp_relay=build2.org buildos.admin_email=admin@build2.org"