path: root/bootstrap

#! /usr/bin/env bash

# Bootstrap build2 Build OS.
#
# Assumptions/expectations:
#
# - Bootstrapping for the host CPU (currently x86_64 or aarch64).
#
# - Host debootstrap/debian-archive-keyring matching release.
#
# - /btrfs/<user> is a btrfs directory where the current user can create
#   snapshots.
#
# - sudo is passwordless (used to run debootstrap, systemd-nspawn, etc).
#
# - This script is executed from the buildos/ source directory.
#
# Options:
#
# --stage <num>
#   Jump straigh to stage <num> by clonning the previous stage snapshot.
#   Stages are:
#
#   1 - bootstrap phase 1
#   2 - bootstrap phase 2
#   3 - kernel build (and package patching, if necessary)
#   4 - setup
#   5 - create footfs
#   6 - create kernel image and initrd
#
# --arch-variant
#   Architecture variant, for example, m1 for aarch64-m1.
#
# --kernel-source
#   Pre-configured Linux kernel source .tar.* archive. If unspecified, Debian
#   source/configuration will be used.
#
usage="usage: $0"

owd="$(pwd)"
trap "{ cd '$owd'; exit 1; }" ERR
set -o errtrace # Trap in functions.

function info () { echo "$*" 1>&2; }
function error () { info "$*"; exit 1; }

id="$(id -un)"
btrfs=/btrfs
root="$btrfs/$id/buildos"

arch="$(uname -m)"

case "$arch" in
  x86_64)
    debian_arch=amd64
    ;;
  aarch64)
    debian_arch=arm64
    ;;
  *)
    error "unsupported architecture: $arch"
    ;;
esac

# Source distribution and packages. Base packages are installed on stage 1 via
# debootstrap. Extra packages are added on stage 4 via apt-get install. The
# idea is to be able to add extra packages without upgrading the base system.
# When we do upgrade the base system we normally move the extras to base.
#
# Notes:
#
#  - some packages (such as CPU microcode updates) are in non-free.
#  - systemd-container seems to be required by host systemd-nspawn.
#  - must explicitly select between dbus and dbus-broker
#  - not installing linux-image-* since building custom below
#
release="testing"
components="main,contrib,non-free"
mirror="http://http.us.debian.org/debian/"
#mirror="https://http.us.debian.org/debian/"

base_pkgs="locales,klibc-utils,sudo"
base_pkgs+=",udev,dbus,systemd-timesyncd,systemd-container"
base_pkgs+=",kmod,linux-base,firmware-linux-free,irqbalance"
base_pkgs+=",pciutils,usbutils,dmidecode"
base_pkgs+=",hdparm,btrfs-progs"
base_pkgs+=",lm-sensors,smartmontools,linux-cpupower"

base_pkgs+=",psmisc"
base_pkgs+=",net-tools,iproute2,iptables,isc-dhcp-client"
base_pkgs+=",ifupdown,bridge-utils,dnsmasq,postfix"
base_pkgs+=",iputils-ping,wget,curl,ca-certificates"
base_pkgs+=",openssh-client,openssh-server"
base_pkgs+=",tftp-hpa,tftpd-hpa"

base_pkgs+=",zstd,xz-utils"
base_pkgs+=",less,nano,time"
base_pkgs+=",g++,make"

base_pkgs+=",qemu-utils,socat"

case "$arch" in
  x86_64)
    base_pkgs+=",cpuid"
    base_pkgs+=",intel-microcode"  #,amd64-microcode
    base_pkgs+=",qemu-system-x86"
    ;;
  aarch64)
    base_pkgs+=",qemu-system-arm"
    ;;
esac

extra_pkgs=""

# Options.
#
stage=1
stage_max=6
arch_variant=
kernel_source=

while [ "$#" -gt 0 ]; do
  case "$1" in
    --stage)
      shift
      stage="$1"
      shift
      ;;
    --arch-variant)
      shift
      arch_variant="$1"
      shift
      ;;
    --kernel-source)
      shift
      kernel_source="$1"
      shift
      ;;
    -*)
      error "unknown option: $1"
      ;;
    *)
      break
      ;;
  esac
done

if [ "$stage" -lt "1" -o "$stage" -gt "$stage_max" ]; then
  error "invalid stage number $stage"
fi

arch_with_variant="$arch"
if [ -n "$arch_variant" ]; then
  arch_with_variant="$arch_with_variant-$arch_variant"
fi

# Extract version.
#
version="$(sed -n -re 's/^version: ([0-9]+\.[0-9]+\.[0-9]+).*$/\1/p' ./manifest)"
version_id="$(sed -n -re 's/^([0-9]+\.[0-9]+)\.[0-9]+$/\1/p' <<<"$version")"
if [ -z "$version" -o -z "$version_id" ]; then
  error "unable to extract version from manifest"
fi

# Btrfs subvolume manipulation.
#
function subvol_delete () # <path>
{
  # The subvol show (just as list) needs root.
  #
  if sudo btrfs subvol show "$1" 1>/dev/null 2>&1; then
    btrfs property set -ts "$1" ro false
    btrfs subvol delete "$1"
  fi
}

function subvol_create () # <path>
{
  btrfs subvol create "$@"
}

function subvol_snapshot () # [-r] <src-path> <dst-path>
{
  btrfs subvol snapshot "$@"
}

# Clean up the working subvolume and all the snapshots starting from the
# requested stage. Also, if stage is not 1, then restore the working subvolume
# from the previous stage snapshot.
#
subvol_delete "$root"

for i in $(seq "$stage" "$stage_max"); do
  subvol_delete "$root-$i"
done

if [ "$stage" -gt "1" ]; then
  i="$(($stage - 1))"
  info "restoring working subvolume from stage $i snapshot"
  subvol_snapshot "$root-$i" "$root"
fi

# Spawn a systemd namespace container (systemd-nspawn)
#
function nspawn () # <root> <systemd-nspawn-args>
{
  local r="$1"
  shift

  # systemd-nspawn appears to carry over the root directory owner into the
  # container which then causes other issues (Debian bug#950684).
  #
  # @@ Looking around didn't reveal any way (e.g., an option) to fix this.
  #    Perhaps newer systemd-nspawn does the right thing automatically?
  #
  ug="$(stat --format="%G:%U" "$r")"
  sudo chown root:root "$r"

  sudo systemd-nspawn --register=no -D "$r" "$@"

  sudo chown "$ug" "$r"

  # systemd-nspawn may create the /var/lib/{machines,portables} subvolumes
  # which prevents the deletion of the containing submodule. So we clean'em
  # up.
  #
  for s in /var/lib/machines /var/lib/portables; do
    if sudo btrfs subvol show "$r$s" 1>/dev/null 2>&1; then
      sudo btrfs subvol delete "$r$s"
    fi
  done
}

# (Over)write or append to a file in the installation root, for example:
#
# write <<<'localhost' /etc/hostname
#
function write () # <path> [<root>]
{
  local r="$2"
  if [ -z "$r" ]; then
    r="$root"
  fi

  sudo tee "$r$1" >/dev/null
}

function append () # <path> [<root>]
{
  local r="$2"
  if [ -z "$r" ]; then
    r="$root"
  fi

  sudo tee -a "$r$1" >/dev/null
}

# Stage 1: debootstrap, phase 1.
#
if [ "$stage" -eq "1" ]; then

  subvol_create "$root"

  sudo debootstrap \
    --foreign \
    --arch="$debian_arch" \
    --merged-usr \
    --variant=minbase \
    --components="$components" \
    --include="$base_pkgs" \
    "$release" "$root" "$mirror"

  # Post-phase 1 fixups.
  #

  # Set the initial hostname to 'localhost'. This value is detected and
  # overriden by /sbin/dhclient-script if the DHCP server sends host-name.
  #
  write <<<'localhost' /etc/hostname

  # Set timezone to UTC (picked up by tzdata package during stage 2).
  #
  write <<<'Etc/UTC' /etc/timezone

  subvol_snapshot -r "$root" "$root-1"
fi

# Stage 2: debootstrap, phase 2.
#
if [ "$stage" -le "2" ]; then

  # Create a bootstrap script that will finish the bootstrap from within the
  # installation via systemd-nspawn.
  #
  sudo mkdir "$root/bootstrap"

  write <<EOF /bootstrap/bootstrap
#!/bin/bash

trap "exit 1" ERR
set -x

# Hack around systemd bug#79306 (changes /etc/localtime) by removing it now
# and making readonly below. @@ See --timezone systemd-nspawn option?
#
rm /etc/localtime

# Both nspawn and debootstrap try to mount /proc /sys (Debian bug#840372).
#
# @@ TMP this now causes issues with newer systemd.
#
#mkdir /tmp/proc /tmp/sys
#mount --move /proc /tmp/proc
#mount --move /sys /tmp/sys

# systemd-nspawn "helpfully" creates a /lib64 symlink that then trips
# up is-usr-merged package (Debain bug #1019575).
#
if [ $arch = aarch64 ]; then
  rm /lib64
fi

# Run second stage of debootstrap.
#
if ! /debootstrap/debootstrap --second-stage; then
  cat /debootstrap/debootstrap.log 1>&2
  bash
  exit 1
fi

rm -f /etc/localtime
cp /usr/share/zoneinfo/UTC /etc/localtime
chattr +i /etc/localtime

# Change /etc/os-release, /etc/issue, /etc/motd.
#
cat <<EOF1 >/etc/os-release
NAME="Build OS"
VERSION="$version"
ID=buildos
ID_LIKE=debian
PRETTY_NAME="Build OS $version (Based on Debian)"
VERSION_ID="$version_id"
HOME_URL="https://build2.org/"
SUPPORT_URL="https://lists.build2.org/"
BUG_REPORT_URL="https://lists.build2.org/"
EOF1

cat <<EOF1 >/etc/issue
Build OS $version (Based on Debian) \n \l

EOF1

cat <<EOF1 >/etc/motd

Welcome to Build OS $version (https://build2.org)!

EOF1

# Make root login passwordless (we disable SSH root login in init).
#
passwd -d root

# Enable IPv4 forwarding (used for private bridge NAT).
#
sed -i -e 's/^# *\(net.ipv4.ip_forward\).*/\1=1/' /etc/sysctl.conf

# Setup locale. We only support en_US.UTF-8.
#
sed -i -e 's/^# *\(en_US.UTF-8\)/\1/' /etc/locale.gen
locale-gen --purge

cat <<EOF1 >/etc/default/locale
LANG="en_US.UTF-8"
LANGUAGE="en_US:en"
EOF1

EOF
  sudo chmod u+x "$root/bootstrap/bootstrap"

  # Notes:
  #
  # - Failed to create directory .../sys/fs/selinux: Read-only file system is
  #   harmless and fixed upstream (systemd issue#3748).
  #
  nspawn "$root" /bootstrap/bootstrap

  subvol_snapshot -r "$root" "$root-2"
fi

# Stage 3: kernel build.
#
if [ "$stage" -le "3" ]; then

  # Create the setup service that will be used by both this stage and the
  # setup stage below. Note that we will do actual building (which requires
  # installing extra packages) in a snapshot on the side.
  #
  sudo mkdir -p "$root/bootstrap"

  # Note that when started via systemd-nspawn, we get /dev/console, not
  # /dev/tty0.
  #
  write <<EOF /usr/lib/systemd/system/buildos-setup.service
[Unit]
Description=build os setup
After=multi-user.target
Conflicts=console-getty.service

[Service]
Type=idle
TimeoutStartSec=infinity
RemainAfterExit=true
ExecStart=/bootstrap/setup
StandardInput=tty-force
StandardOutput=inherit
StandardError=inherit
TTYPath=/dev/console
TTYReset=yes
TTYVHangup=yes

[Install]
WantedBy=multi-user.target
EOF

  sudo mkdir -p "$root/etc/systemd/system/multi-user.target.wants"
  sudo ln -sf "$root/usr/lib/systemd/system/buildos-setup.service" \
    "$root/etc/systemd/system/multi-user.target.wants/buildos-setup.service"

  # Create the kernel build snapshot, write the script that does the build
  # from within the installation and boot it up via systemd-nspawn --boot.
  #
  # Add `bash` before shutdown if need to debug/check things. But note that it
  # does not work well with `... | tee bootstrap.log`!
  #
  subvol_delete "$root-3-kernel"
  subvol_snapshot "$root" "$root-3-kernel"

  # Copy custom kernel source.
  #
  if [ -n "$kernel_source" ]; then
    sudo install -m 644 "$kernel_source" "$root-3-kernel/usr/src/"
  fi

  # Copy patches.
  #
  #sudo cp ./patches/tftp-hpa-partial-upload.patch "$root-3-kernel/bootstrap/"

  write <<EOF /bootstrap/setup "$root-3-kernel"
#!/bin/bash

trap "exit 1" ERR
set -x

# Add deb-src to each deb entry in /etc/apt/sources.list.
#
sed -ri -e 's/^deb (.+)/deb \1\ndeb-src \1/' /etc/apt/sources.list

apt-get update
apt-get install -y build-essential devscripts

# Build custom/patched packages.
#
cd /usr/src

if false; then
mkdir tftpd-hpa
cd tftpd-hpa
apt-get install -y libwrap0-dev
apt-get source tftpd-hpa
cd tftp-hpa-*/
patch -p1 </bootstrap/tftp-hpa-partial-upload.patch
dch -n "Apply patches."
cd ../tftp-hpa-*/ # May get renamed.
dpkg-buildpackage -us -uc
cd ../..
fi

# Build kernel.
#
# This seems to be the simplest method of building the standard Debian
# kernel with adjusted configuration. Taken from the Debian Kernel Handbook.
#
apt-get update
apt-get install -y bison flex
apt-get install -y libelf-dev
apt-get install -y libssl-dev
apt-get install -y rsync
apt-get install -y dwarves

cd /usr/src

if [ -z "$kernel_source" ]; then
  apt-get install -y linux-source
  tar xf linux-source-*
  mv linux-source-*/ linux
  xzcat linux-config-*/config.${debian_arch}_none_${debian_arch}.xz >linux/.config
else
  tar xf $kernel_source
  mv \$(sed -re 's/(.+)\.tar\..+/\1/' <<<$kernel_source) linux
fi

cd linux

# Adjust configuration.
#
if [ -z "$kernel_source" ]; then

  # Note that SECURITY_LOCKDOWN_LSM forces MODULE_SIG ('select' in Kconfig).
  #
  # Generally, if you disable an option but it still appears enabled after
  # the kernel build, search for 'select XXX' in Kconfig* and also disable
  # any found symbols.
  #
  scripts/config --disable KCSAN
  scripts/config --disable SECURITY_LOCKDOWN_LSM
  scripts/config --disable MODULE_SIG
  scripts/config --set-str BUILD_SALT ''
  scripts/config --set-str SYSTEM_TRUSTED_KEYS ''

  scripts/config --enable  INIT_STACK_NONE
  scripts/config --disable INIT_STACK_ALL_PATTERN
  scripts/config --disable INIT_STACK_ALL_ZERO

  scripts/config --enable  DEBUG_INFO_NONE
  scripts/config --disable DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT

  # aarch64/5.19 additions:
  #
  scripts/config --disable KCOV
  scripts/config --disable SHADOW_CALL_STACK
  scripts/config --disable VIDEO_ADV7511

  # Disable sound subsystem/drivers.
  #
  sed -i -re '/^CONFIG_SND_.+/d' .config
  sed -i -re '/^CONFIG_SOUND_.+/d' .config
  scripts/config --disable CONFIG_SOUND
  scripts/config --disable CONFIG_SND

  # Disable GPU subsystem/drivers.
  #
  # NOTE: this seems to have killed the high-resolution console.
  #
  sed -i -re '/^CONFIG_DRM_.+/d' .config
  scripts/config --disable CONFIG_DRM

  # Disable wireless network drivers.
  #
  scripts/config --disable CONFIG_WLAN
fi

# Adjust kernel command line size limit.
#
sed -i -re 's/^(#define COMMAND_LINE_SIZE).+\$/\1 4096/' arch/x86/include/asm/setup.h
sed -i -re 's/^(#define COMMAND_LINE_SIZE).+\$/\1 4096/' arch/arm64/include/uapi/asm/setup.h

# Change certain network diagnostics from pr_info() to pr_debug() (otherwise
# to drowns out any useful diagnostics).
#
patch net/core/dev.c <<EOP
@@ -0 +0 @@
-		pr_info("device %s %s promiscuous mode\n",
+		pr_debug("device %s %s promiscuous mode\n",
EOP
patch net/bridge/br_stp.c <<EOP
@@ -0 +0 @@
-		br_info(p->br, "port %u(%s) entered %s state\n",
+		br_debug(p->br, "port %u(%s) entered %s state\n",
EOP
patch net/ipv6/addrconf.c <<EOP
@@ -0 +0 @@
-			pr_info("ADDRCONF(NETDEV_CHANGE): %s: link becomes ready\n",
+			pr_debug("ADDRCONF(NETDEV_CHANGE): %s: link becomes ready\n",
EOP

#bash
make oldconfig

make clean
make deb-pkg LOCALVERSION=-buildos KDEB_PKGVERSION=1-1 -j 8

# Clean up and shutdown.
#
rm /bootstrap/setup
systemctl poweroff

EOF
  sudo chmod u+x "$root-3-kernel/bootstrap/setup"

  nspawn "$root-3-kernel" --boot

  # Copy the kernel and packages over and install them.
  #
  #sudo cp "$root-3-kernel/usr/src/tftpd-hpa/tftpd-hpa_"*.deb "$root/usr/src/"
  sudo cp "$root-3-kernel/usr/src/linux-image-"*.deb "$root/usr/src/"

  write <<EOF /bootstrap/setup
#!/bin/bash

trap "exit 1" ERR
set -x

#dpkg -i /usr/src/tftpd-hpa_*.deb
dpkg -i /usr/src/linux-image-*.deb

rm -rf /usr/src/*
cd /
ln -s boot/vmlinuz-* /vmlinuz

# Clean up and shutdown.
#
rm /bootstrap/setup
systemctl poweroff

EOF
  sudo chmod u+x "$root/bootstrap/setup"

  nspawn "$root" --boot

  subvol_snapshot -r "$root" "$root-3"
fi

# Stage 4: setup.
#
if [ "$stage" -le "4" ]; then

  # Write the setup script that will finish the setup (the service is already
  # there from stage 3).
  #
  write <<EOF /bootstrap/setup
#!/bin/bash

trap "exit 1" ERR
set -x

# Install extra packages.
#
apt-get update
for p in \$(sed -e 's/,/ /g' <<<"$extra_pkgs"); do
  apt-get install -y --no-install-recommends "\$p"
done

# We want the utility (smartctl) but not the daemon.
#
systemctl disable smartd

# Create the build user, /build home directory. Make a password-less sudo'er.
#
# Note that we force UID/GID to specific numbers to make sure they are
# consistent across builds.
#
addgroup --gid 2000 build
adduser --uid 2000 --ingroup build --home /build --gecos "" --disabled-password build
adduser build kvm
echo "build ALL=(ALL) NOPASSWD:ALL" >/etc/sudoers.d/build
echo "Defaults:build !syslog" >>/etc/sudoers.d/build
chmod 0440 /etc/sudoers.d/build

# Clean up package cache and fetched repository information.
#
apt-get clean
rm -rf /var/lib/apt/lists

# Strip GCC executables (Debian bug #998841). Fixed but let's keep the command
# around in case it pops up again.
#
#strip \$(find /usr/lib/gcc -type f -executable)

# Clean up /bootstrap.
#
rm /etc/systemd/system/multi-user.target.wants/buildos-setup.service
rm /usr/lib/systemd/system/buildos-setup.service
rm -r /bootstrap

# Shutdown.
#
systemctl poweroff

EOF
  sudo chmod u+x "$root/bootstrap/setup"

  nspawn "$root" --boot

  subvol_snapshot -r "$root" "$root-4"
fi

# Stage 5: generate rootfs.
#
if [ "$stage" -le "5" ]; then

  # Note that there is also initramfs image that is embedded into kernel. In
  # Debian it contains just /dev/console and /root/.
  #
  # Quite a few files/directories are only accessible by root (e.g., /root) so
  # we run under sudo.
  #
  # Note also that without --check=crc32 the kernel will not be able to
  # decompress things.
  #
  root_dirs="build dev etc mnt root usr var"
  root_links="bin sbin lib lib32 lib64"

  info "generating buildos-rootfs.cpio.xz..."

  cd "$root"
  sudo find $root_dirs $root_links -print0 | \
    sudo cpio --null -o -H newc | \
    xz -9 --check=crc32 >"$owd/buildos-rootfs.cpio.xz"
  cd "$owd"

  subvol_snapshot -r "$root" "$root-5"
fi

# Stage 6: generate initrd.
#
if [ "$stage" -le "6" ]; then

  # Generate buildid and store it in /etc/os-release and in buildos-buildid.
  # These are used by the monitor to detect when it's time to reboot.
  #
  # Note that /etc/os-release is a symlink to /usr/lib/os-release.
  #
  buildid="$(uuidgen)"

  sudo tee -a "$root/etc/os-release" >/dev/null <<<"BUILD_ID=\"$buildid\""

  # Install init and buildos monitor.
  #
  sudo install -m 755 ./init "$root/"
  sudo install -m 755 ./buildos "$root/usr/sbin/"
  sudo install -m 644 ./buildos.service "$root/usr/lib/systemd/system/"
  sudo ln -sf "$root/usr/lib/systemd/system/buildos.service" \
    "$root/etc/systemd/system/multi-user.target.wants/buildos.service"

  # Patch in the arch variant.
  #
  sudo sed -i -e "s/^arch_variant=.*/arch_variant=$arch_variant/" \
    "$root/usr/sbin/buildos"

  info "generating buildos-init.cpio.xz..."

  cd "$root"
  sudo cpio -o -H newc <<EOF | \
    xz -9 --check=crc32 >"$owd/buildos-init.cpio.xz"
usr/lib/os-release
init
usr/sbin/buildos
usr/lib/systemd/system/buildos.service
etc/systemd/system/multi-user.target.wants/buildos.service
EOF
  cd "$owd"

  cat buildos-rootfs.cpio.xz \
      buildos-init.cpio.xz >"buildos-initrd-$arch_with_variant"

  # Copy the kernel image next to the initramfs for convenience.
  #
  cp "$root/vmlinuz" "buildos-image-$arch_with_variant"
  echo "$buildid" >"buildos-buildid-$arch_with_variant"

  subvol_snapshot -r "$root" "$root-6"
fi