aboutsummaryrefslogtreecommitdiff
path: root/etc/private/install/brep-install
blob: 37179c2ef0ba51c65aa54d577e5a39ecb743ea03 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
#! /usr/bin/env bash

# file      : etc/private/install/brep-install
# license   : MIT; see accompanying LICENSE file

# Setup HTTP-only brep instance with unsigned package submission support via
# direct repository publishing (brep/handler/submit/submit-pub).
#
# NOTE: this setup should only be used in private/trusted environments.
#
# Unless the --setup option is specified, create the 'brep' group and user and
# re-run itself with the --setup option under this user. In the setup mode
# install and configure the brep instance, automating the instructions from
# the INSTALL file, including:
#
# - Build the build2 toolchain (installing it to /usr/local/) and brep
#   (installing it to ~brep/install/).
#
# - Install PostgreSQL and create brep users/databases.
#
# - Installing Apache2 and configure HTTP server with the brep module.
#
# Note that the script is written for use on Debian-based distributions so you
# will need to adjust it to match other distributions or operating systems.
#
# Options:
#
# --mount
#
#   Mount the virtio-9p device with id 'state' as the /var/brep directory.
#   This directory is expected to either contain the pkg repository or be
#   empty, in which case an empty repository will be automatically
#   initialized. If this option is unspecified, the directory will be created
#   in the local filesystem.
#
# --brep-user
#
#   User and group ids to use when creating the 'brep' group and user. If
#   unspecified, 63700 is used.
#
# --setup
#
#   Install and configure the brep instance, assuming that the 'brep' user
#   already exists and this script is executed as this user.
#
# --clean
#
#   At the end of the brep instance setup remove installation environment-
#   specific traces (host name/IP from the configuration files, etc). Normally
#   you would use this option to make the "clean" machine copy for
#   distribution. Note that if this option is specified, then the brep
#   instance will only be unusable after the machine reboot.
#
usage="Usage: $0 [<options>]"

# build2 toolchain repository certificate fingerprint. Note: this is a
# repository the toolchain installation script downloads the build2 packages
# from.
#
toolchain_repo_cert_fp="70:64:FE:E4:E0:F3:60:F1:B4:51:E1:FA:12:5C:E0:B3:DB:DF:96:33:39:B9:2E:E5:C2:68:63:4C:A6:47:39:43"
#toolchain_repo_cert_fp="EC:50:13:E2:3D:F7:92:B4:50:0B:BF:2A:1F:7D:31:04:C6:57:6F:BC:BE:04:2E:E0:58:14:FA:66:66:21:1F:14"

# brep package repository URL and certificate fingerprint.
#
#brep_repo_url="https://pkg.cppget.org/1/alpha"
#brep_repo_cert_fp="70:64:FE:E4:E0:F3:60:F1:B4:51:E1:FA:12:5C:E0:B3:DB:DF:96:33:39:B9:2E:E5:C2:68:63:4C:A6:47:39:43"
brep_repo_url="https://stage.build2.org/1"
brep_repo_cert_fp="EC:50:13:E2:3D:F7:92:B4:50:0B:BF:2A:1F:7D:31:04:C6:57:6F:BC:BE:04:2E:E0:58:14:FA:66:66:21:1F:14"

owd=`pwd`
trap "{ exit 1; }"  ERR
trap "{ cd $owd; }" EXIT
set -o errtrace # Trap in functions.

function info  () { echo "$*" 1>&2; }
function error () { info "error: $*"; exit 1; }

# Trace a command line, quoting empty arguments as well as those that contain
# spaces.
#
function trace () # <cmd> <arg>...
{
  local s="+"
  while [ "$#" -gt 0 ]; do
    if [ -z "$1" -o -z "${1##* *}" ]; then
      s="$s \"$1\""
    else
      s="$s $1"
    fi

    shift
  done

  info "$s"
}

# Trace and run a command.
#
run () # <args>...
{
  trace "$@"
  "$@"
}

# The chosen fixed id for the 'brep' user. Note: must match the id of the
# 'brep' user on the host.
#
# Note that Linux assigns the [0 99] range for the statically allocated system
# users and [100 499] -- for dynamic allocations by administrators and post-
# install scripts. Debian, in turn, assigns the [100 999] range for the
# dynamically allocated system users and [60000 64999] -- for statically
# allocated on demand "obscure package users".
#
brep_id=63700 # Update the README file on change.

# Parse the command line options and, while at it, compose the options array
# for potential re-execution as the 'brep' user.
#
mount=
setup=
clean=
ops=()

while [ "$#" -gt 0 ]; do
  case "$1" in
    --mount)
      mount=true
      ops+=("$1")
      shift
      ;;
    --brep-user)
      shift
      brep_id="$1"
      shift
      ;;
    --setup)
      setup=true
      shift
      ;;
    --clean)
      clean=true
      ops+=("$1")
      shift
      ;;
    *)
      break # The end of options is encountered.
      ;;
  esac
done

if [ "$#" -ne 0 ]; then
  error "$usage"
fi

scr_exe="$(realpath "${BASH_SOURCE[0]}")"
scr_dir="$(dirname "$scr_exe")"

# Unless we are not in the setup mode, non-interactively add the 'brep'
# user/group and re-execute the script in the setup mode as this user.
#
if [ ! "$setup" ]; then
  run sudo addgroup --gid "$brep_id" brep

  run sudo adduser --uid "$brep_id" --gid "$brep_id" --disabled-password \
      --gecos "" brep

  run sudo tee -a /etc/sudoers.d/brep >/dev/null <<EOF
brep ALL=(ALL) NOPASSWD:ALL
EOF

  run sudo chmod 0440 /etc/sudoers.d/brep

  # Use --session-command rather than --command|-c to make sure that when the
  # su program receives SIGINT (Ctrl-C) it kills not just its child process
  # but also all its descendants.
  #
  # Note: here we rely on ops to not contain spaces or be empty.
  #
  run exec sudo su -l brep --session-command "'$scr_exe' --setup ${ops[*]}"
fi

# Here we assume that we are executed as brep user.
#
run cd "$HOME"

# Mount the brep state directory, if requested. Note that otherwise, the
# directory will be created later, in the local filesystem by the brep-startup
# script.
#
if [ "$mount" ]; then
  run sudo mkdir -p /var/brep

  run sudo tee -a /etc/fstab >/dev/null <<EOF
state /var/brep 9p trans=virtio,version=9p2000.L,posixacl,cache=none,_netdev 0 0
EOF

  run sudo mount -a
fi

# Install the prerequisite binary packages.
#
run sudo apt-get --yes update
run sudo apt-get --yes install --no-install-recommends g++
run sudo apt-get --yes install --no-install-recommends postgresql postgresql-contrib libpq-dev
run sudo apt-get --yes install --no-install-recommends apache2 libapr1-dev libapreq2-dev apache2-dev
run sudo apt-get --yes install --no-install-recommends acl rsync
run sudo apt-get clean

# Install build2 toolchain.
#
run mkdir build2-build
run cd build2-build

# Look for the toolchain installation script in this script directory.
#
run cp "$(echo "$scr_dir"/build2-install-*.sh)" .
run sh ./build2-install-*.sh --no-check --yes --trust "$toolchain_repo_cert_fp"
#run sh ./build2-install-*.sh --no-check --yes --local

run cd .. # Back to brep home.

# Grant Apache2 read access to the module and configuration.
#
run setfacl -m  g:www-data:rx "$HOME"
run setfacl -dm g:www-data:rx "$HOME"

# Install brep.
#
run mkdir brep
run cd brep

run bpkg create                               \
 cc                                           \
 config.cc.coptions="-O3"                     \
 config.cc.poptions="-I$(apxs -q includedir)" \
 config.bin.lib=shared                        \
 config.bin.rpath="$HOME/install/lib"         \
 config.install.root="$HOME/install"

run bpkg add "$brep_repo_url"
run bpkg fetch --trust "$brep_repo_cert_fp"
run bpkg build --yes brep ?sys:libapr1 ?sys:libapreq2 ?sys:libpq
run bpkg install brep

run cd .. # Back to brep home.

# Create PostgreSQL user and databases.
#
# Note that while we could probably omit the build-related setup, let's keep
# it to stay close to the instructions in the INSTALL file and to simplify the
# potential future configuration of the brep instance as a build2 build bot
# controller.
#
run sudo sudo -u postgres psql <<EOF
CREATE DATABASE brep_package
TEMPLATE template0
ENCODING 'UTF8'
LC_COLLATE 'en_US.UTF8'
LC_CTYPE 'en_US.UTF8';

CREATE DATABASE brep_build
TEMPLATE template0
ENCODING 'UTF8'
LC_COLLATE 'en_US.UTF8'
LC_CTYPE 'en_US.UTF8';

CREATE USER brep;

GRANT ALL PRIVILEGES ON DATABASE brep_package, brep_build TO brep;

CREATE USER "www-data" INHERIT IN ROLE brep;

CREATE USER "brep-build" INHERIT IN ROLE brep PASSWORD '-';

\c brep_package
GRANT ALL PRIVILEGES ON SCHEMA public TO brep;

\c brep_build
GRANT ALL PRIVILEGES ON SCHEMA public TO brep;
EOF

# Create the "staging" package database for the submit-pub package submission
# handler.
#
run sudo sudo -u postgres psql <<EOF
CREATE DATABASE brep_submit_package
TEMPLATE template0
ENCODING 'UTF8'
LC_COLLATE 'en_US.UTF8'
LC_CTYPE 'en_US.UTF8';

GRANT ALL PRIVILEGES ON DATABASE brep_submit_package TO brep;

\c brep_submit_package
GRANT ALL PRIVILEGES ON SCHEMA public TO brep;
EOF

# Make sure the 'brep' and Apache2 user's logins work properly.
#
q="SELECT current_database();"
run psql -d brep_package        -c "$q" >/dev/null
run psql -d brep_build          -c "$q" >/dev/null
run psql -d brep_submit_package -c "$q" >/dev/null

run sudo sudo -u www-data psql -d brep_package -c "$q" >/dev/null
run sudo sudo -u www-data psql -d brep_build   -c "$q" >/dev/null

# Setup the connection between the databases.
#
run sudo sudo -u postgres psql -d brep_build <<EOF
CREATE EXTENSION postgres_fdw;

CREATE SERVER package_server
FOREIGN DATA WRAPPER postgres_fdw
OPTIONS (dbname 'brep_package', updatable 'true');

GRANT USAGE ON FOREIGN SERVER package_server to brep;

CREATE USER MAPPING FOR PUBLIC
SERVER package_server
OPTIONS (user 'brep-build', password '-');
EOF

# Allow brep-build user to access the brep_package database.
#
f="$(run sudo sudo -u postgres psql -t -A -c "show hba_file;")"
s="# TYPE  DATABASE      USER        ADDRESS  METHOD\nlocal   brep_package  brep-build           md5\n\n"

run sudo sed --in-place=.bak "1s/^/$s/" "$f"
run sudo systemctl restart postgresql

# Enable creating database tables with columns of the case-insensitive
# character string type.
#
q="CREATE EXTENSION citext;"
run sudo sudo -u postgres psql -d brep_package        <<<"$q"
run sudo sudo -u postgres psql -d brep_build          <<<"$q"
run sudo sudo -u postgres psql -d brep_submit_package <<<"$q"

# Copy the brep module configuration.
#
# Note: must be done before bin/brep-startup execution, which adjusts the
# configuration.
#
run mkdir config
run cp "$scr_dir/brep-module.conf" config/

# Initialize the brep private instance, in particular creating the database
# schemas and running the brep loader.
#
run mkdir bin/
run cp "$scr_dir/brep-startup" bin/
run bin/brep-startup

# Smoke test the database schemas.
#
run psql -d brep_package        -c 'SELECT canonical_name, summary FROM repository' >/dev/null
run psql -d brep_build          -c 'SELECT package_name FROM build'                 >/dev/null
run psql -d brep_build          -c 'SELECT DISTINCT name FROM build_package'        >/dev/null
run psql -d brep_submit_package -c 'SELECT canonical_name, summary FROM repository' >/dev/null

# Setup executing the brep-startup script on boot.
#
run sudo cp "$scr_dir/brep-startup.service" /etc/systemd/system/

run sudo systemctl start brep-startup.service  # Make sure there are no issues.
run sudo systemctl enable brep-startup.service

# Prepare directories for the package submission service.
#
run mkdir submit-data
run mkdir submit-temp
run setfacl -m g:www-data:rwx submit-data
run setfacl -m g:www-data:rwx submit-temp

# Make the Apache2 user owned directories fully accessible by the 'brep' user
# (which the submit-pub submission handler will run as).
#
run setfacl -dm g:brep:rwx submit-data
run setfacl -dm g:brep:rwx submit-temp

# Add the Apache2 user to sudoers, so the submission handler can re-execute
# itself as the 'brep' user.
#
run sudo tee -a /etc/sudoers.d/www-data >/dev/null <<EOF
www-data ALL=(ALL) NOPASSWD:ALL
EOF

run sudo chmod 0440 /etc/sudoers.d/www-data

# Setup the Apache2 module.
#
run sudo mkdir -p /var/www/brep/log/

run sudo cp "$scr_dir/brep-apache2.conf" /etc/apache2/sites-available/000-brep.conf
run sudo cp "$scr_dir/brep-logrotate"    /etc/logrotate.d/brep

run sudo a2dissite --purge -q 000-default
run sudo a2ensite          -q 000-brep

run sudo systemctl restart apache2
run sudo systemctl status  apache2 >/dev/null

# Make sure the Apache2 service depends on PostgreSQL and
# brep-startup.service, so that they are started in proper order.
#
run sudo mkdir -p /etc/systemd/system/apache2.service.d/
run sudo tee      /etc/systemd/system/apache2.service.d/postgresql.conf >/dev/null <<EOF
[Unit]
Requires=postgresql.service
After=postgresql.service
EOF

run sudo tee      /etc/systemd/system/apache2.service.d/brep-startup.conf >/dev/null <<EOF
[Unit]
Requires=brep-startup.service
After=brep-startup.service
EOF

run sudo mkdir -p /etc/systemd/system/postgresql.service.d/
run sudo tee      /etc/systemd/system/postgresql.service.d/apache2.conf >/dev/null <<EOF
[Unit]
Wants=apache2.service
EOF

run sudo systemctl daemon-reload

# Verify that Apache2 is stopped after PostgreSQL is stopped.
#
run sudo systemctl stop postgresql

ec="0"
run sudo systemctl status apache2 >/dev/null || ec="$?"

if [ "$ec" -ne 3  ]; then
  error "exit code 3 (unit is not active) is expected instead of $ec"
fi

# Verify that Apache2 is started after PostgreSQL is started.
#
run sudo systemctl start postgresql

run sleep 3
run sudo systemctl status apache2 >/dev/null

# Setup periodic loader execution.
#
run sudo cp "$scr_dir/brep-load.service" /etc/systemd/system/
run sudo cp "$scr_dir/brep-load.timer"   /etc/systemd/system/

run sudo systemctl start brep-load.service # Make sure there are no issues.

run sudo systemctl start  brep-load.timer
run sudo systemctl status brep-load.timer >/dev/null
run sudo systemctl enable brep-load.timer
run sudo systemctl status brep-load.timer >/dev/null

# Cleanup the installation environment-specific traces, if requested.
#
if [ "$clean" ]; then

  # Stop the relevant services.
  #
  run sudo systemctl stop brep-load.timer
  run sudo systemctl stop apache2

  # Remove the host name/IP from the configuration.
  #
  run cp "$scr_dir/brep-module.conf" config/ # Adjusted by brep-startup.
  run rm config/loadtab                      # Recreated by brep-startup.

  # Finally, stop networking and cleanup the DHCP lease information.
  #
  # Note that after networking is stopped, sudo prints the 'unable to resolve
  # host' diagnostics while trying to obtain the host IP. Thus, we execute the
  # last two commands via a single sudo call.
  #
  run sudo bash -c "systemctl stop networking && rm -rf /var/lib/dhcp/*.leases"
fi