aboutsummaryrefslogtreecommitdiff
path: root/bpkg-rep
diff options
context:
space:
mode:
authorKaren Arutyunov <karen@codesynthesis.com>2018-10-18 18:42:00 +0300
committerKaren Arutyunov <karen@codesynthesis.com>2018-10-19 19:49:55 +0300
commitfd4b8ec3d690fa7341159b0166b382dd43c6c967 (patch)
tree611f28b71b6e31f2f04b64f7b0db095b8af98d30 /bpkg-rep
parent81a57bde7ad17f6be6b3c604b2c63c623117d0ab (diff)
Add implementation
Diffstat (limited to 'bpkg-rep')
-rw-r--r--bpkg-rep/.gitignore1
-rw-r--r--bpkg-rep/buildfile9
-rw-r--r--bpkg-rep/publish.in309
-rw-r--r--bpkg-rep/utility.bash.in87
4 files changed, 406 insertions, 0 deletions
diff --git a/bpkg-rep/.gitignore b/bpkg-rep/.gitignore
new file mode 100644
index 0000000..39d8ec1
--- /dev/null
+++ b/bpkg-rep/.gitignore
@@ -0,0 +1 @@
+bpkg-rep-publish
diff --git a/bpkg-rep/buildfile b/bpkg-rep/buildfile
new file mode 100644
index 0000000..6f4a025
--- /dev/null
+++ b/bpkg-rep/buildfile
@@ -0,0 +1,9 @@
+# file : bpkg-rep/buildfile
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+./: exe{bpkg-rep-publish}
+
+exe{bpkg-rep-publish}: in{publish} bash{utility}
+
+bash{utility}: in{utility}
diff --git a/bpkg-rep/publish.in b/bpkg-rep/publish.in
new file mode 100644
index 0000000..e828692
--- /dev/null
+++ b/bpkg-rep/publish.in
@@ -0,0 +1,309 @@
+#!/usr/bin/env bash
+
+# file : bpkg-rep/publish.in
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+# Update (bpkg-rep-create(1)) and publish (rsync(1)) an archive-based
+# repository.
+#
+# Pull a pre-cloned (read-only) git repository with the contents of an
+# archive-based bpkg repository. Bail out if nothing changed from the the
+# previous run. Otherwise, regenerate the repository meta-data by running
+# bpkg-rep-create(1) on each section of the repository and, optionally,
+# synchronize it to one or more destinations with rsync.
+#
+# The repository contents are expected to be in the <dir>/1/ subdirectory. The
+# script saves the last successfully published commit in the <dir>.publish
+# file.
+#
+# --destination|-d <host>:<dir>
+#
+# Remote host and directory to rsync the repository to. Note that the
+# trailing 1/ will be added automatically. In other words, the rsync command
+# will be in the form:
+#
+# rsync ... <dir>/1/ <host>:<dir>/1/
+#
+# Repeat this option to specify multiple destinations. In this case, the
+# destinations are synced in the order specified with the first failure
+# terminating the process (so if you have a "primary" destination and a
+# "mirror", you probably want to specify the former first).
+#
+# --timeout <seconds>
+#
+# Git and rsync operation timeout. Specifically, the operation will be
+# aborted if there is no network activity for the specified time. Default is
+# 60 seconds. Note that currently the git timeout is only supported for the
+# http(s) transport.
+#
+# --lock-timeout <seconds>
+#
+# The repository lock timeout. Fail if another instance of the script does
+# not release the repository in the specified time. The default is 0 (do
+# not wait).
+#
+# Note that you will most likely want to specify a non-zero timeout for cron
+# jobs that may potentially overlap.
+#
+# --log-dir <dir>
+#
+# Directory to create the temporary log files in. If unspecified, the stderr
+# is not redirected and no log is created by default.
+#
+# The log is dumped to stderr in case of an error or at the end of execution
+# unless in the quiet mode and is then deleted.
+#
+# --quiet
+#
+# Run quiet. Specifically, don't dump the log to stderr on exit with zero
+# status.
+#
+# --bpkg <path>
+#
+# The package manager program to be used for the repository update. This
+# should be the path to the bpkg executable.
+#
+usage="usage: $0 [<options>] <dir> [<rep-create-options>] [-- <rsync-options>]"
+
+trap "{ exit 1; }" ERR
+set -o errtrace # Trap ERR in functions.
+
+@import bpkg-rep/utility@
+
+# The script own options.
+#
+repo_ver=1
+destinations=()
+timeout=60
+lock_timeout=0
+log_dir=
+quiet=
+bpkg=
+
+while [ $# -gt 0 ]; do
+ case $1 in
+ --destination|-d)
+ shift
+ destinations+=("$1")
+ shift || true
+ ;;
+ --timeout)
+ shift
+ timeout="$1"
+ shift || true
+ ;;
+ --lock-timeout)
+ shift
+ lock_timeout="$1"
+ shift || true
+ ;;
+ --log-dir)
+ shift
+ log_dir="${1%/}"
+ shift || true
+ ;;
+ --quiet)
+ shift
+ quiet=true
+ ;;
+ --bpkg)
+ shift
+ bpkg="$1"
+ shift || true
+ ;;
+ --)
+ shift
+ break
+ ;;
+ *)
+ break
+ ;;
+ esac
+done
+
+# The repository directory.
+#
+repo_dir="${1%/}"
+shift || true
+
+# bpkg-rep-create options.
+#
+rep_create_options=()
+
+while [ $# -gt 0 ]; do
+ case $1 in
+ --)
+ shift
+ break
+ ;;
+ *)
+ rep_create_options+=("$1")
+ shift
+ ;;
+ esac
+done
+
+# rsync options.
+#
+rsync_options=()
+
+while [ $# -gt 0 ]; do
+ rsync_options+=("$1")
+ shift
+done
+
+# Validate options and arguments.
+#
+if [ -z "$repo_dir" ]; then
+ error "$usage"
+fi
+
+if [ ! -d "$repo_dir" ]; then
+ error "'$repo_dir' does not exist or is not a directory"
+fi
+
+# If the log directory is specified then redirect stderr to the log file and
+# setup the trap that dumps it on exit, if required.
+#
+if [ -n "$log_dir" ]; then
+
+ if [ ! -d "$log_dir" ]; then
+ error "'$log_dir' does not exist or is not a directory"
+ fi
+
+ # Create the log file.
+ #
+ log="$(mktemp "$log_dir/$(basename "$repo_dir").XXXXXXXXXX")"
+
+ # Save the stderr file descriptor so we can dump the log into it on exit, if
+ # required. Then redirect it to the log file.
+ #
+ exec {stderr}>&2
+ exec 2>>"$log"
+
+ function exit_trap ()
+ {
+ local status="$?"
+
+ # Dump the log to stderr if exiting with non-zero status or verbose.
+ #
+ if [ $status -ne 0 -o ! "$quiet" ]; then
+
+ # Keep the log if failed to dump for any reason.
+ #
+ if ! cat "$log" >&$stderr; then
+ return
+ fi
+ fi
+
+ rm -f "$log"
+ }
+
+ trap exit_trap EXIT
+fi
+
+# Make sure the commit file is present.
+#
+published_commit="$repo_dir.publish"
+touch "$published_commit"
+
+# Open the reading file descriptor and lock the repository. Fail if unable to
+# lock before timeout.
+#
+exec {cfd}<"$published_commit"
+
+if ! flock -w "$lock_timeout" "$cfd"; then
+ info "another instance is already running"
+ exit 2
+fi
+
+# Pull the repository.
+#
+# Git doesn't support the connection timeout option. The options we use are
+# just an approximation of the former, that, in particular, don't cover the
+# connection establishing. To work around this problem, before running a git
+# command that assumes the remote repository communication we manually check
+# connectivity with the remote repository.
+#
+if ! remote_url="$(git -C "$repo_dir" config --get remote.origin.url)"; then
+ error "'$repo_dir' is not a git repository"
+fi
+
+run check_git_connectivity "$remote_url" "$timeout"
+
+# Fail if no network activity happens during the time specified.
+#
+run git -c http.lowSpeedLimit=1 -c "http.lowSpeedTime=$timeout" \
+-C "$repo_dir" pull -v >&2
+
+# Match the HEAD commit id to the one stored in the file. If it matches, then
+# nothing changed in the repository from the previous run and we can silently
+# bail out.
+#
+commit="$(git -C "$repo_dir" rev-parse HEAD)"
+pc="$(cat <&"$cfd")"
+
+if [ "$commit" == "$pc" ]; then
+ quiet=true
+ exit 0
+fi
+
+# If bpkg path is not specified, then use the bpkg program from the script
+# directory, if present. Otherwise, use the 'bpkg' path.
+#
+if [ -z "$bpkg" ]; then
+ bpkg="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/bpkg"
+
+ if [ ! -x "$bpkg" ]; then
+ bpkg=bpkg
+ fi
+fi
+
+# Find repository sections.
+#
+manifests="$(find "$repo_dir/$repo_ver" -type f -name repositories.manifest)"
+
+# Update the repository sections.
+#
+while read f; do
+ run "$bpkg" rep-create "${rep_create_options[@]}" "$(dirname "$f")"
+done <<<"$manifests"
+
+# rsync (over ssh) the repository to the destinations.
+#
+# Approximate the data transfer timeout via the ServerAlive* ssh options,
+# rounding the timeout up to the nearest multiple of ten.
+#
+# Note: must not contain spaces/use quoting (see rsync -e option).
+#
+n=$(($timeout > 0 ? ($timeout + 9) / 10 : 1))
+ssh_options=(-o ConnectTimeout=$timeout \
+ -o ServerAliveInterval=10 \
+ -o ServerAliveCountMax=$n)
+
+for d in "${destinations[@]}"; do
+
+ # -r (recursive)
+ # -l (copy symlinks and symlinks)
+ # -p (preserve perms)
+ # -t (preserve timestamps)
+ # -O (omit dir timestamps)
+ #
+ # -c (use checksum)
+ # -e (remote shell command)
+ #
+ # --safe-links (ignore symlinks pointing outside the tree)
+ # --delay-updates (first upload all files on the side then move)
+ # --prune-empty-dirs (remove empty dirs)
+ # --delete-after (delete entries after the transfer)
+ #
+ # We also exclude hiddent files (start with dot).
+ #
+ run rsync -v -rlptO -c --safe-links --delay-updates --exclude '.*' \
+--prune-empty-dirs --delete-after -e "ssh ${ssh_options[*]}" \
+"${rsync_options[@]}" "$repo_dir/$repo_ver/" "$d/$repo_ver/" >&2
+
+done
+
+echo "$commit" >"$published_commit"
diff --git a/bpkg-rep/utility.bash.in b/bpkg-rep/utility.bash.in
new file mode 100644
index 0000000..bcf25d8
--- /dev/null
+++ b/bpkg-rep/utility.bash.in
@@ -0,0 +1,87 @@
+# file : bpkg-rep/utility.bash.in
+# copyright : Copyright (c) 2014-2018 Code Synthesis Ltd
+# license : MIT; see accompanying LICENSE file
+
+# Utility functions useful for implementing bpkg repository utilities.
+
+if [ "$bpkg_rep_utility" ]; then
+ return 0
+else
+ bpkg_rep_utility=true
+fi
+
+# Diagnostics.
+#
+function info () { echo "$*" 1>&2; }
+function error () { info "$*"; exit 1; }
+
+# Trace a command line, quoting empty arguments as well as those that contain
+# spaces.
+#
+function trace_cmd () # <cmd> <arg>...
+{
+ local s="+"
+ while [ $# -gt 0 ]; do
+ if [ -z "$1" -o -z "${1##* *}" ]; then
+ s="$s '$1'"
+ else
+ s="$s $1"
+ fi
+
+ shift
+ done
+
+ info "$s"
+}
+
+# Trace the current function name and arguments.
+#
+function trace_func () # <args>...
+{
+ trace_cmd "${FUNCNAME[1]}" "$@"
+}
+
+# Trace and run a command.
+#
+function run () # <cmd> <arg>...
+{
+ trace_cmd "$@"
+ "$@"
+}
+
+# Return lower-case URL scheme or empty string if the argument doesn't look
+# like a URL.
+#
+function url_scheme () # <url>
+{
+ sed -n -re 's%^(.*)://.*$%\L\1%p' <<<"$1"
+}
+
+# Check that the git repository properly responds to the probing request
+# before the timeout (in seconds). Noop for protocols other than HTTP(S).
+#
+function check_git_connectivity () # <repo-url> <timeout>
+{
+ local url="$1"
+ local tmo="$2"
+
+ local s
+ s="$(url_scheme "$url")"
+
+ if [ "$s" == "http" -o "$s" == "https" ]; then
+ local u q
+
+ u="$(sed -n -re 's%^([^?]*).*$%\1%p' <<<"$url")" # Strips query part.
+ q="$(sed -n -re 's%^[^?]*(.*)$%\1%p' <<<"$url")" # Query part.
+
+ if [ -z "$q" ]; then
+ u="$u/info/refs?service=git-upload-pack"
+ else
+ u="$u/info/refs$q&service=git-upload-pack"
+ fi
+
+ # Here we limit the time for the whole operation.
+ #
+ curl -S -s --max-time "$tmo" "$u" >/dev/null
+ fi
+}