Send patches - preferably formatted by git format-patch - to patches at archlinux32 dot org.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDavid Runge <dvzrv@archlinux.org>2021-05-12 20:31:44 +0200
committerDavid Runge <dvzrv@archlinux.org>2021-05-13 18:32:10 +0200
commitcde7296e6ae1eceec0fe0cfc32e37dbe390d1d35 (patch)
treedbca44b58205531325f2670c93da6ef1fd4de362
parent8e44a8b72e279b6c19b10e735fee711bc38365b7 (diff)
ci: Consolidate build-host script
.gitlab/ci/build-host.sh: Change all script-local variables to lower-case and make some of them overridable using environment variables (by providing defaults). Break down overly long commands by splitting them into a list of strings. Use local variables where possible. Change `main()` to use rsync instead of cp to copy the project to the build location more generically. Change `main()` to use rsync instead of cp to copy the build artifacts on the VM from the project's build directory to the output. Remove all unnecessary `function` keywords for function declarations. Replace the dependency on libisoburn's `xorriso` with libarchive's `bsdtar` and util-linux's `blkid` in `prepare_boot()`. Add `print_section_start()` and `print_section_end()` to reduce code duplication and error potential when printing lines for gitlab's collapsible sections (https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections). Document the script's behavior and expectations. Document the understood environment variables and add links to documentation on understood units (in case of size units).
-rwxr-xr-x.gitlab/ci/build-host.sh226
1 files changed, 173 insertions, 53 deletions
diff --git a/.gitlab/ci/build-host.sh b/.gitlab/ci/build-host.sh
index 3377ba7..78bb067 100755
--- a/.gitlab/ci/build-host.sh
+++ b/.gitlab/ci/build-host.sh
@@ -1,88 +1,181 @@
#!/usr/bin/env bash
-# build-host.sh runs build-inside-vm.sh in a qemu VM running the latest Arch installer iso
#
-# nounset: "Treat unset variables and parameters [...] as an error when performing parameter expansion."
-# errexit: "Exit immediately if [...] command exits with a non-zero status."
-set -o nounset -o errexit
-readonly MIRROR="https://mirror.pkgbuild.com"
+# This script runs a build script in a QEMU VM using the latest Arch Linux installation medium.
+# The build script is expected to create an './output' directory in the project's directory (when running in the VM) and
+# place any build artifacts there.
+# After the build script has finished this script will copy all artifacts to a (local) './output' directory and shutdown
+# the VM.
+#
+# Dependencies:
+# - coreutils
+# - curl
+# - libarchive
+# - qemu-headless
+# - util-linux
+#
+# Considered environment variables:
+# ARCHISO_COW_SPACE_SIZE: The amount of RAM to allocate for the copy-on-write space used by archiso (defaults to 1g -
+# see https://man.archlinux.org/man/tmpfs.5 for understood units)
+# ARCHITECTURE: A string to set the CPU architecture (defaults to x86_64)
+# BUILD_SCRIPT: A script that will be called on the host (defaults to ./build-inside-vm.sh)
+# BUILD_SCRIPT_ARGS: The arguments to BUILD_SCRIPT (as a space delimited list)
+# PACKAGE_LIST: A space delimited list of packages to install to the virtual machine
+# PACMAN_MIRROR: The pacman mirror to use (defaults to "https://mirror.pkgbuild.com")
+# QEMU_DISK_SIZE: A string given to fallocate to create a scratch disk to build in (defaults to 8G - see
+# https://man.archlinux.org/man/fallocate.1 for understood units)
+# QEMU_VM_MEMORY: The amount of RAM (in MiB) allocated for the QEMU virtual machine (defaults to 1024)
+# QEMU_LOGIN_TIMEOUT: The maximum time (in seconds) to wait for the initial prompt in the VM to appear (defaults to 60)
+# QEMU_PACKAGES_TIMEOUT: The maximum time (in seconds) to wait for output from pacman when installing packages (defaults
+# to 120)
+# QEMU_BUILD_TIMEOUT: The maximum time (in seconds) to wait for output from the build script (defaults to 1800)
+# QEMU_COPY_ARTIFACTS_TIMEOUT: The maximum time (in seconds) to wait for output from the action of copying the build
+# artifacts from the VM to a local directory (defaults to 60)
+
+
+set -euo pipefail
+readonly orig_pwd="${PWD}"
+readonly output="${PWD}/output"
+
+# variables with presets/ environmental overrides
+arch="${ARCHITECTURE:-x86_64}"
+script="${BUILD_SCRIPT:-./build-inside-vm.sh}"
+script_args="${BUILD_SCRIPT_ARGS:-}"
+mirror="${PACMAN_MIRROR:-https://mirror.pkgbuild.com}"
+disk_size="${QEMU_DISK_SIZE:-8G}"
+vm_memory="${QEMU_VM_MEMORY:-1024}"
+login_timeout="${QEMU_LOGIN_TIMEOUT:-60}"
+packages_timeout="${QEMU_PACKAGES_TIMEOUT:-120}"
+build_timeout="${QEMU_BUILD_TIMEOUT:-1800}"
+copy_artifacts_timeout="${QEMU_COPY_ARTIFACTS_TIMEOUT:-60}"
+cow_space_size="${ARCHISO_COW_SPACE_SIZE:-1g}"
+packages="${PACKAGE_LIST:-}"
+
+# variables without presets/ environmental overrides
+iso=""
+iso_volume_id=""
tmpdir=""
+tmpdir="$(mktemp --dry-run --directory --tmpdir="${PWD}/tmp")"
-function init() {
- readonly ORIG_PWD="${PWD}"
- readonly OUTPUT="${PWD}/output"
- tmpdir="$(mktemp --dry-run --directory --tmpdir="${PWD}/tmp")"
- mkdir -p "${OUTPUT}" "${tmpdir}"
+print_section_start() {
+ # gitlab collapsible sections start: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections
+ local _section _title
+ _section="${1}"
+ _title="${2}"
+
+ printf "\e[0Ksection_start:%(%s)T:%s\r\e[0K%s\n" '-1' "${_section}" "${_title}"
+}
+print_section_end() {
+ # gitlab collapsible sections end: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections
+ local _section
+ _section="${1}"
+
+ printf "\e[0Ksection_end:%(%s)T:%s\r\e[0K\n" '-1' "${_section}"
+}
+
+init() {
+ print_section_start "create_dirs" "Create required directories"
+
+ mkdir -p "${output}" "${tmpdir}"
cd "${tmpdir}"
+
+ print_section_end "create_dirs"
}
# Do some cleanup when the script exits
-function cleanup() {
+cleanup() {
+ print_section_start "cleanup" "Cleaning up"
+
rm -rf -- "${tmpdir}"
jobs -p | xargs --no-run-if-empty kill
+
+ print_section_end "cleanup"
}
trap cleanup EXIT
# Use local Arch iso or download the latest iso and extract the relevant files
-function prepare_boot() {
- local iso
- local isos=()
+prepare_boot() {
+ local _latest_iso _iso
+ local _isos=()
+
+ print_section_start "prepare_boot" "Prepare boot media"
# retrieve any local images and sort them
- for iso in "${ORIG_PWD}/"archlinux-*-x86_64.iso; do
- if [[ -f "$iso" ]]; then
- isos+=("${iso}")
+ for _iso in "${orig_pwd}/"archlinux-*-"${arch}.iso"; do
+ if [[ -f "${_iso}" ]]; then
+ _isos+=("${_iso}")
fi
done
- if (( ${#isos[@]} >= 1 )); then
- ISO="$(printf '%s\n' "${isos[@]}" | sort -r | head -n1)"
- printf "Using local iso: %s\n" "$ISO"
+ if (( ${#_isos[@]} >= 1 )); then
+ iso="$(printf '%s\n' "${_isos[@]}" | sort -r | head -n1)"
+ printf "Using local iso: %s\n" "$iso"
fi
- if (( ${#isos[@]} < 1 )); then
- LATEST_ISO="$(curl -fs "${MIRROR}/iso/latest/" | grep -Eo 'archlinux-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64.iso' | head -n 1)"
- if [[ -z "${LATEST_ISO}" ]]; then
- echo "Error: Couldn't find latest iso'"
+ if (( ${#_isos[@]} < 1 )); then
+ _latest_iso="$(
+ curl -fs "${mirror}/iso/latest/" | \
+ grep -Eo "archlinux-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-${arch}.iso" | \
+ head -n 1
+ )"
+ if [[ -z "${_latest_iso}" ]]; then
+ echo "Error: Could not find latest iso"
exit 1
fi
- curl -fO "${MIRROR}/iso/latest/${LATEST_ISO}"
- ISO="${PWD}/${LATEST_ISO}"
+ curl -fO "${mirror}/iso/latest/${_latest_iso}"
+ iso="${PWD}/${_latest_iso}"
fi
- # We need to extract the kernel and initrd so we can set a custom cmdline:
- # console=ttyS0, so the kernel and systemd sends output to the serial.
- xorriso -osirrox on -indev "${ISO}" -extract arch/boot/x86_64 .
- ISO_VOLUME_ID="$(xorriso -indev "${ISO}" |& awk -F : '$1 ~ "Volume id" {print $2}' | tr -d "' ")"
+ # Extract the kernel and initrd so that a custom kernel cmdline can be set:
+ # console=ttyS0, so that the kernel and systemd send output to the serial.
+ bsdtar -x -f "${iso}" -C . "arch/boot/${arch}"
+ iso_volume_id="$(blkid -s LABEL -o value "${iso}")"
+
+ print_section_end "prepare_boot"
}
-function start_qemu() {
+start_qemu() {
+ local _kernel_params=(
+ "archisobasedir=arch"
+ "archisolabel=${iso_volume_id}"
+ "cow_spacesize=${cow_space_size}"
+ "ip=dhcp"
+ "net.ifnames=0"
+ "console=ttyS0"
+ "mirror=${mirror}"
+ )
+
+ print_section_start "start_qemu" "Start VM using QEMU"
+
# Used to communicate with qemu
mkfifo guest.out guest.in
# We could use a sparse file but we want to fail early
- fallocate -l 8G scratch-disk.img
+ fallocate -l "${disk_size}" scratch-disk.img
{ qemu-system-x86_64 \
-machine accel=kvm:tcg \
-smp "$(nproc)" \
- -m 4096 \
- -device virtio-net-pci,romfile=,netdev=net0 -netdev user,id=net0 \
- -kernel vmlinuz-linux \
- -initrd initramfs-linux.img \
- -append "archisobasedir=arch archisolabel=${ISO_VOLUME_ID} cow_spacesize=4G ip=dhcp net.ifnames=0 console=ttyS0 mirror=${MIRROR}" \
+ -m "${vm_memory}" \
+ -device virtio-net-pci,romfile=,netdev=net0 \
+ -netdev user,id=net0 \
+ -kernel "arch/boot/${arch}/vmlinuz-linux" \
+ -initrd "arch/boot/${arch}/initramfs-linux.img" \
+ -append "${_kernel_params[*]}" \
-drive file=scratch-disk.img,format=raw,if=virtio \
- -drive file="${ISO}",format=raw,if=virtio,media=cdrom,read-only=on \
- -virtfs "local,path=${ORIG_PWD},mount_tag=host,security_model=none" \
+ -drive "file=${iso},format=raw,if=virtio,media=cdrom,read-only=on" \
+ -virtfs "local,path=${orig_pwd},mount_tag=host,security_model=none" \
-monitor none \
-serial pipe:guest \
-nographic || kill "${$}"; } &
# We want to send the output to both stdout (fd1) and fd10 (used by the expect function)
exec 3>&1 10< <(tee /dev/fd/3 <guest.out)
+
+ print_section_end "start_qemu"
}
# Wait for a specific string from qemu
-function expect() {
+expect() {
local length="${#1}"
local i=0
local timeout="${2:-30}"
@@ -103,17 +196,28 @@ function expect() {
}
# Send string to qemu
-function send() {
+send() {
echo -en "${1}" >guest.in
}
-function main() {
+main() {
+ local _pacman_command=(
+ "pacman -Fy &&"
+ "pacman -Syu --ignore"
+ "\$(pacman -Fq --machinereadable /usr/lib/modules/"
+ "| awk 'BEGIN { FS =\"\\\0\";ORS=\",\" }; { print \$2 }'"
+ "| sort -ut , | head -c -2)"
+ "--noconfirm --needed ${packages}\n"
+ )
+
init
prepare_boot
start_qemu
+ print_section_start "init_build_environment" "Initialize build environment"
+
# Login
- expect "archiso login:" 60
+ expect "archiso login:" "${login_timeout}"
send "root\n"
expect "# "
@@ -128,7 +232,7 @@ function main() {
expect "# "
send "mkfs.ext4 /dev/vda && mkdir /mnt/scratch-disk/ && mount /dev/vda /mnt/scratch-disk && cd /mnt/scratch-disk\n"
expect "# "
- send "cp -a -- /mnt/project/{.gitlab,archiso,configs,scripts} .\n"
+ send "rsync -a --exclude tmp --exclude .git -- /mnt/project/ .\n"
expect "# "
send "mkdir pkg && mount --bind pkg /var/cache/pacman/pkg\n"
expect "# "
@@ -138,22 +242,38 @@ function main() {
expect "# "
# Explicitly lookup mirror address as we'd get random failures otherwise during pacman
- send "curl -sSo /dev/null ${MIRROR}\n"
+ send "curl -sSo /dev/null ${mirror}\n"
expect "# "
- # Install required packages
- send "pacman -Fy && pacman -Syu --ignore \$(pacman -Fq --machinereadable /usr/lib/modules/ | awk 'BEGIN { FS = \"\\\0\";ORS=\",\" }; { print \$2 } ' | sort -ut , | head -c -2) --noconfirm --needed qemu-headless jq dosfstools erofs-utils e2fsprogs libisoburn mtools squashfs-tools zsync\n"
- expect "# " 120
+ print_section_end "init_build_environment"
+ print_section_start "install_packages" "Install packages"
+
+ if [[ -n "${packages}" ]]; then
+ # Install required packages
+ send "${_pacman_command[*]}"
+ expect "# " "${packages_timeout}"
+ fi
+
+ print_section_end "install_packages"
## Start build and copy output to local disk
- send "bash -x ./.gitlab/ci/build-inside-vm.sh ${PROFILE} ${BUILDMODE}\n "
- expect "# " 2400 # mksquashfs can take a very long time
- send "cp -r --preserve=mode,timestamps -- output /mnt/project/tmp/$(basename "${tmpdir}")/\n"
- expect "# " 60
- mv output/* "${OUTPUT}/"
+ send "bash -x ${script} ${script_args}\n "
+ expect "# " "${build_timeout}"
+
+ print_section_start "move_artifacts" "Move artifacts to output directory"
+
+ send "rsync -av -- output /mnt/project/tmp/$(basename "${tmpdir}")/\n"
+ expect "# " "${copy_artifacts_timeout}"
+ mv -- output/* "${output}/"
+
+ print_section_end "move_artifacts"
+ print_section_start "shutdown" "Shutdown the VM"
# Shutdown the VM
send "systemctl poweroff -i\n"
wait
+
+ print_section_end "shutdown"
}
+
main