From a44310b220fab8ddc602ffd757c293288bb2cd3b Mon Sep 17 00:00:00 2001 From: David Runge Date: Thu, 1 Jul 2021 00:34:21 +0200 Subject: Rename build script .gitlab/ci/build_archiso.sh: Rename .gitlab/ci/build-inside-vm.sh to .gitlab/ci/build_archiso.sh. .gitlab-ci.yml: Rename BUILD_SCRIPT to build_archiso.sh. --- .gitlab/ci/build-inside-vm.sh | 270 ------------------------------------------ .gitlab/ci/build_archiso.sh | 270 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 270 insertions(+), 270 deletions(-) delete mode 100755 .gitlab/ci/build-inside-vm.sh create mode 100755 .gitlab/ci/build_archiso.sh (limited to '.gitlab') diff --git a/.gitlab/ci/build-inside-vm.sh b/.gitlab/ci/build-inside-vm.sh deleted file mode 100755 index 6424b32..0000000 --- a/.gitlab/ci/build-inside-vm.sh +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env bash -# -# This script is run within a virtual environment to build the available archiso profiles and their available build -# modes and create checksum files for the resulting images. -# The script needs to be run as root and assumes $PWD to be the root of the repository. -# -# Dependencies: -# * all archiso dependencies -# * coreutils -# * gnupg -# * openssl -# * zsync -# -# $1: profile -# $2: buildmode - -set -euo pipefail -shopt -s extglob - -readonly orig_pwd="${PWD}" -readonly output="${orig_pwd}/output" -readonly tmpdir_base="${orig_pwd}/tmp" -readonly profile="${1}" -readonly buildmode="${2}" -readonly install_dir="arch" - -tmpdir="" -tmpdir="$(mktemp --dry-run --directory --tmpdir="${tmpdir_base}")" -gnupg_homedir="" -codesigning_dir="" -codesigning_cert="" -codesigning_key="" -pgp_key_id="" - -print_section_start() { - # gitlab collapsible sections start: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections - local _section _title - _section="${1}" - _title="${2}" - - printf "\e[0Ksection_start:%(%s)T:%s\r\e[0K%s\n" '-1' "${_section}" "${_title}" -} - -print_section_end() { - # gitlab collapsible sections end: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections - local _section - _section="${1}" - - printf "\e[0Ksection_end:%(%s)T:%s\r\e[0K\n" '-1' "${_section}" -} - -cleanup() { - # clean up temporary directories - print_section_start "cleanup" "Cleaning up temporary directory" - - if [ -n "${tmpdir_base:-}" ]; then - rm -fr "${tmpdir_base}" - fi - - print_section_end "cleanup" -} - -create_checksums() { - # create checksums for files - # $@: files - local _file_path _file_name _current_pwd - _current_pwd="${PWD}" - - print_section_start "checksums" "Creating checksums" - - for _file_path in "$@"; do - cd "$(dirname "${_file_path}")" - _file_name="$(basename "${_file_path}")" - b2sum "${_file_name}" > "${_file_name}.b2" - md5sum "${_file_name}" > "${_file_name}.md5" - sha1sum "${_file_name}" > "${_file_name}.sha1" - sha256sum "${_file_name}" > "${_file_name}.sha256" - sha512sum "${_file_name}" > "${_file_name}.sha512" - ls -lah "${_file_name}."{b2,md5,sha{1,256,512}} - cat "${_file_name}."{b2,md5,sha{1,256,512}} - done - cd "${_current_pwd}" - - print_section_end "checksums" -} - -create_zsync_delta() { - # create zsync control files for files - # $@: files - local _file - - print_section_start "zsync_delta" "Creating zsync delta" - - for _file in "$@"; do - if [[ "${buildmode}" == "bootstrap" ]]; then - # zsyncmake fails on 'too long between blocks' with default block size on bootstrap image - zsyncmake -v -b 512 -C -u "${_file##*/}" -o "${_file}".zsync "${_file}" - else - zsyncmake -v -C -u "${_file##*/}" -o "${_file}".zsync "${_file}" - fi - done - - print_section_end "zsync_delta" -} - -create_metrics() { - local _metrics="${output}/${profile}/metrics.txt" - # create metrics - print_section_start "metrics" "Creating metrics" - - { - # create metrics based on buildmode - case "${buildmode}" in - iso) - printf 'image_size_mebibytes{image="%s"} %s\n' \ - "${profile}" \ - "$(du -m -- "${output}/${profile}/"*.iso | cut -f1)" - printf 'package_count{image="%s"} %s\n' \ - "${profile}" \ - "$(sort -u -- "${tmpdir}/${profile}/iso/"*/pkglist.*.txt | wc -l)" - if [[ -e "${tmpdir}/${profile}/efiboot.img" ]]; then - printf 'eltorito_efi_image_size_mebibytes{image="%s"} %s\n' \ - "${profile}" \ - "$(du -m -- "${tmpdir}/${profile}/efiboot.img" | cut -f1)" - fi - # shellcheck disable=SC2046 - # shellcheck disable=SC2183 - printf 'initramfs_size_mebibytes{image="%s",initramfs="%s"} %s\n' \ - $(du -m -- "${tmpdir}/${profile}/iso/"*/boot/**/initramfs*.img | \ - awk -v profile="${profile}" \ - 'function basename(file) { - sub(".*/", "", file) - return file - } - { print profile, basename($2), $1 }' - ) - ;; - netboot) - printf 'netboot_size_mebibytes{image="%s"} %s\n' \ - "${profile}" \ - "$(du -m -- "${output}/${profile}/${install_dir}/" | tail -n1 | cut -f1)" - printf 'netboot_package_count{image="%s"} %s\n' \ - "${profile}" \ - "$(sort -u -- "${tmpdir}/${profile}/iso/"*/pkglist.*.txt | wc -l)" - ;; - bootstrap) - printf 'bootstrap_size_mebibytes{image="%s"} %s\n' \ - "${profile}" \ - "$(du -m -- "${output}/${profile}/"*.tar*(.gz|.xz|.zst) | cut -f1)" - printf 'bootstrap_package_count{image="%s"} %s\n' \ - "${profile}" \ - "$(sort -u -- "${tmpdir}/${profile}/"*/bootstrap/root.*/pkglist.*.txt | wc -l)" - ;; - esac - } > "${_metrics}" - ls -lah "${_metrics}" - cat "${_metrics}" - - print_section_end "metrics" -} - -create_ephemeral_pgp_key() { - # create an ephemeral PGP key for signing the rootfs image - print_section_start "ephemeral_pgp_key" "Creating ephemeral PGP key" - - gnupg_homedir="$tmpdir/.gnupg" - mkdir -p "${gnupg_homedir}" - chmod 700 "${gnupg_homedir}" - - cat << __EOF__ > "${gnupg_homedir}"/gpg.conf -quiet -batch -no-tty -no-permission-warning -export-options no-export-attributes,export-clean -list-options no-show-keyring -armor -no-emit-version -__EOF__ - - gpg --homedir "${gnupg_homedir}" --gen-key <> "${codesigning_conf}" - openssl req \ - -newkey rsa:4096 \ - -keyout "${codesigning_key}" \ - -nodes \ - -sha256 \ - -x509 \ - -days 365 \ - -out "${codesigning_cert}" \ - -config "${codesigning_conf}" \ - -subj "${codesigning_subj}" \ - -extensions codesigning - - print_section_end "ephemeral_codesigning_key" -} - -run_mkarchiso() { - # run mkarchiso - create_ephemeral_pgp_key - create_ephemeral_codesigning_key - - print_section_start "mkarchiso" "Running mkarchiso" - mkdir -p "${output}/${profile}" "${tmpdir}/${profile}" - GNUPGHOME="${gnupg_homedir}" ./archiso/mkarchiso \ - -D "${install_dir}" \ - -c "${codesigning_cert} ${codesigning_key}" \ - -g "${pgp_key_id}" \ - -o "${output}/${profile}" \ - -w "${tmpdir}/${profile}" \ - -m "${buildmode}" \ - -v "configs/${profile}" - - print_section_end "mkarchiso" - - if [[ "${buildmode}" =~ "iso" ]]; then - create_zsync_delta "${output}/${profile}/"*.iso - create_checksums "${output}/${profile}/"*.iso - fi - if [[ "${buildmode}" == "bootstrap" ]]; then - create_zsync_delta "${output}/${profile}/"*.tar*(.gz|.xz|.zst) - create_checksums "${output}/${profile}/"*.tar*(.gz|.xz|.zst) - fi - create_metrics - - print_section_start "ownership" "Setting ownership on output" - - if [[ -n "${SUDO_UID:-}" ]] && [[ -n "${SUDO_GID:-}" ]]; then - chown -Rv "${SUDO_UID}:${SUDO_GID}" -- "${output}" - fi - print_section_end "ownership" -} - -trap cleanup EXIT - -run_mkarchiso diff --git a/.gitlab/ci/build_archiso.sh b/.gitlab/ci/build_archiso.sh new file mode 100755 index 0000000..6424b32 --- /dev/null +++ b/.gitlab/ci/build_archiso.sh @@ -0,0 +1,270 @@ +#!/usr/bin/env bash +# +# This script is run within a virtual environment to build the available archiso profiles and their available build +# modes and create checksum files for the resulting images. +# The script needs to be run as root and assumes $PWD to be the root of the repository. +# +# Dependencies: +# * all archiso dependencies +# * coreutils +# * gnupg +# * openssl +# * zsync +# +# $1: profile +# $2: buildmode + +set -euo pipefail +shopt -s extglob + +readonly orig_pwd="${PWD}" +readonly output="${orig_pwd}/output" +readonly tmpdir_base="${orig_pwd}/tmp" +readonly profile="${1}" +readonly buildmode="${2}" +readonly install_dir="arch" + +tmpdir="" +tmpdir="$(mktemp --dry-run --directory --tmpdir="${tmpdir_base}")" +gnupg_homedir="" +codesigning_dir="" +codesigning_cert="" +codesigning_key="" +pgp_key_id="" + +print_section_start() { + # gitlab collapsible sections start: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections + local _section _title + _section="${1}" + _title="${2}" + + printf "\e[0Ksection_start:%(%s)T:%s\r\e[0K%s\n" '-1' "${_section}" "${_title}" +} + +print_section_end() { + # gitlab collapsible sections end: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections + local _section + _section="${1}" + + printf "\e[0Ksection_end:%(%s)T:%s\r\e[0K\n" '-1' "${_section}" +} + +cleanup() { + # clean up temporary directories + print_section_start "cleanup" "Cleaning up temporary directory" + + if [ -n "${tmpdir_base:-}" ]; then + rm -fr "${tmpdir_base}" + fi + + print_section_end "cleanup" +} + +create_checksums() { + # create checksums for files + # $@: files + local _file_path _file_name _current_pwd + _current_pwd="${PWD}" + + print_section_start "checksums" "Creating checksums" + + for _file_path in "$@"; do + cd "$(dirname "${_file_path}")" + _file_name="$(basename "${_file_path}")" + b2sum "${_file_name}" > "${_file_name}.b2" + md5sum "${_file_name}" > "${_file_name}.md5" + sha1sum "${_file_name}" > "${_file_name}.sha1" + sha256sum "${_file_name}" > "${_file_name}.sha256" + sha512sum "${_file_name}" > "${_file_name}.sha512" + ls -lah "${_file_name}."{b2,md5,sha{1,256,512}} + cat "${_file_name}."{b2,md5,sha{1,256,512}} + done + cd "${_current_pwd}" + + print_section_end "checksums" +} + +create_zsync_delta() { + # create zsync control files for files + # $@: files + local _file + + print_section_start "zsync_delta" "Creating zsync delta" + + for _file in "$@"; do + if [[ "${buildmode}" == "bootstrap" ]]; then + # zsyncmake fails on 'too long between blocks' with default block size on bootstrap image + zsyncmake -v -b 512 -C -u "${_file##*/}" -o "${_file}".zsync "${_file}" + else + zsyncmake -v -C -u "${_file##*/}" -o "${_file}".zsync "${_file}" + fi + done + + print_section_end "zsync_delta" +} + +create_metrics() { + local _metrics="${output}/${profile}/metrics.txt" + # create metrics + print_section_start "metrics" "Creating metrics" + + { + # create metrics based on buildmode + case "${buildmode}" in + iso) + printf 'image_size_mebibytes{image="%s"} %s\n' \ + "${profile}" \ + "$(du -m -- "${output}/${profile}/"*.iso | cut -f1)" + printf 'package_count{image="%s"} %s\n' \ + "${profile}" \ + "$(sort -u -- "${tmpdir}/${profile}/iso/"*/pkglist.*.txt | wc -l)" + if [[ -e "${tmpdir}/${profile}/efiboot.img" ]]; then + printf 'eltorito_efi_image_size_mebibytes{image="%s"} %s\n' \ + "${profile}" \ + "$(du -m -- "${tmpdir}/${profile}/efiboot.img" | cut -f1)" + fi + # shellcheck disable=SC2046 + # shellcheck disable=SC2183 + printf 'initramfs_size_mebibytes{image="%s",initramfs="%s"} %s\n' \ + $(du -m -- "${tmpdir}/${profile}/iso/"*/boot/**/initramfs*.img | \ + awk -v profile="${profile}" \ + 'function basename(file) { + sub(".*/", "", file) + return file + } + { print profile, basename($2), $1 }' + ) + ;; + netboot) + printf 'netboot_size_mebibytes{image="%s"} %s\n' \ + "${profile}" \ + "$(du -m -- "${output}/${profile}/${install_dir}/" | tail -n1 | cut -f1)" + printf 'netboot_package_count{image="%s"} %s\n' \ + "${profile}" \ + "$(sort -u -- "${tmpdir}/${profile}/iso/"*/pkglist.*.txt | wc -l)" + ;; + bootstrap) + printf 'bootstrap_size_mebibytes{image="%s"} %s\n' \ + "${profile}" \ + "$(du -m -- "${output}/${profile}/"*.tar*(.gz|.xz|.zst) | cut -f1)" + printf 'bootstrap_package_count{image="%s"} %s\n' \ + "${profile}" \ + "$(sort -u -- "${tmpdir}/${profile}/"*/bootstrap/root.*/pkglist.*.txt | wc -l)" + ;; + esac + } > "${_metrics}" + ls -lah "${_metrics}" + cat "${_metrics}" + + print_section_end "metrics" +} + +create_ephemeral_pgp_key() { + # create an ephemeral PGP key for signing the rootfs image + print_section_start "ephemeral_pgp_key" "Creating ephemeral PGP key" + + gnupg_homedir="$tmpdir/.gnupg" + mkdir -p "${gnupg_homedir}" + chmod 700 "${gnupg_homedir}" + + cat << __EOF__ > "${gnupg_homedir}"/gpg.conf +quiet +batch +no-tty +no-permission-warning +export-options no-export-attributes,export-clean +list-options no-show-keyring +armor +no-emit-version +__EOF__ + + gpg --homedir "${gnupg_homedir}" --gen-key <> "${codesigning_conf}" + openssl req \ + -newkey rsa:4096 \ + -keyout "${codesigning_key}" \ + -nodes \ + -sha256 \ + -x509 \ + -days 365 \ + -out "${codesigning_cert}" \ + -config "${codesigning_conf}" \ + -subj "${codesigning_subj}" \ + -extensions codesigning + + print_section_end "ephemeral_codesigning_key" +} + +run_mkarchiso() { + # run mkarchiso + create_ephemeral_pgp_key + create_ephemeral_codesigning_key + + print_section_start "mkarchiso" "Running mkarchiso" + mkdir -p "${output}/${profile}" "${tmpdir}/${profile}" + GNUPGHOME="${gnupg_homedir}" ./archiso/mkarchiso \ + -D "${install_dir}" \ + -c "${codesigning_cert} ${codesigning_key}" \ + -g "${pgp_key_id}" \ + -o "${output}/${profile}" \ + -w "${tmpdir}/${profile}" \ + -m "${buildmode}" \ + -v "configs/${profile}" + + print_section_end "mkarchiso" + + if [[ "${buildmode}" =~ "iso" ]]; then + create_zsync_delta "${output}/${profile}/"*.iso + create_checksums "${output}/${profile}/"*.iso + fi + if [[ "${buildmode}" == "bootstrap" ]]; then + create_zsync_delta "${output}/${profile}/"*.tar*(.gz|.xz|.zst) + create_checksums "${output}/${profile}/"*.tar*(.gz|.xz|.zst) + fi + create_metrics + + print_section_start "ownership" "Setting ownership on output" + + if [[ -n "${SUDO_UID:-}" ]] && [[ -n "${SUDO_GID:-}" ]]; then + chown -Rv "${SUDO_UID}:${SUDO_GID}" -- "${output}" + fi + print_section_end "ownership" +} + +trap cleanup EXIT + +run_mkarchiso -- cgit v1.2.3-54-g00ecf