From f31ea3a48ebea33f85bcd682c72848cadebb7ad1 Mon Sep 17 00:00:00 2001 From: Christian Heusel Date: Sat, 29 Jul 2023 12:05:35 +0200 Subject: fix(build): fix repo detection for special chars in package name Currently we used grep -E, in a way that special characters in the package name such as the '+' in 'ls++' also became part of the regex. This commit switches this to become a literal string matching using awk. Signed-off-by: Christian Heusel --- src/lib/util/pacman.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/util') diff --git a/src/lib/util/pacman.sh b/src/lib/util/pacman.sh index f6c2d5f..541147c 100644 --- a/src/lib/util/pacman.sh +++ b/src/lib/util/pacman.sh @@ -44,7 +44,7 @@ get_pacman_repo_from_pkgbuild() { -S \ --print \ --print-format '%n %r' \ - "${pkgnames[0]}" | grep -E "^${pkgnames[0]} " | awk '{print $2}' + "${pkgnames[0]}" | awk '$1=="'"${pkgnames[0]}"'"{print $2}' ) lock_close 10 -- cgit v1.2.3-70-g09d2 From 547961681a1941bbe480dba3de8a1d6743965e71 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Tue, 5 Sep 2023 00:00:56 +0200 Subject: fix(pkgctl): avoid pacman db dependency checks during repo detection By default the sync operation enables dependency checks during the preparation of the transaction. This isn't necessary for the print option, but is done nevertheless, which leads to issues for example during soname dependency problems. The result is a none functional automatic repo detection. Fix this by explicitly disabling dependency version checks. We specify this option twice to skip all dependency checks. Fixes #189 Signed-off-by: Levente Polyak --- src/lib/util/pacman.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/lib/util') diff --git a/src/lib/util/pacman.sh b/src/lib/util/pacman.sh index 541147c..82a227b 100644 --- a/src/lib/util/pacman.sh +++ b/src/lib/util/pacman.sh @@ -39,9 +39,12 @@ get_pacman_repo_from_pkgbuild() { fi slock 10 "${_DEVTOOLS_PACMAN_CACHE_DIR}.lock" "Locking pacman database cache" + # query repo of passed pkgname, specify --nodeps twice to skip all dependency checks mapfile -t repos < <(pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/multilib.conf" \ --dbpath "${_DEVTOOLS_PACMAN_CACHE_DIR}" \ - -S \ + --sync \ + --nodeps \ + --nodeps \ --print \ --print-format '%n %r' \ "${pkgnames[0]}" | awk '$1=="'"${pkgnames[0]}"'"{print $2}' -- cgit v1.2.3-70-g09d2 From f632659563acb7b8d481fdff9acd8305acf29b98 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Tue, 26 Sep 2023 22:09:41 +0200 Subject: fix(clone): ssh connection may require user input (key unlocking etc) Anything that requires user input (such as a key unlock or hostkey verify) will block the terminal and wait for input which will never come. When cloning or configuring a repo via ssh we therefore initially connect to gitlab to warm the connection. Afterwards users are expected to either have setup a ssh ControlMaster or use something like a ssh agent. Fixes #148 Component: pkgctl repo clone/configure Co-Authored-by: Christian Heusel Signed-off-by: Christian Heusel Signed-off-by: Levente Polyak --- src/lib/repo/clone.sh | 11 +++++++++++ src/lib/repo/configure.sh | 8 ++++++++ src/lib/util/git.sh | 10 ++++++++++ 3 files changed, 29 insertions(+) (limited to 'src/lib/util') diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh index 08bded4..fb927d2 100644 --- a/src/lib/repo/clone.sh +++ b/src/lib/repo/clone.sh @@ -12,6 +12,8 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/gitlab.sh # shellcheck source=src/lib/repo/configure.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/repo/configure.sh +# shellcheck source=src/lib/util/git.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh source /usr/share/makepkg/util/message.sh @@ -52,6 +54,7 @@ pkgctl_repo_clone() { fi # options + local protocol=ssh local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_SSH} local CLONE_ALL=0 local MAINTAINER= @@ -72,6 +75,7 @@ pkgctl_repo_clone() { ;; --protocol=https) GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS} + protocol=https CONFIGURE_OPTIONS+=("$1") shift ;; @@ -82,6 +86,7 @@ pkgctl_repo_clone() { else die "unsupported protocol: %s" "$2" fi + protocol="$2" CONFIGURE_OPTIONS+=("$1" "$2") shift 2 ;; @@ -171,6 +176,12 @@ pkgctl_repo_clone() { if [[ -n "${VERSION}" ]]; then command+=" --switch '${VERSION}'" fi + + # warm up ssh connection as it may require user input (key unlock, hostkey verification etc) + if [[ ${protocol} == ssh ]]; then + git_warmup_ssh_connection + fi + if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${pkgbases[@]}"; then die 'Failed to clone some packages, please check the output' exit 1 diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh index 73300ae..1ddb1b8 100644 --- a/src/lib/repo/configure.sh +++ b/src/lib/repo/configure.sh @@ -10,6 +10,8 @@ _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh # shellcheck source=src/lib/api/gitlab.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/gitlab.sh +# shellcheck source=src/lib/util/git.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh source /usr/share/makepkg/util/config.sh source /usr/share/makepkg/util/message.sh @@ -188,6 +190,12 @@ pkgctl_repo_configure() { if [[ -n ${BOLD} ]]; then export DEVTOOLS_COLOR=always fi + + # warm up ssh connection as it may require user input (key unlock, hostkey verification etc) + if [[ ${proto} == ssh ]]; then + git_warmup_ssh_connection + fi + if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${paths[@]}"; then die 'Failed to configure some packages, please check the output' exit 1 diff --git a/src/lib/util/git.sh b/src/lib/util/git.sh index c4af662..82e4beb 100644 --- a/src/lib/util/git.sh +++ b/src/lib/util/git.sh @@ -7,6 +7,9 @@ DEVTOOLS_INCLUDE_UTIL_GIT_SH=1 _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh + git_diff_tree() { local commit=$1 @@ -22,3 +25,10 @@ git_diff_tree() { "${commit}" \ -- "${path}" } + +git_warmup_ssh_connection() { + msg 'Establishing ssh connection to git@%s' "${GITLAB_HOST}" + if ! ssh -T "git@${GITLAB_HOST}" >/dev/null; then + die 'Failed to establish ssh connection to git@%s' "${GITLAB_HOST}" + fi +} -- cgit v1.2.3-70-g09d2 From b264c7f1c771790c53561f5d3f4de589e0000c05 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sun, 10 Sep 2023 00:00:27 +0200 Subject: feat(util): parallelize srcinfo generation Heavily improve the runtime of huge split packages, by creating an own parallelized high level implementation of the makepkg low level building blocks for srcinfo generation. This reduces the runtine to generate the srcinfo file for thunderbird from 24 seconds down to 1 second. --- src/commitpkg.in | 10 ++++--- src/lib/build/build.sh | 5 ++-- src/lib/util/srcinfo.sh | 69 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 src/lib/util/srcinfo.sh (limited to 'src/lib/util') diff --git a/src/commitpkg.in b/src/commitpkg.in index 016ab22..e17b270 100644 --- a/src/commitpkg.in +++ b/src/commitpkg.in @@ -5,9 +5,12 @@ _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} # shellcheck source=src/lib/common.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/util/srcinfo.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/srcinfo.sh source /usr/share/makepkg/util/util.sh -source /usr/share/makepkg/srcinfo.sh + +set -eo pipefail check_pkgbuild_validity() { @@ -185,10 +188,9 @@ done check_pkgbuild_validity # auto generate .SRCINFO -stat_busy 'Generating .SRCINFO' -write_srcinfo_content > .SRCINFO +# shellcheck disable=SC2119 +write_srcinfo_file git add --force .SRCINFO -stat_done if [[ -n $(git status --porcelain --untracked-files=no) ]]; then stat_busy 'Staging files' diff --git a/src/lib/build/build.sh b/src/lib/build/build.sh index e0982e4..8b4240c 100644 --- a/src/lib/build/build.sh +++ b/src/lib/build/build.sh @@ -14,6 +14,8 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/db/update.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/release.sh # shellcheck source=src/lib/util/git.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh +# shellcheck source=src/lib/util/srcinfo.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/srcinfo.sh # shellcheck source=src/lib/util/pacman.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/pacman.sh # shellcheck source=src/lib/valid-repos.sh @@ -26,8 +28,7 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-inspect.sh source /usr/share/makepkg/util/config.sh source /usr/share/makepkg/util/message.sh -set -e -set -o pipefail +set -eo pipefail pkgctl_build_usage() { diff --git a/src/lib/util/srcinfo.sh b/src/lib/util/srcinfo.sh new file mode 100644 index 0000000..b646dc3 --- /dev/null +++ b/src/lib/util/srcinfo.sh @@ -0,0 +1,69 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_UTIL_SRCINFO_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_UTIL_SRCINFO_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh + +source /usr/share/makepkg/util/util.sh +source /usr/share/makepkg/srcinfo.sh + +set -eo pipefail + + +print_srcinfo() { + local pkgpath=${1:-.} + local outdir pkg pid + local pids=() + + # source the PKGBUILD + # shellcheck source=contrib/makepkg/PKGBUILD.proto + . "${pkgpath}"/PKGBUILD + + # run without parallelization for single packages + if (( ${#pkgname[@]} == 1 )); then + write_srcinfo_content + return 0 + fi + + [[ -z ${WORKDIR:-} ]] && setup_workdir + outdir=$(mktemp --directory --tmpdir="${WORKDIR}" pkgctl-srcinfo.XXXXXXXXXX) + + # fork workload for each split pkgname + for pkg in "${pkgname[@]}"; do + ( + # deactivate errexit to avoid makepkg abort on grep_function + set +e + srcinfo_write_package "$pkg" > "${outdir}/${pkg}" + )& + pids+=($!) + done + + # join workload + for pid in "${pids[@]}"; do + if ! wait "${pid}"; then + return 1 + fi + done + + # collect output + srcinfo_write_global + for pkg in "${pkgname[@]}"; do + srcinfo_separate_section + cat "${outdir}/${pkg}" + done +} + +write_srcinfo_file() { + local pkgpath=${1:-.} + stat_busy 'Generating .SRCINFO' + if ! print_srcinfo "${pkgpath}" > "${pkgpath}"/.SRCINFO; then + error 'Failed to write .SRCINFO file' + return 1 + fi + stat_done +} -- cgit v1.2.3-70-g09d2 From d210079037d34fd32fc9a84e490655204bae103a Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Tue, 9 Jan 2024 00:39:43 +0100 Subject: feat(util): add library function to generate source packages --- src/lib/util/makepkg.sh | 35 +++++++++++++++++++++++++++++++++++ src/offload-build.in | 11 +++++++---- 2 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 src/lib/util/makepkg.sh (limited to 'src/lib/util') diff --git a/src/lib/util/makepkg.sh b/src/lib/util/makepkg.sh new file mode 100644 index 0000000..adb3af6 --- /dev/null +++ b/src/lib/util/makepkg.sh @@ -0,0 +1,35 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_UTIL_MAKEPKG_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_UTIL_MAKEPKG_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/util/srcinfo.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/srcinfo.sh + + +set -e + +makepkg_source_package() { + if (( EUID != 0 )); then + [[ -z ${WORKDIR:-} ]] && setup_workdir + export WORKDIR DEVTOOLS_INCLUDE_COMMON_SH + fakeroot -- bash -$- -c "source '${BASH_SOURCE[0]}' && ${FUNCNAME[0]}" + return + fi + ( + export LIBMAKEPKG_LINT_PKGBUILD_SH=1 + lint_pkgbuild() { :; } + + export LIBMAKEPKG_SRCINFO_SH=1 + write_srcinfo() { print_srcinfo; } + + set +e -- -F --source + # shellcheck source=/usr/bin/makepkg + source "$(command -v makepkg)" + ) +} diff --git a/src/offload-build.in b/src/offload-build.in index 0aa2e38..5238129 100644 --- a/src/offload-build.in +++ b/src/offload-build.in @@ -6,11 +6,16 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/util/makepkg.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/makepkg.sh + source /usr/share/makepkg/util/config.sh # Deprecation warning if [[ -z $_DEVTOOLS_COMMAND ]]; then - colorize warning "${0##*/} is deprecated and will be removed. Use 'pkgctl build --offload' instead" fi @@ -19,8 +24,6 @@ repo=extra arch=x86_64 server=build.archlinux.org -die() { printf "error: $1\n" "${@:2}"; exit 1; } - usage() { cat <<- _EOF_ Usage: ${BASH_SOURCE[0]##*/} [--repo REPO] [--arch ARCHITECTURE] [--server SERVER] -- [ARCHBUILD_ARGS] @@ -84,7 +87,7 @@ load_makepkg_config # transferred, including local sources, install scripts, and changelogs. export TEMPDIR=$(mktemp -d --tmpdir offload-build.XXXXXXXXXX) export SRCPKGDEST=${TEMPDIR} -makepkg --source || die "unable to make source package" +makepkg_source_package || die "unable to make source package" # Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else # but an empty src dir is created in PWD. Remove once fixed in makepkg. -- cgit v1.2.3-70-g09d2 From db8c157eeaac98d23247dac1236642c79a1d257f Mon Sep 17 00:00:00 2001 From: Christian Heusel Date: Fri, 5 Jan 2024 17:43:34 +0100 Subject: chore(build): factor out functions to modify PKGBUILDs Component: pkgctl build Signed-off-by: Christian Heusel --- src/lib/build/build.sh | 12 ++++-------- src/lib/util/pkgbuild.sh | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 src/lib/util/pkgbuild.sh (limited to 'src/lib/util') diff --git a/src/lib/build/build.sh b/src/lib/build/build.sh index 5276653..712be22 100644 --- a/src/lib/build/build.sh +++ b/src/lib/build/build.sh @@ -18,6 +18,8 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/srcinfo.sh # shellcheck source=src/lib/util/pacman.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/pacman.sh +# shellcheck source=src/lib/util/pkgbuild.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/pkgbuild.sh # shellcheck source=src/lib/valid-repos.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-repos.sh # shellcheck source=src/lib/valid-tags.sh @@ -374,20 +376,14 @@ pkgctl_build() { # update pkgver if [[ -n ${PKGVER} ]]; then - if [[ $(type -t pkgver) == function ]]; then - # TODO: check if die or warn, if we provide _commit _gitcommit setter maybe? - warning 'setting pkgver variable has no effect if the PKGBUILD has a pkgver() function' - fi msg "Bumping pkgver to ${PKGVER}" - grep --extended-regexp --quiet --max-count=1 "^pkgver=${pkgver}$" PKGBUILD || die "Non-standard pkgver declaration" - sed --regexp-extended "s|^(pkgver=)${pkgver}$|\1${PKGVER}|g" -i PKGBUILD + pkgbuild_set_pkgver "${PKGVER}" fi # update pkgrel if [[ -n ${PKGREL} ]]; then msg "Bumping pkgrel to ${PKGREL}" - grep --extended-regexp --quiet --max-count=1 "^pkgrel=${pkgrel}$" PKGBUILD || die "Non-standard pkgrel declaration" - sed --regexp-extended "s|^(pkgrel=)${pkgrel}$|\1${PKGREL}|g" -i PKGBUILD + pkgbuild_set_pkgrel "${PKGREL}" fi # edit PKGBUILD diff --git a/src/lib/util/pkgbuild.sh b/src/lib/util/pkgbuild.sh new file mode 100644 index 0000000..ebf8e5f --- /dev/null +++ b/src/lib/util/pkgbuild.sh @@ -0,0 +1,43 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_UTIL_PKGBUILD_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_UTIL_PKGBUILD_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} + +source /usr/share/makepkg/util/message.sh + +set -e + + +# set the pkgver variable in a PKGBUILD +# assumes that the pkgbuild is sourced to detect the presence of a pkgver function +pkgbuild_set_pkgver() { + local new_pkgver=$1 + local pkgver=${pkgver} + + if [[ $(type -t pkgver) == function ]]; then + # TODO: check if die or warn, if we provide _commit _gitcommit setter maybe? + warning 'setting pkgver variable has no effect if the PKGBUILD has a pkgver() function' + fi + + if ! grep --extended-regexp --quiet --max-count=1 "^pkgver=${pkgver}$" PKGBUILD; then + die "Non-standard pkgver declaration" + fi + sed --regexp-extended "s|^(pkgver=)${pkgver}$|\1${new_pkgver}|g" --in-place PKGBUILD +} + +# set the pkgrel variable in a PKGBUILD +# assumes that the pkgbuild is sourced so pkgrel is present +pkgbuild_set_pkgrel() { + local new_pkgrel=$1 + local pkgrel=${pkgrel} + + if ! grep --extended-regexp --quiet --max-count=1 "^pkgrel=${pkgrel}$" PKGBUILD; then + die "Non-standard pkgrel declaration" + fi + sed --regexp-extended "s|^(pkgrel=)${pkgrel}$|\1${new_pkgrel}|g" --in-place PKGBUILD +} + -- cgit v1.2.3-70-g09d2 From fedfc80ca15a196d565b9f5dc5159be594f74da3 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Thu, 18 Jan 2024 02:39:34 +0100 Subject: feat(term): add terminal utils to handle a dynamic spinner The spinner uses a status file that can be used to dynamically update the message. The spinner itself buffers the output in a frame buffer variable before flushing a frame in one go. Signed-off-by: Levente Polyak --- src/lib/common.sh | 3 +- src/lib/util/term.sh | 182 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 src/lib/util/term.sh (limited to 'src/lib/util') diff --git a/src/lib/common.sh b/src/lib/common.sh index 9d5622e..63f43f1 100644 --- a/src/lib/common.sh +++ b/src/lib/common.sh @@ -13,7 +13,7 @@ set +u +o posix $DEVTOOLS_INCLUDE_COMMON_SH # Avoid any encoding problems -export LANG=C +export LANG=C.UTF-8 # Set buildtool properties export BUILDTOOL=devtools @@ -108,6 +108,7 @@ cleanup() { if [[ -n ${WORKDIR:-} ]] && $_setup_workdir; then rm -rf "$WORKDIR" fi + tput cnorm >&2 exit "${1:-0}" } diff --git a/src/lib/util/term.sh b/src/lib/util/term.sh new file mode 100644 index 0000000..853dccf --- /dev/null +++ b/src/lib/util/term.sh @@ -0,0 +1,182 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_UTIL_TERM_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_UTIL_TERM_SH=1 + +set -eo pipefail + + +readonly PKGCTL_TERM_SPINNER_DOTS=Dots +export PKGCTL_TERM_SPINNER_DOTS +readonly PKGCTL_TERM_SPINNER_DOTS12=Dots12 +export PKGCTL_TERM_SPINNER_DOTS12 +readonly PKGCTL_TERM_SPINNER_LINE=Line +export PKGCTL_TERM_SPINNER_LINE +readonly PKGCTL_TERM_SPINNER_SIMPLE_DOTS_SCROLLING=SimpleDotsScrolling +export PKGCTL_TERM_SPINNER_SIMPLE_DOTS_SCROLLING +readonly PKGCTL_TERM_SPINNER_TRIANGLE=Triangle +export PKGCTL_TERM_SPINNER_TRIANGLE +readonly PKGCTL_TERM_SPINNER_RANDOM=Random +export PKGCTL_TERM_SPINNER_RANDOM + +readonly PKGCTL_TERM_SPINNER_TYPES=( + "${PKGCTL_TERM_SPINNER_DOTS}" + "${PKGCTL_TERM_SPINNER_DOTS12}" + "${PKGCTL_TERM_SPINNER_LINE}" + "${PKGCTL_TERM_SPINNER_SIMPLE_DOTS_SCROLLING}" + "${PKGCTL_TERM_SPINNER_TRIANGLE}" +) +export PKGCTL_TERM_SPINNER_TYPES + + +term_cursor_hide() { + tput civis >&2 +} + +term_cursor_show() { + tput cnorm >&2 +} + +term_cursor_up() { + tput cuu1 +} + +term_carriage_return() { + tput cr +} + +term_erase_line() { + tput el +} + +term_erase_lines() { + local lines=$1 + + local cursor_up erase_line + cursor_up=$(term_cursor_up) + erase_line="$(term_carriage_return)$(term_erase_line)" + + local prefix='' + for _ in $(seq 1 "${lines}"); do + printf '%s' "${prefix}${erase_line}" + prefix="${cursor_up}" + done +} + +_pkgctl_spinner_type=${PKGCTL_TERM_SPINNER_RANDOM} +term_spinner_set_type() { + _pkgctl_spinner_type=$1 +} + +# takes a status directory that can be used to dynamically update the spinner +# by writing to the `status` file inside that directory atomically. +# replace the placeholder %spinner% with the currently configured spinner type +term_spinner_start() { + local status_dir=$1 + local parent_pid=$$ + ( + local spinner_type=${_pkgctl_spinner_type} + local spinner_offset=0 + local frame_buffer='' + local spinner status_message line + + local status_file="${status_dir}/status" + local next_file="${status_dir}/next" + local drawn_file="${status_dir}/drawn" + + # assign random spinner type + if [[ ${spinner_type} == "${PKGCTL_TERM_SPINNER_RANDOM}" ]]; then + spinner_type=${PKGCTL_TERM_SPINNER_TYPES[$((RANDOM % ${#PKGCTL_TERM_SPINNER_TYPES[@]}))]} + fi + + # select spinner based on the named type + case "${spinner_type}" in + "${PKGCTL_TERM_SPINNER_DOTS}") + spinner=("⠋" "⠙" "⠹" "⠸" "⠼" "⠴" "⠦" "⠧" "⠇" "⠏") + update_interval=0.08 + ;; + "${PKGCTL_TERM_SPINNER_DOTS12}") + spinner=("⢀⠀" "⡀⠀" "⠄⠀" "⢂⠀" "⡂⠀" "⠅⠀" "⢃⠀" "⡃⠀" "⠍⠀" "⢋⠀" "⡋⠀" "⠍⠁" "⢋⠁" "⡋⠁" "⠍⠉" "⠋⠉" "⠋⠉" "⠉⠙" "⠉⠙" "⠉⠩" "⠈⢙" "⠈⡙" "⢈⠩" "⡀⢙" "⠄⡙" "⢂⠩" "⡂⢘" "⠅⡘" "⢃⠨" "⡃⢐" "⠍⡐" "⢋⠠" "⡋⢀" "⠍⡁" "⢋⠁" "⡋⠁" "⠍⠉" "⠋⠉" "⠋⠉" "⠉⠙" "⠉⠙" "⠉⠩" "⠈⢙" "⠈⡙" "⠈⠩" "⠀⢙" "⠀⡙" "⠀⠩" "⠀⢘" "⠀⡘" "⠀⠨" "⠀⢐" "⠀⡐" "⠀⠠" "⠀⢀" "⠀⡀") + update_interval=0.08 + ;; + "${PKGCTL_TERM_SPINNER_LINE}") + spinner=("⎯" "\\" "|" "/") + update_interval=0.13 + ;; + "${PKGCTL_TERM_SPINNER_SIMPLE_DOTS_SCROLLING}") + spinner=(". " ".. " "..." " .." " ." " ") + update_interval=0.2 + ;; + "${PKGCTL_TERM_SPINNER_TRIANGLE}") + spinner=("◢" "◣" "◤" "◥") + update_interval=0.05 + ;; + esac + + # hide the cursor while spinning + term_cursor_hide + + # run the spinner as long as the parent process didn't terminate + while ps -p "${parent_pid}" &>/dev/null; do + # cache the new status template if it exists + if mv "${status_file}" "${next_file}" &>/dev/null; then + status_message="$(cat "$next_file")" + elif [[ -z "${status_message}" ]]; then + # wait until we either have a new or cached status + sleep 0.05 + fi + + # fill the frame buffer with the current status + local prefix='' + while IFS= read -r line; do + # replace spinner placeholder + line=${line//%spinner%/${spinner[spinner_offset%${#spinner[@]}]}} + + # append the current line to the frame buffer + frame_buffer+="${prefix}${line}" + prefix=$'\n' + done <<< "${status_message}" + + # print current frame buffer + echo -n "${frame_buffer}" >&2 + mv "${next_file}" "${drawn_file}" &>/dev/null ||: + + # setup next frame buffer to clear current content + frame_buffer=$(term_erase_lines "$(awk 'END {print NR}' <<< "${status_message}")") + + # advance the spinner animation offset + (( ++spinner_offset )) + + # sleep for the spinner update interval + sleep "${update_interval}" + done + )& + _pkgctl_spinner_pid=$! + disown +} + +term_spinner_stop() { + local status_dir=$1 + local frame_buffer status_file + + # kill the spinner process + if ! kill "${_pkgctl_spinner_pid}" > /dev/null 2>&1; then + return 1 + fi + unset _pkgctl_spinner_pid + + # acquire last drawn status + status_file="${status_dir}/drawn" + if [[ ! -f ${status_file} ]]; then + return 0 + fi + + # clear terminal based on last status line + frame_buffer=$(term_erase_lines "$(awk 'END {print NR}' < "${status_file}")") + echo -n "${frame_buffer}" >&2 + + # show the cursor after stopping the spinner + term_cursor_show +} -- cgit v1.2.3-70-g09d2 From 3ecba314fc3a7508d80c5450aaae4b5fc507d62b Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Tue, 6 Feb 2024 21:59:11 +0100 Subject: feat(build): consolidate repo flags in build and release subcommand Previously the behavior was inconsistent and not fully fulfilling its purpose of only using --repo once when initially pushing a new and unknown package to the official repositories. Consolidate the behavior by only allowing to use --repo during the initial packaging and disallow any subsequent usage. The expected user experience is to subsequently use --testing or --staging to influence the auto-detection of the build target. This avoids any kind of human error which leads to releasing core packages to extra-testing by accident. Furthermore, allow the build subcommand to automatically fallback to extra as the default stable repository target which greatly improves the usability for AUR or local override builds. Fixes #193 Fixes #191 Component: pkgctl build Component: pkgctl release Signed-off-by: Levente Polyak --- contrib/completion/zsh/_devtools.in | 2 +- doc/man/pkgctl-build.1.asciidoc | 5 ++++- doc/man/pkgctl-release.1.asciidoc | 4 +++- src/lib/build/build.sh | 35 +++++++++++++++++++++-------------- src/lib/release.sh | 29 ++++++++++++++++++----------- src/lib/util/pacman.sh | 7 ++++++- 6 files changed, 53 insertions(+), 29 deletions(-) (limited to 'src/lib/util') diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in index 24e57bf..3aa0f96 100644 --- a/contrib/completion/zsh/_devtools.in +++ b/contrib/completion/zsh/_devtools.in @@ -91,7 +91,7 @@ _pkgctl_db_update_args=( _pkgctl_release_args=( '(-m --message=)'{-m,--message=}"[Use the given as the commit message]:message:" - '(-r --repo=)'{-r,--repo=}"[Specify a target repository (disables auto-detection)]:repo:($DEVTOOLS_VALID_REPOS[*])" + '(-r --repo=)'{-r,--repo=}"[Specify a target repository for new packages]:repo:($DEVTOOLS_VALID_REPOS[*])" '(-s --staging)'{-s,--staging}'[Release to the staging counterpart of the auto-detected repo]' '(-t --testing)'{-t,--testing}'[Release to the testing counterpart of the auto-detected repo]' '(-u --db-update)'{-u,--db-update}'[Automatically update the pacman database after uploading]' diff --git a/doc/man/pkgctl-build.1.asciidoc b/doc/man/pkgctl-build.1.asciidoc index e7ec714..f7abdeb 100644 --- a/doc/man/pkgctl-build.1.asciidoc +++ b/doc/man/pkgctl-build.1.asciidoc @@ -21,7 +21,10 @@ Build Options Specify architectures to build for (disables auto-detection) *--repo* 'REPO':: - Specify a target repository (disables auto-detection) + Specify target repository for new packages not in any official repo. + Fallback to `'extra'` when building packages that are not present in any + official repository yet. Using this option is disallowed if the package is + already released, as it would circumvent the auto-detection safeguard. *-s, --staging*:: Build against the staging counterpart of the auto-detected repo diff --git a/doc/man/pkgctl-release.1.asciidoc b/doc/man/pkgctl-release.1.asciidoc index c991db4..01a0c9e 100644 --- a/doc/man/pkgctl-release.1.asciidoc +++ b/doc/man/pkgctl-release.1.asciidoc @@ -27,7 +27,9 @@ Options Use the given as the commit message *-r, --repo* 'REPO':: - Specify a target repository (disables auto-detection) + Specify target repository for new packages not in any official repo. + Using this option is disallowed if the package is already released, as it + would circumvent the auto-detection safeguard. *-s, --staging*:: Build against the staging counterpart of the auto-detected repo diff --git a/src/lib/build/build.sh b/src/lib/build/build.sh index 64a6ce3..8e9a3bb 100644 --- a/src/lib/build/build.sh +++ b/src/lib/build/build.sh @@ -49,7 +49,7 @@ pkgctl_build_usage() { BUILD OPTIONS --arch ARCH Specify architectures to build for (disables auto-detection) - --repo REPO Specify a target repository (disables auto-detection) + --repo REPO Specify target repository for new packages not in any official repo -s, --staging Build against the staging counterpart of the auto-detected repo -t, --testing Build against the testing counterpart of the auto-detected repo -o, --offload Build on a remote server and transfer artifacts afterwards @@ -89,8 +89,7 @@ pkgctl_build_check_option_group_repo() { local repo=$2 local testing=$3 local staging=$4 - if ( (( testing )) && (( staging )) ) || - ( [[ $repo =~ ^.*-(staging|testing)$ ]] && ( (( testing )) || (( staging )) )); then + if [[ -n "${repo}" ]] || (( testing )) || (( staging )); then die "The argument '%s' cannot be used with one or more of the other specified arguments" "${option}" exit 1 fi @@ -153,8 +152,8 @@ pkgctl_build() { ;; --repo) (( $# <= 1 )) && die "missing argument for %s" "$1" - REPO="${2}" pkgctl_build_check_option_group_repo '--repo' "${REPO}" "${TESTING}" "${STAGING}" + REPO="${2}" shift 2 ;; --arch) @@ -203,13 +202,13 @@ pkgctl_build() { shift ;; -s|--staging) - STAGING=1 pkgctl_build_check_option_group_repo '--staging' "${REPO}" "${TESTING}" "${STAGING}" + STAGING=1 shift ;; -t|--testing) - TESTING=1 pkgctl_build_check_option_group_repo '--testing' "${REPO}" "${TESTING}" "${STAGING}" + TESTING=1 shift ;; -c|--clean) @@ -331,14 +330,22 @@ pkgctl_build() { pkgbuild_checksum=$(b2sum PKGBUILD | awk '{print $1}') msg "Building ${pkgbase}" - # auto-detection of build target - if [[ -z ${pkgrepo} ]]; then - if ! pkgrepo=$(get_pacman_repo_from_pkgbuild PKGBUILD); then - die 'failed to get pacman repo' - fi - if [[ -z "${pkgrepo}" ]]; then - die 'unknown repo, specify --repo for packages not currently in any official repo' - fi + # auto-detect target repository + if ! repo=$(get_pacman_repo_from_pkgbuild PKGBUILD); then + die 'Failed to query pacman repo' + fi + + # fail if an existing package specifies --repo + if [[ -n "${repo}" ]] && [[ -n ${pkgrepo} ]]; then + die 'Using --repo for packages that exist in official repositories is disallowed' + fi + + # assign auto-detected target repository + if [[ -n ${repo} ]]; then + pkgrepo=${repo} + # fallback to extra for unreleased packages + elif [[ -z ${pkgrepo} ]]; then + pkgrepo=extra fi # special cases to resolve final build target diff --git a/src/lib/release.sh b/src/lib/release.sh index 028d92e..700b2b0 100644 --- a/src/lib/release.sh +++ b/src/lib/release.sh @@ -35,7 +35,7 @@ pkgctl_release_usage() { OPTIONS -m, --message MSG Use the given as the commit message - -r, --repo REPO Specify a target repository (disables auto-detection) + -r, --repo REPO Specify target repository for new packages not in any official repo -s, --staging Release to the staging counterpart of the auto-detected repo -t, --testing Release to the testing counterpart of the auto-detected repo -u, --db-update Automatically update the pacman database after uploading @@ -43,8 +43,8 @@ pkgctl_release_usage() { EXAMPLES $ ${COMMAND} - $ ${COMMAND} --repo core-testing --message 'libyay 0.42 rebuild' libfoo libbar - $ ${COMMAND} --staging --db-update libfoo + $ ${COMMAND} --staging --message 'libyay 0.42 rebuild' libfoo libbar + $ ${COMMAND} --repo extra --db-update new-package _EOF_ } @@ -134,15 +134,22 @@ pkgctl_release() { pushd "${path}" >/dev/null pkgbase=$(basename "${path}") - if [[ -n ${REPO} ]]; then - repo=${REPO} - else - if ! repo=$(get_pacman_repo_from_pkgbuild PKGBUILD); then - die 'Failed to get pacman repo' - fi - if [[ -z "${repo}" ]]; then - die 'Unknown repo, please specify --repo for new packages' + # auto-detect target repository + if ! repo=$(get_pacman_repo_from_pkgbuild PKGBUILD); then + die 'Failed to query pacman repo' + fi + + # fail if an existing package specifies --repo + if [[ -n "${repo}" ]] && [[ -n ${REPO} ]]; then + die 'Using --repo for packages that exist in official repositories is disallowed' + fi + + # fail if a new package does not specify --repo + if [[ -z "${repo}" ]]; then + if [[ -z ${REPO} ]]; then + die 'Specify --repo for packages that do not yet exist in official repositories' fi + repo=${REPO} fi if (( TESTING )); then diff --git a/src/lib/util/pacman.sh b/src/lib/util/pacman.sh index 82a227b..620e1a8 100644 --- a/src/lib/util/pacman.sh +++ b/src/lib/util/pacman.sh @@ -38,6 +38,11 @@ get_pacman_repo_from_pkgbuild() { return fi + # update the pacman repo cache if it doesn't exist yet + if [[ ! -d "${_DEVTOOLS_PACMAN_CACHE_DIR}" ]]; then + update_pacman_repo_cache + fi + slock 10 "${_DEVTOOLS_PACMAN_CACHE_DIR}.lock" "Locking pacman database cache" # query repo of passed pkgname, specify --nodeps twice to skip all dependency checks mapfile -t repos < <(pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/multilib.conf" \ @@ -47,7 +52,7 @@ get_pacman_repo_from_pkgbuild() { --nodeps \ --print \ --print-format '%n %r' \ - "${pkgnames[0]}" | awk '$1=="'"${pkgnames[0]}"'"{print $2}' + "${pkgnames[0]}" 2>/dev/null | awk '$1=="'"${pkgnames[0]}"'"{print $2}' ) lock_close 10 -- cgit v1.2.3-70-g09d2 From 98bd7e376091ef8a19d848f865521cb13fdfb577 Mon Sep 17 00:00:00 2001 From: Jakub Klinkovský Date: Sun, 11 Feb 2024 13:37:49 +0100 Subject: fix(build): explicitly add --nosign when building a srcpkg We should explicitly instruct makepkg to not sign the source package, even when the BUILDENV array in makepkg.conf contains 'sign'. The devtools workflow is to always sign separately from building, which is different from makepkg and it should not depend on its configuration. Furthermore, this function is currently used only in offload-build to collect sources that are transferred to the server before the build itself. Signing this source package does not provide any benefits. --- src/lib/util/makepkg.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/lib/util') diff --git a/src/lib/util/makepkg.sh b/src/lib/util/makepkg.sh index adb3af6..22df247 100644 --- a/src/lib/util/makepkg.sh +++ b/src/lib/util/makepkg.sh @@ -28,7 +28,9 @@ makepkg_source_package() { export LIBMAKEPKG_SRCINFO_SH=1 write_srcinfo() { print_srcinfo; } - set +e -- -F --source + # explicitly instruct makepkg to not sign the source package, even when + # the BUILDENV array in makepkg.conf contains 'sign' + set +e -- -F --source --nosign # shellcheck source=/usr/bin/makepkg source "$(command -v makepkg)" ) -- cgit v1.2.3-70-g09d2