From 9a5181db5bfa78d33d3123145ea4c84375f2e8f2 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Fri, 19 Apr 2024 21:16:06 +0200 Subject: feat(pkgctl): add internal update checksums to better control output This allows us to have more control over the output and status logs. Using this method we are able to avoid cluttering the version upgrade subcommand while downloading sources for updating the checksums. Having this internally will also allow us in the future to have smart checksums updating by only trying to change the checksums of sources that have actually changed, for example when adjusting a patch file we should avoid trying to overwrite the archive checksums unintentionally. Component: pkgctl version upgrade --- src/lib/util/makepkg.sh | 20 +++++++++++++++++++ src/lib/util/pkgbuild.sh | 50 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) (limited to 'src/lib/util') diff --git a/src/lib/util/makepkg.sh b/src/lib/util/makepkg.sh index 22df247..d7ec74c 100644 --- a/src/lib/util/makepkg.sh +++ b/src/lib/util/makepkg.sh @@ -22,9 +22,11 @@ makepkg_source_package() { return fi ( + # shellcheck disable=SC2030 disable=SC2031 export LIBMAKEPKG_LINT_PKGBUILD_SH=1 lint_pkgbuild() { :; } + # shellcheck disable=SC2030 disable=SC2031 export LIBMAKEPKG_SRCINFO_SH=1 write_srcinfo() { print_srcinfo; } @@ -35,3 +37,21 @@ makepkg_source_package() { source "$(command -v makepkg)" ) } + +makepkg_generate_integrity() { + if [[ -z ${DEVTOOLS_GENERATE_INTEGRITY} ]]; then + [[ -z ${WORKDIR:-} ]] && setup_workdir + export WORKDIR DEVTOOLS_INCLUDE_COMMON_SH + bash -$- -c "DEVTOOLS_GENERATE_INTEGRITY=1; source '${BASH_SOURCE[0]}' && ${FUNCNAME[0]}" + return + fi + ( + # shellcheck disable=SC2030 disable=SC2031 + export LIBMAKEPKG_LINT_PKGBUILD_SH=1 + lint_pkgbuild() { :; } + + set +e -- --geninteg + # shellcheck source=/usr/bin/makepkg + source "$(command -v makepkg)" + ) +} diff --git a/src/lib/util/pkgbuild.sh b/src/lib/util/pkgbuild.sh index ebf8e5f..245a82f 100644 --- a/src/lib/util/pkgbuild.sh +++ b/src/lib/util/pkgbuild.sh @@ -6,10 +6,13 @@ DEVTOOLS_INCLUDE_UTIL_PKGBUILD_SH=1 _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/util/makepkg.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/makepkg.sh source /usr/share/makepkg/util/message.sh +source /usr/share/makepkg/util/schema.sh -set -e +set -eo pipefail # set the pkgver variable in a PKGBUILD @@ -41,3 +44,48 @@ pkgbuild_set_pkgrel() { sed --regexp-extended "s|^(pkgrel=)${pkgrel}$|\1${new_pkgrel}|g" --in-place PKGBUILD } +pkgbuild_update_checksums() { + local status_file=$1 + local builddir newbuildfile sumtypes newsums + + [[ -z ${WORKDIR:-} ]] && setup_workdir + + builddir=$(mktemp --tmpdir="${WORKDIR}" --directory update-checksums.XXXXXX) + newbuildfile="${builddir}/PKGBUILD" + + # generate new integrity checksums + if ! newsums=$(BUILDDIR=${builddir} makepkg_generate_integrity 2>"${status_file}"); then + printf 'Failed to generate new checksums' + return 1 + fi + + # early exit if no integrity checksums are needed + if [[ -z ${newsums} ]]; then + return 0 + fi + + # replace the integrity sums and write it to a temporary file + sumtypes=$(IFS='|'; echo "${known_hash_algos[*]}") + if ! awk --assign=sumtypes="${sumtypes}" --assign=newsums="${newsums}" ' + $0 ~"^[[:blank:]]*(" sumtypes ")sums(_[^=]+)?\\+?=", $0 ~ "\\)[[:blank:]]*(#.*)?$" { + if (!w) { + print newsums + w++ + } + next + } + + 1 + END { if (!w) print newsums }' PKGBUILD > "${newbuildfile}"; then + printf 'Failed to replace the generated checksums' + return 1 + fi + + # overwrite the original PKGBUILD while preserving permissions + if ! cat -- "${newbuildfile}" > PKGBUILD; then + printf "Failed to write to the PKGBUILD file" + return 1 + fi + + return 0 +} -- cgit v1.2.3-70-g09d2 From 1d433f600e6eecfe685650a06e58d1a8edae9b5d Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sat, 27 Apr 2024 01:50:57 +0200 Subject: feat(db): confirm list of all packages that will be removed Sometimes it isn't obvious which set of packages are removed from a split package when the pkgbase matches also a subset of a pkgbase. This can happen for example with bootstrapping packages, when the intention is to just remove a partial part of the bootstrap pkgbase. To make the intention more explicit, list all to be removed packages and await for confirmation. Component: pkgctl db remove Signed-off-by: Levente Polyak --- README.md | 1 + config/pacman/universe.conf | 112 ++++++++++++++++++++++++++++++++++++ config/pacman/unstable.conf | 83 ++++++++++++++++++++++++++ contrib/completion/bash/devtools.in | 1 + contrib/completion/zsh/_devtools.in | 1 + doc/man/pkgctl-db-remove.1.asciidoc | 3 + src/lib/build/build.sh | 2 +- src/lib/db/remove.sh | 57 +++++++++++++++++- src/lib/release.sh | 2 +- src/lib/util/pacman.sh | 36 +++++++++++- src/lib/util/term.sh | 16 ++++++ 11 files changed, 308 insertions(+), 6 deletions(-) create mode 100644 config/pacman/universe.conf create mode 100644 config/pacman/unstable.conf (limited to 'src/lib/util') diff --git a/README.md b/README.md index eea9c0a..c0bcfdd 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Component: pkgctl db remove - coreutils - curl - diffutils +- expac - fakeroot - findutils - grep diff --git a/config/pacman/universe.conf b/config/pacman/universe.conf new file mode 100644 index 0000000..417114b --- /dev/null +++ b/config/pacman/universe.conf @@ -0,0 +1,112 @@ +# +# /etc/pacman.conf +# +# See the pacman.conf(5) manpage for option and repository directives + +# +# GENERAL OPTIONS +# +[options] +# The following paths are commented out with their default values listed. +# If you wish to use different paths, uncomment and update the paths. +#RootDir = / +#DBPath = /var/lib/pacman/ +#CacheDir = /var/cache/pacman/pkg/ +#LogFile = /var/log/pacman.log +#GPGDir = /etc/pacman.d/gnupg/ +#HookDir = /etc/pacman.d/hooks/ +HoldPkg = pacman glibc +#XferCommand = /usr/bin/curl -L -C - -f -o %o %u +#XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u +#CleanMethod = KeepInstalled +Architecture = auto + +# Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup +#IgnorePkg = +#IgnoreGroup = + +#NoUpgrade = +#NoExtract = + +# Misc options +#UseSyslog +#Color +NoProgressBar +# We cannot check disk space from within a chroot environment +#CheckSpace +VerbosePkgLists +ParallelDownloads = 5 + +# By default, pacman accepts packages signed by keys that its local keyring +# trusts (see pacman-key and its man page), as well as unsigned packages. +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional +#RemoteFileSigLevel = Required + +# NOTE: You must run `pacman-key --init` before first using pacman; the local +# keyring can then be populated with the keys of all official Arch Linux +# packagers with `pacman-key --populate archlinux`. + +# +# REPOSITORIES +# - can be defined here or included from another file +# - pacman will search repositories in the order defined here +# - local/custom mirrors can be added here or in separate files +# - repositories listed first will take precedence when packages +# have identical names, regardless of version number +# - URLs will have $repo replaced by the name of the current repo +# - URLs will have $arch replaced by the name of the architecture +# +# Repository entries are of the format: +# [repo-name] +# Server = ServerName +# Include = IncludePath +# +# The header [repo-name] is crucial - it must be present and +# uncommented to enable the repo. +# + +# The testing repositories are disabled by default. To enable, uncomment the +# repo name header and Include lines. You can add preferred servers immediately +# after the header, and they will be used before the default mirrors. + +[gnome-unstable] +Include = /etc/pacman.d/mirrorlist + +[kde-unstable] +Include = /etc/pacman.d/mirrorlist + +[core-staging] +Include = /etc/pacman.d/mirrorlist + +[core-testing] +Include = /etc/pacman.d/mirrorlist + +[core] +Include = /etc/pacman.d/mirrorlist + +[extra-staging] +Include = /etc/pacman.d/mirrorlist + +[extra-testing] +Include = /etc/pacman.d/mirrorlist + +[extra] +Include = /etc/pacman.d/mirrorlist + +# If you want to run 32 bit applications on your x86_64 system, +# enable the multilib repositories as required here. +[multilib-staging] +Include = /etc/pacman.d/mirrorlist + +[multilib-testing] +Include = /etc/pacman.d/mirrorlist + +[multilib] +Include = /etc/pacman.d/mirrorlist + +# An example of a custom package repository. See the pacman manpage for +# tips on creating your own repositories. +#[custom] +#SigLevel = Optional TrustAll +#Server = file:///home/custompkgs diff --git a/config/pacman/unstable.conf b/config/pacman/unstable.conf new file mode 100644 index 0000000..408d0ce --- /dev/null +++ b/config/pacman/unstable.conf @@ -0,0 +1,83 @@ +# +# /etc/pacman.conf +# +# See the pacman.conf(5) manpage for option and repository directives + +# +# GENERAL OPTIONS +# +[options] +# The following paths are commented out with their default values listed. +# If you wish to use different paths, uncomment and update the paths. +#RootDir = / +#DBPath = /var/lib/pacman/ +#CacheDir = /var/cache/pacman/pkg/ +#LogFile = /var/log/pacman.log +#GPGDir = /etc/pacman.d/gnupg/ +#HookDir = /etc/pacman.d/hooks/ +HoldPkg = pacman glibc +#XferCommand = /usr/bin/curl -L -C - -f -o %o %u +#XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u +#CleanMethod = KeepInstalled +Architecture = auto + +# Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup +#IgnorePkg = +#IgnoreGroup = + +#NoUpgrade = +#NoExtract = + +# Misc options +#UseSyslog +#Color +NoProgressBar +# We cannot check disk space from within a chroot environment +#CheckSpace +VerbosePkgLists +ParallelDownloads = 5 + +# By default, pacman accepts packages signed by keys that its local keyring +# trusts (see pacman-key and its man page), as well as unsigned packages. +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional +#RemoteFileSigLevel = Required + +# NOTE: You must run `pacman-key --init` before first using pacman; the local +# keyring can then be populated with the keys of all official Arch Linux +# packagers with `pacman-key --populate archlinux`. + +# +# REPOSITORIES +# - can be defined here or included from another file +# - pacman will search repositories in the order defined here +# - local/custom mirrors can be added here or in separate files +# - repositories listed first will take precedence when packages +# have identical names, regardless of version number +# - URLs will have $repo replaced by the name of the current repo +# - URLs will have $arch replaced by the name of the architecture +# +# Repository entries are of the format: +# [repo-name] +# Server = ServerName +# Include = IncludePath +# +# The header [repo-name] is crucial - it must be present and +# uncommented to enable the repo. +# + +# The testing repositories are disabled by default. To enable, uncomment the +# repo name header and Include lines. You can add preferred servers immediately +# after the header, and they will be used before the default mirrors. + +[gnome-unstable] +Include = /etc/pacman.d/mirrorlist + +[kde-unstable] +Include = /etc/pacman.d/mirrorlist + +# An example of a custom package repository. See the pacman manpage for +# tips on creating your own repositories. +#[custom] +#SigLevel = Optional TrustAll +#Server = file:///home/custompkgs diff --git a/contrib/completion/bash/devtools.in b/contrib/completion/bash/devtools.in index ec45b62..5125ceb 100644 --- a/contrib/completion/bash/devtools.in +++ b/contrib/completion/bash/devtools.in @@ -241,6 +241,7 @@ _pkgctl_db_move_opts() { _pkgctl_db_remove_args=( --partial + --noconfirm -a --arch -h --help ) diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in index 6dc0340..48ff373 100644 --- a/contrib/completion/zsh/_devtools.in +++ b/contrib/completion/zsh/_devtools.in @@ -79,6 +79,7 @@ _pkgctl_db_move_args=( _pkgctl_db_remove_args=( '--partial[Remove only partial pkgnames from a split package]' + '--noconfirm[Bypass any confirmation messages, should only be used with caution]' '(-a --arch)'{-a,--arch}"[Override the architecture (disables auto-detection)]:arch:($DEVTOOLS_VALID_BINARY_ARCHES[*])" '(-h --help)'{-h,--help}'[Display usage]' "1:repo:($DEVTOOLS_VALID_REPOS[*])" diff --git a/doc/man/pkgctl-db-remove.1.asciidoc b/doc/man/pkgctl-db-remove.1.asciidoc index 9fe07c3..85f616b 100644 --- a/doc/man/pkgctl-db-remove.1.asciidoc +++ b/doc/man/pkgctl-db-remove.1.asciidoc @@ -31,6 +31,9 @@ Options Remove only one specific architecture (disables auto-detection). By default all architectures are removed when this option is not used. +*--noconfirm*:: + Bypass any confirmation messages, should only be used with caution. + *-h, --help*:: Show a help text diff --git a/src/lib/build/build.sh b/src/lib/build/build.sh index c3e05be..b82011b 100644 --- a/src/lib/build/build.sh +++ b/src/lib/build/build.sh @@ -312,7 +312,7 @@ pkgctl_build() { # Update pacman cache for auto-detection if [[ -z ${REPO} ]]; then - update_pacman_repo_cache + update_pacman_repo_cache multilib # Check valid repos if not resolved dynamically elif ! in_array "${REPO}" "${DEVTOOLS_VALID_REPOS[@]}"; then die "Invalid repository target: %s" "${REPO}" diff --git a/src/lib/db/remove.sh b/src/lib/db/remove.sh index 018b793..6ca091d 100644 --- a/src/lib/db/remove.sh +++ b/src/lib/db/remove.sh @@ -8,6 +8,10 @@ DEVTOOLS_INCLUDE_DB_REMOVE_SH=1 _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} # shellcheck source=src/lib/common.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/util/pacman.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/pacman.sh +# shellcheck source=src/lib/util/term.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/term.sh set -e @@ -29,6 +33,7 @@ pkgctl_db_remove_usage() { -a, --arch Remove only one specific architecture (disables auto-detection) --partial Remove only partial pkgnames from a split package. This leaves debug packages behind and pkgbase entries in the state repo. + --noconfirm Bypass any confirmation messages, should only be used with caution -h, --help Show this help text EXAMPLES @@ -40,8 +45,12 @@ _EOF_ pkgctl_db_remove() { local REPO="" local PKGBASES=() + local pkgnames=() local partial=0 + local confirm=1 local dbscripts_options=() + local lookup_repo=multilib + local pkgname # option checking while (( $# )); do @@ -60,6 +69,10 @@ pkgctl_db_remove() { dbscripts_options+=(--arch "$2") shift 2 ;; + --noconfirm) + confirm=0 + shift + ;; -*) die "invalid argument: %s" "$1" ;; @@ -77,6 +90,40 @@ pkgctl_db_remove() { REPO=$1 shift PKGBASES+=("$@") + pkgnames=("${PKGBASES[@]}") + + # update pacman cache to query all pkgnames + if (( ! partial )); then + case ${REPO} in + *-unstable) + update_pacman_repo_cache unstable + ;; + *-staging) + update_pacman_repo_cache multilib-staging + ;; + *-testing) + update_pacman_repo_cache multilib-testing + ;; + *) + update_pacman_repo_cache multilib + ;; + esac + + # fetch the pkgnames of all pkgbase as present in the repo + mapfile -t pkgnames < <(get_pkgnames_from_repo_pkgbase "${REPO}" "${PKGBASES[@]}") + echo + + if (( ! ${#pkgnames[@]} )); then + error "Packages not found in %s" "${REPO}" + exit 1 + fi + fi + + # print list of packages + printf "%sRemoving packages from %s:%s\n" "${RED}" "${REPO}" "${ALL_OFF}" + for pkgname in "${pkgnames[@]}"; do + printf "• %s\n" "${pkgname}" + done # print explenation about partial removal if (( partial )); then @@ -85,7 +132,15 @@ pkgctl_db_remove() { msg_warn "${YELLOW}This leaves debug packages and pkgbase entries in the state repo!${ALL_OFF}" fi - # shellcheck disable=SC2029 + # ask for confirmation + if (( confirm )); then + echo + if ! prompt "${GREEN}${BOLD}?${ALL_OFF} Are you sure this is correct?"; then + exit 1 + fi + fi + echo + # shellcheck disable=SC2029 ssh "${PACKAGING_REPO_RELEASE_HOST}" db-remove "${dbscripts_options[@]}" "${REPO}" "${PKGBASES[@]}" } diff --git a/src/lib/release.sh b/src/lib/release.sh index acb3b54..ba21384 100644 --- a/src/lib/release.sh +++ b/src/lib/release.sh @@ -124,7 +124,7 @@ pkgctl_release() { # Update pacman cache for auto-detection if [[ -z ${REPO} ]]; then - update_pacman_repo_cache + update_pacman_repo_cache multilib # Check valid repos if not resolved dynamically elif ! in_array "${REPO}" "${DEVTOOLS_VALID_REPOS[@]}"; then die "Invalid repository target: %s" "${REPO}" diff --git a/src/lib/util/pacman.sh b/src/lib/util/pacman.sh index 620e1a8..4637d28 100644 --- a/src/lib/util/pacman.sh +++ b/src/lib/util/pacman.sh @@ -18,10 +18,12 @@ readonly _DEVTOOLS_MAKEPKG_CONF_DIR=${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d update_pacman_repo_cache() { + local repo=${1:-multilib} + mkdir -p "${_DEVTOOLS_PACMAN_CACHE_DIR}" msg "Updating pacman database cache" lock 10 "${_DEVTOOLS_PACMAN_CACHE_DIR}.lock" "Locking pacman database cache" - fakeroot -- pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/multilib.conf" \ + fakeroot -- pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/${repo}.conf" \ --dbpath "${_DEVTOOLS_PACMAN_CACHE_DIR}" \ -Sy lock_close 10 @@ -29,6 +31,7 @@ update_pacman_repo_cache() { get_pacman_repo_from_pkgbuild() { local path=${1:-PKGBUILD} + local repo=${2:-multilib} # shellcheck source=contrib/makepkg/PKGBUILD.proto mapfile -t pkgnames < <(source "${path}"; printf "%s\n" "${pkgname[@]}") @@ -40,12 +43,12 @@ get_pacman_repo_from_pkgbuild() { # update the pacman repo cache if it doesn't exist yet if [[ ! -d "${_DEVTOOLS_PACMAN_CACHE_DIR}" ]]; then - update_pacman_repo_cache + update_pacman_repo_cache "${repo}" fi slock 10 "${_DEVTOOLS_PACMAN_CACHE_DIR}.lock" "Locking pacman database cache" # query repo of passed pkgname, specify --nodeps twice to skip all dependency checks - mapfile -t repos < <(pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/multilib.conf" \ + mapfile -t repos < <(pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/${repo}.conf" \ --dbpath "${_DEVTOOLS_PACMAN_CACHE_DIR}" \ --sync \ --nodeps \ @@ -58,3 +61,30 @@ get_pacman_repo_from_pkgbuild() { printf "%s" "${repos[0]}" } + +get_pkgnames_from_repo_pkgbase() { + local repo=$1 + shift + local pkgbases=("$@") + + # update the pacman repo cache if it doesn't exist yet + if [[ ! -d "${_DEVTOOLS_PACMAN_CACHE_DIR}" ]]; then + update_pacman_repo_cache universe + fi + + slock 10 "${_DEVTOOLS_PACMAN_CACHE_DIR}.lock" "Locking pacman database cache" + # query pkgnames of passed pkgbase inside a repo + mapfile -t pkgnames < <(expac --config <(sed "s|#DBPath.*|DBPath = $(realpath "${_DEVTOOLS_PACMAN_CACHE_DIR}")|" < "${_DEVTOOLS_PACMAN_CONF_DIR}/universe.conf") \ + --sync '%r %e %n' 2>/dev/null \ + | sort | awk -v pkgbase="${pkgbases[*]}" \ + 'BEGIN { split(pkgbase, array); for (item in array) filter[array[item]]=1 } $1=="'"${repo}"'" && $2 in filter {print $3}' + ) + lock_close 10 + + if (( ! ${#pkgnames[@]} )); then + return 1 + fi + + printf "%s\n" "${pkgnames[@]}" + return 0 +} diff --git a/src/lib/util/term.sh b/src/lib/util/term.sh index 853dccf..08d044f 100644 --- a/src/lib/util/term.sh +++ b/src/lib/util/term.sh @@ -180,3 +180,19 @@ term_spinner_stop() { # show the cursor after stopping the spinner term_cursor_show } + +prompt() { + local message=$1 + local answer + + read -r -p "${message} (y/N) " answer + + case "${answer}" in + y|Y|yes|Yes|YES) + true + ;; + *) + false + ;; + esac +} -- cgit v1.2.3-70-g09d2