From 7b553afcb25286d04dcb4cbf12e18745e8b0139a Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sat, 27 Apr 2024 00:40:57 +0200 Subject: feat(db): add partial split package option to db remove By default passing a pkgbase removes all split packages, debug packages as well as entries from the state repo for all existing architectures. When using the `--partial` option it may most likely lead to undesired effects by leaving debug packages behind as well as dangling entries in the state repository. However, for specific use cases its required to get rid of old split package parts. Fixes #218 Component: pkgctl db remove Signed-off-by: Levente Polyak --- src/lib/db/remove.sh | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) (limited to 'src/lib/db/remove.sh') diff --git a/src/lib/db/remove.sh b/src/lib/db/remove.sh index ba21c83..018b793 100644 --- a/src/lib/db/remove.sh +++ b/src/lib/db/remove.sh @@ -17,10 +17,18 @@ pkgctl_db_remove_usage() { cat <<- _EOF_ Usage: ${COMMAND} [OPTIONS] [REPO] [PKGBASE]... - Remove packages from binary repositories. + Remove packages from pacman repositories. By default passing a pkgbase removes + all split packages, debug packages as well as entries from the state repo for + all existing architectures. + + Beware when using the --partial option, as it may most likely lead to + undesired effects by leaving debug packages behind as well as dangling entries + in the state repository. OPTIONS - -a, --arch Override the architecture (disables auto-detection) + -a, --arch Remove only one specific architecture (disables auto-detection) + --partial Remove only partial pkgnames from a split package. This leaves + debug packages behind and pkgbase entries in the state repo. -h, --help Show this help text EXAMPLES @@ -31,8 +39,9 @@ _EOF_ pkgctl_db_remove() { local REPO="" - local ARCH=any local PKGBASES=() + local partial=0 + local dbscripts_options=() # option checking while (( $# )); do @@ -41,9 +50,14 @@ pkgctl_db_remove() { pkgctl_db_remove_usage exit 0 ;; + --partial) + partial=1 + dbscripts_options+=(--partial) + shift + ;; -a|--arch) (( $# <= 1 )) && die "missing argument for %s" "$1" - ARCH=$2 + dbscripts_options+=(--arch "$2") shift 2 ;; -*) @@ -64,6 +78,14 @@ pkgctl_db_remove() { shift PKGBASES+=("$@") + # print explenation about partial removal + if (( partial )); then + echo + msg_warn "${YELLOW}Removing only partial pkgnames from a split package.${ALL_OFF}" + msg_warn "${YELLOW}This leaves debug packages and pkgbase entries in the state repo!${ALL_OFF}" + fi + # shellcheck disable=SC2029 - ssh "${PACKAGING_REPO_RELEASE_HOST}" db-remove "${REPO}" "${ARCH}" "${PKGBASES[@]}" + echo + ssh "${PACKAGING_REPO_RELEASE_HOST}" db-remove "${dbscripts_options[@]}" "${REPO}" "${PKGBASES[@]}" } -- cgit v1.2.3-70-g09d2 From 1d433f600e6eecfe685650a06e58d1a8edae9b5d Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sat, 27 Apr 2024 01:50:57 +0200 Subject: feat(db): confirm list of all packages that will be removed Sometimes it isn't obvious which set of packages are removed from a split package when the pkgbase matches also a subset of a pkgbase. This can happen for example with bootstrapping packages, when the intention is to just remove a partial part of the bootstrap pkgbase. To make the intention more explicit, list all to be removed packages and await for confirmation. Component: pkgctl db remove Signed-off-by: Levente Polyak --- README.md | 1 + config/pacman/universe.conf | 112 ++++++++++++++++++++++++++++++++++++ config/pacman/unstable.conf | 83 ++++++++++++++++++++++++++ contrib/completion/bash/devtools.in | 1 + contrib/completion/zsh/_devtools.in | 1 + doc/man/pkgctl-db-remove.1.asciidoc | 3 + src/lib/build/build.sh | 2 +- src/lib/db/remove.sh | 57 +++++++++++++++++- src/lib/release.sh | 2 +- src/lib/util/pacman.sh | 36 +++++++++++- src/lib/util/term.sh | 16 ++++++ 11 files changed, 308 insertions(+), 6 deletions(-) create mode 100644 config/pacman/universe.conf create mode 100644 config/pacman/unstable.conf (limited to 'src/lib/db/remove.sh') diff --git a/README.md b/README.md index eea9c0a..c0bcfdd 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Component: pkgctl db remove - coreutils - curl - diffutils +- expac - fakeroot - findutils - grep diff --git a/config/pacman/universe.conf b/config/pacman/universe.conf new file mode 100644 index 0000000..417114b --- /dev/null +++ b/config/pacman/universe.conf @@ -0,0 +1,112 @@ +# +# /etc/pacman.conf +# +# See the pacman.conf(5) manpage for option and repository directives + +# +# GENERAL OPTIONS +# +[options] +# The following paths are commented out with their default values listed. +# If you wish to use different paths, uncomment and update the paths. +#RootDir = / +#DBPath = /var/lib/pacman/ +#CacheDir = /var/cache/pacman/pkg/ +#LogFile = /var/log/pacman.log +#GPGDir = /etc/pacman.d/gnupg/ +#HookDir = /etc/pacman.d/hooks/ +HoldPkg = pacman glibc +#XferCommand = /usr/bin/curl -L -C - -f -o %o %u +#XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u +#CleanMethod = KeepInstalled +Architecture = auto + +# Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup +#IgnorePkg = +#IgnoreGroup = + +#NoUpgrade = +#NoExtract = + +# Misc options +#UseSyslog +#Color +NoProgressBar +# We cannot check disk space from within a chroot environment +#CheckSpace +VerbosePkgLists +ParallelDownloads = 5 + +# By default, pacman accepts packages signed by keys that its local keyring +# trusts (see pacman-key and its man page), as well as unsigned packages. +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional +#RemoteFileSigLevel = Required + +# NOTE: You must run `pacman-key --init` before first using pacman; the local +# keyring can then be populated with the keys of all official Arch Linux +# packagers with `pacman-key --populate archlinux`. + +# +# REPOSITORIES +# - can be defined here or included from another file +# - pacman will search repositories in the order defined here +# - local/custom mirrors can be added here or in separate files +# - repositories listed first will take precedence when packages +# have identical names, regardless of version number +# - URLs will have $repo replaced by the name of the current repo +# - URLs will have $arch replaced by the name of the architecture +# +# Repository entries are of the format: +# [repo-name] +# Server = ServerName +# Include = IncludePath +# +# The header [repo-name] is crucial - it must be present and +# uncommented to enable the repo. +# + +# The testing repositories are disabled by default. To enable, uncomment the +# repo name header and Include lines. You can add preferred servers immediately +# after the header, and they will be used before the default mirrors. + +[gnome-unstable] +Include = /etc/pacman.d/mirrorlist + +[kde-unstable] +Include = /etc/pacman.d/mirrorlist + +[core-staging] +Include = /etc/pacman.d/mirrorlist + +[core-testing] +Include = /etc/pacman.d/mirrorlist + +[core] +Include = /etc/pacman.d/mirrorlist + +[extra-staging] +Include = /etc/pacman.d/mirrorlist + +[extra-testing] +Include = /etc/pacman.d/mirrorlist + +[extra] +Include = /etc/pacman.d/mirrorlist + +# If you want to run 32 bit applications on your x86_64 system, +# enable the multilib repositories as required here. +[multilib-staging] +Include = /etc/pacman.d/mirrorlist + +[multilib-testing] +Include = /etc/pacman.d/mirrorlist + +[multilib] +Include = /etc/pacman.d/mirrorlist + +# An example of a custom package repository. See the pacman manpage for +# tips on creating your own repositories. +#[custom] +#SigLevel = Optional TrustAll +#Server = file:///home/custompkgs diff --git a/config/pacman/unstable.conf b/config/pacman/unstable.conf new file mode 100644 index 0000000..408d0ce --- /dev/null +++ b/config/pacman/unstable.conf @@ -0,0 +1,83 @@ +# +# /etc/pacman.conf +# +# See the pacman.conf(5) manpage for option and repository directives + +# +# GENERAL OPTIONS +# +[options] +# The following paths are commented out with their default values listed. +# If you wish to use different paths, uncomment and update the paths. +#RootDir = / +#DBPath = /var/lib/pacman/ +#CacheDir = /var/cache/pacman/pkg/ +#LogFile = /var/log/pacman.log +#GPGDir = /etc/pacman.d/gnupg/ +#HookDir = /etc/pacman.d/hooks/ +HoldPkg = pacman glibc +#XferCommand = /usr/bin/curl -L -C - -f -o %o %u +#XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u +#CleanMethod = KeepInstalled +Architecture = auto + +# Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup +#IgnorePkg = +#IgnoreGroup = + +#NoUpgrade = +#NoExtract = + +# Misc options +#UseSyslog +#Color +NoProgressBar +# We cannot check disk space from within a chroot environment +#CheckSpace +VerbosePkgLists +ParallelDownloads = 5 + +# By default, pacman accepts packages signed by keys that its local keyring +# trusts (see pacman-key and its man page), as well as unsigned packages. +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional +#RemoteFileSigLevel = Required + +# NOTE: You must run `pacman-key --init` before first using pacman; the local +# keyring can then be populated with the keys of all official Arch Linux +# packagers with `pacman-key --populate archlinux`. + +# +# REPOSITORIES +# - can be defined here or included from another file +# - pacman will search repositories in the order defined here +# - local/custom mirrors can be added here or in separate files +# - repositories listed first will take precedence when packages +# have identical names, regardless of version number +# - URLs will have $repo replaced by the name of the current repo +# - URLs will have $arch replaced by the name of the architecture +# +# Repository entries are of the format: +# [repo-name] +# Server = ServerName +# Include = IncludePath +# +# The header [repo-name] is crucial - it must be present and +# uncommented to enable the repo. +# + +# The testing repositories are disabled by default. To enable, uncomment the +# repo name header and Include lines. You can add preferred servers immediately +# after the header, and they will be used before the default mirrors. + +[gnome-unstable] +Include = /etc/pacman.d/mirrorlist + +[kde-unstable] +Include = /etc/pacman.d/mirrorlist + +# An example of a custom package repository. See the pacman manpage for +# tips on creating your own repositories. +#[custom] +#SigLevel = Optional TrustAll +#Server = file:///home/custompkgs diff --git a/contrib/completion/bash/devtools.in b/contrib/completion/bash/devtools.in index ec45b62..5125ceb 100644 --- a/contrib/completion/bash/devtools.in +++ b/contrib/completion/bash/devtools.in @@ -241,6 +241,7 @@ _pkgctl_db_move_opts() { _pkgctl_db_remove_args=( --partial + --noconfirm -a --arch -h --help ) diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in index 6dc0340..48ff373 100644 --- a/contrib/completion/zsh/_devtools.in +++ b/contrib/completion/zsh/_devtools.in @@ -79,6 +79,7 @@ _pkgctl_db_move_args=( _pkgctl_db_remove_args=( '--partial[Remove only partial pkgnames from a split package]' + '--noconfirm[Bypass any confirmation messages, should only be used with caution]' '(-a --arch)'{-a,--arch}"[Override the architecture (disables auto-detection)]:arch:($DEVTOOLS_VALID_BINARY_ARCHES[*])" '(-h --help)'{-h,--help}'[Display usage]' "1:repo:($DEVTOOLS_VALID_REPOS[*])" diff --git a/doc/man/pkgctl-db-remove.1.asciidoc b/doc/man/pkgctl-db-remove.1.asciidoc index 9fe07c3..85f616b 100644 --- a/doc/man/pkgctl-db-remove.1.asciidoc +++ b/doc/man/pkgctl-db-remove.1.asciidoc @@ -31,6 +31,9 @@ Options Remove only one specific architecture (disables auto-detection). By default all architectures are removed when this option is not used. +*--noconfirm*:: + Bypass any confirmation messages, should only be used with caution. + *-h, --help*:: Show a help text diff --git a/src/lib/build/build.sh b/src/lib/build/build.sh index c3e05be..b82011b 100644 --- a/src/lib/build/build.sh +++ b/src/lib/build/build.sh @@ -312,7 +312,7 @@ pkgctl_build() { # Update pacman cache for auto-detection if [[ -z ${REPO} ]]; then - update_pacman_repo_cache + update_pacman_repo_cache multilib # Check valid repos if not resolved dynamically elif ! in_array "${REPO}" "${DEVTOOLS_VALID_REPOS[@]}"; then die "Invalid repository target: %s" "${REPO}" diff --git a/src/lib/db/remove.sh b/src/lib/db/remove.sh index 018b793..6ca091d 100644 --- a/src/lib/db/remove.sh +++ b/src/lib/db/remove.sh @@ -8,6 +8,10 @@ DEVTOOLS_INCLUDE_DB_REMOVE_SH=1 _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} # shellcheck source=src/lib/common.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/util/pacman.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/pacman.sh +# shellcheck source=src/lib/util/term.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/term.sh set -e @@ -29,6 +33,7 @@ pkgctl_db_remove_usage() { -a, --arch Remove only one specific architecture (disables auto-detection) --partial Remove only partial pkgnames from a split package. This leaves debug packages behind and pkgbase entries in the state repo. + --noconfirm Bypass any confirmation messages, should only be used with caution -h, --help Show this help text EXAMPLES @@ -40,8 +45,12 @@ _EOF_ pkgctl_db_remove() { local REPO="" local PKGBASES=() + local pkgnames=() local partial=0 + local confirm=1 local dbscripts_options=() + local lookup_repo=multilib + local pkgname # option checking while (( $# )); do @@ -60,6 +69,10 @@ pkgctl_db_remove() { dbscripts_options+=(--arch "$2") shift 2 ;; + --noconfirm) + confirm=0 + shift + ;; -*) die "invalid argument: %s" "$1" ;; @@ -77,6 +90,40 @@ pkgctl_db_remove() { REPO=$1 shift PKGBASES+=("$@") + pkgnames=("${PKGBASES[@]}") + + # update pacman cache to query all pkgnames + if (( ! partial )); then + case ${REPO} in + *-unstable) + update_pacman_repo_cache unstable + ;; + *-staging) + update_pacman_repo_cache multilib-staging + ;; + *-testing) + update_pacman_repo_cache multilib-testing + ;; + *) + update_pacman_repo_cache multilib + ;; + esac + + # fetch the pkgnames of all pkgbase as present in the repo + mapfile -t pkgnames < <(get_pkgnames_from_repo_pkgbase "${REPO}" "${PKGBASES[@]}") + echo + + if (( ! ${#pkgnames[@]} )); then + error "Packages not found in %s" "${REPO}" + exit 1 + fi + fi + + # print list of packages + printf "%sRemoving packages from %s:%s\n" "${RED}" "${REPO}" "${ALL_OFF}" + for pkgname in "${pkgnames[@]}"; do + printf "• %s\n" "${pkgname}" + done # print explenation about partial removal if (( partial )); then @@ -85,7 +132,15 @@ pkgctl_db_remove() { msg_warn "${YELLOW}This leaves debug packages and pkgbase entries in the state repo!${ALL_OFF}" fi - # shellcheck disable=SC2029 + # ask for confirmation + if (( confirm )); then + echo + if ! prompt "${GREEN}${BOLD}?${ALL_OFF} Are you sure this is correct?"; then + exit 1 + fi + fi + echo + # shellcheck disable=SC2029 ssh "${PACKAGING_REPO_RELEASE_HOST}" db-remove "${dbscripts_options[@]}" "${REPO}" "${PKGBASES[@]}" } diff --git a/src/lib/release.sh b/src/lib/release.sh index acb3b54..ba21384 100644 --- a/src/lib/release.sh +++ b/src/lib/release.sh @@ -124,7 +124,7 @@ pkgctl_release() { # Update pacman cache for auto-detection if [[ -z ${REPO} ]]; then - update_pacman_repo_cache + update_pacman_repo_cache multilib # Check valid repos if not resolved dynamically elif ! in_array "${REPO}" "${DEVTOOLS_VALID_REPOS[@]}"; then die "Invalid repository target: %s" "${REPO}" diff --git a/src/lib/util/pacman.sh b/src/lib/util/pacman.sh index 620e1a8..4637d28 100644 --- a/src/lib/util/pacman.sh +++ b/src/lib/util/pacman.sh @@ -18,10 +18,12 @@ readonly _DEVTOOLS_MAKEPKG_CONF_DIR=${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d update_pacman_repo_cache() { + local repo=${1:-multilib} + mkdir -p "${_DEVTOOLS_PACMAN_CACHE_DIR}" msg "Updating pacman database cache" lock 10 "${_DEVTOOLS_PACMAN_CACHE_DIR}.lock" "Locking pacman database cache" - fakeroot -- pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/multilib.conf" \ + fakeroot -- pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/${repo}.conf" \ --dbpath "${_DEVTOOLS_PACMAN_CACHE_DIR}" \ -Sy lock_close 10 @@ -29,6 +31,7 @@ update_pacman_repo_cache() { get_pacman_repo_from_pkgbuild() { local path=${1:-PKGBUILD} + local repo=${2:-multilib} # shellcheck source=contrib/makepkg/PKGBUILD.proto mapfile -t pkgnames < <(source "${path}"; printf "%s\n" "${pkgname[@]}") @@ -40,12 +43,12 @@ get_pacman_repo_from_pkgbuild() { # update the pacman repo cache if it doesn't exist yet if [[ ! -d "${_DEVTOOLS_PACMAN_CACHE_DIR}" ]]; then - update_pacman_repo_cache + update_pacman_repo_cache "${repo}" fi slock 10 "${_DEVTOOLS_PACMAN_CACHE_DIR}.lock" "Locking pacman database cache" # query repo of passed pkgname, specify --nodeps twice to skip all dependency checks - mapfile -t repos < <(pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/multilib.conf" \ + mapfile -t repos < <(pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/${repo}.conf" \ --dbpath "${_DEVTOOLS_PACMAN_CACHE_DIR}" \ --sync \ --nodeps \ @@ -58,3 +61,30 @@ get_pacman_repo_from_pkgbuild() { printf "%s" "${repos[0]}" } + +get_pkgnames_from_repo_pkgbase() { + local repo=$1 + shift + local pkgbases=("$@") + + # update the pacman repo cache if it doesn't exist yet + if [[ ! -d "${_DEVTOOLS_PACMAN_CACHE_DIR}" ]]; then + update_pacman_repo_cache universe + fi + + slock 10 "${_DEVTOOLS_PACMAN_CACHE_DIR}.lock" "Locking pacman database cache" + # query pkgnames of passed pkgbase inside a repo + mapfile -t pkgnames < <(expac --config <(sed "s|#DBPath.*|DBPath = $(realpath "${_DEVTOOLS_PACMAN_CACHE_DIR}")|" < "${_DEVTOOLS_PACMAN_CONF_DIR}/universe.conf") \ + --sync '%r %e %n' 2>/dev/null \ + | sort | awk -v pkgbase="${pkgbases[*]}" \ + 'BEGIN { split(pkgbase, array); for (item in array) filter[array[item]]=1 } $1=="'"${repo}"'" && $2 in filter {print $3}' + ) + lock_close 10 + + if (( ! ${#pkgnames[@]} )); then + return 1 + fi + + printf "%s\n" "${pkgnames[@]}" + return 0 +} diff --git a/src/lib/util/term.sh b/src/lib/util/term.sh index 853dccf..08d044f 100644 --- a/src/lib/util/term.sh +++ b/src/lib/util/term.sh @@ -180,3 +180,19 @@ term_spinner_stop() { # show the cursor after stopping the spinner term_cursor_show } + +prompt() { + local message=$1 + local answer + + read -r -p "${message} (y/N) " answer + + case "${answer}" in + y|Y|yes|Yes|YES) + true + ;; + *) + false + ;; + esac +} -- cgit v1.2.3-70-g09d2 From 35b417d226fc3d11ac0015abdcb3594d2c696c2e Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sat, 27 Apr 2024 02:18:05 +0200 Subject: fix(db): check valid target repo on db remove Return an error and abort operation when an invalid target repo is passed. Component: pkgctl db remove Signed-off-by: Levente Polyak --- src/lib/db/remove.sh | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/lib/db/remove.sh') diff --git a/src/lib/db/remove.sh b/src/lib/db/remove.sh index 6ca091d..cddcc1d 100644 --- a/src/lib/db/remove.sh +++ b/src/lib/db/remove.sh @@ -12,6 +12,8 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/pacman.sh # shellcheck source=src/lib/util/term.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/term.sh +# shellcheck source=src/lib/valid-repos.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-repos.sh set -e @@ -92,6 +94,11 @@ pkgctl_db_remove() { PKGBASES+=("$@") pkgnames=("${PKGBASES[@]}") + # check if the target repo is valid + if ! in_array "${REPO}" "${DEVTOOLS_VALID_REPOS[@]}"; then + die "Invalid repository target: %s" "${REPO}" + fi + # update pacman cache to query all pkgnames if (( ! partial )); then case ${REPO} in -- cgit v1.2.3-70-g09d2