| index : devtools32 | |
| Archlinux32 fork of devtools | gitolite user |
| summaryrefslogtreecommitdiff |
diff --git a/src/arch-nspawn.in b/src/arch-nspawn.in index 6fbed18..8fcdead 100644 --- a/src/arch-nspawn.in +++ b/src/arch-nspawn.in @@ -16,7 +16,6 @@ umask 0022 working_dir='' files=() -mount_args=() usage() { echo "Usage: ${0##*/} [options] working-dir [systemd-nspawn arguments]" @@ -56,6 +55,16 @@ shift 1 [[ -z $working_dir ]] && die 'Please specify a working directory.' +nspawn_args=( + --quiet + --directory="$working_dir" + --setenv="PATH=/usr/local/sbin:/usr/local/bin:/usr/bin" + --register=no + --slice="devtools-$(systemd-escape "${SUDO_USER:-$USER}")" + --machine="arch-nspawn-$$" + --as-pid2 +) + if (( ${#cache_dirs[@]} == 0 )); then mapfile -t cache_dirs < <(pacman-conf --config "${pac_conf:-$working_dir/etc/pacman.conf}" CacheDir) fi @@ -83,10 +92,10 @@ while read -r line; do done done < <(pacman-conf --config "${pac_conf:-$working_dir/etc/pacman.conf}" --repo-list) -mount_args+=("--bind=${cache_dirs[0]//:/\\:}") +nspawn_args+=(--bind="${cache_dirs[0]//:/\\:}") for cache_dir in "${cache_dirs[@]:1}"; do - mount_args+=("--bind-ro=${cache_dir//:/\\:}") + nspawn_args+=(--bind-ro="${cache_dir//:/\\:}") done # {{{ functions @@ -131,9 +140,4 @@ else set_arch="${CARCH}" fi -exec ${CARCH:+setarch "$set_arch"} systemd-nspawn -q \ - -D "$working_dir" \ - -E "PATH=/usr/local/sbin:/usr/local/bin:/usr/bin" \ - --register=no --keep-unit --as-pid2 \ - "${mount_args[@]}" \ - "$@" +exec ${CARCH:+setarch "$set_arch"} systemd-nspawn "${nspawn_args[@]}" "$@" diff --git a/src/archrelease.in b/src/archrelease.in index 818b0ca..84aed28 100644 --- a/src/archrelease.in +++ b/src/archrelease.in @@ -35,7 +35,7 @@ fi # validate repo is really repo-arch if [[ -z $FORCE ]]; then for tag in "$@"; do - if ! in_array "$tag" "${_tags[@]}"; then + if ! in_array "$tag" "${DEVTOOLS_VALID_TAGS[@]}"; then die "archrelease: Invalid tag: '%s' (use -f to force release)" "$tag" fi done diff --git a/src/commitpkg.in b/src/commitpkg.in index 8a8087a..e17b270 100644 --- a/src/commitpkg.in +++ b/src/commitpkg.in @@ -5,9 +5,13 @@ _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} # shellcheck source=src/lib/common.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/util/srcinfo.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/srcinfo.sh source /usr/share/makepkg/util/util.sh +set -eo pipefail + check_pkgbuild_validity() { # shellcheck source=contrib/makepkg/PKGBUILD.proto @@ -70,6 +74,12 @@ if ! repo_spec=$(git config --local devtools.version) || [[ ${repo_spec} != "${G exit 1 fi +if ! repo_variant=$(git config --local devtools.variant) || [[ ${repo_variant} != canonical ]]; then + error "cannot release from a repository with none canonical specs (%s), try:" "${repo_variant:-development}" + msg2 'pkgctl repo configure' + exit 1 +fi + if [[ "$(git symbolic-ref --short HEAD)" != main ]]; then die 'must be run from the main branch' fi @@ -111,7 +121,7 @@ if (( ${#validpgpkeys[@]} != 0 )); then fi # find files which should be under source control -needsversioning=() +needsversioning=(PKGBUILD) for s in "${source[@]}"; do [[ $s != *://* ]] && needsversioning+=("$s") done @@ -177,13 +187,10 @@ done # check for PKGBUILD standards check_pkgbuild_validity -# auto generate .SRCINFO if present -if [[ -f .SRCINFO ]]; then - stat_busy 'Generating .SRCINFO' - makepkg --printsrcinfo > .SRCINFO - git add .SRCINFO - stat_done -fi +# auto generate .SRCINFO +# shellcheck disable=SC2119 +write_srcinfo_file +git add --force .SRCINFO if [[ -n $(git status --porcelain --untracked-files=no) ]]; then stat_busy 'Staging files' @@ -206,14 +213,14 @@ if [[ -n $(git status --porcelain --untracked-files=no) ]]; then echo "$msgtemplate" > "$msgfile" if [[ -n $GIT_EDITOR ]]; then $GIT_EDITOR "$msgfile" || die + elif giteditor=$(git config --get core.editor); then + $giteditor "$msgfile" || die elif [[ -n $VISUAL ]]; then $VISUAL "$msgfile" || die elif [[ -n $EDITOR ]]; then $EDITOR "$msgfile" || die - elif giteditor=$(git config --get core.editor); then - $giteditor "$msgfile" || die else - die "No usable editor found (tried \$GIT_EDITOR, \$VISUAL, \$EDITOR, git config [core.editor])." + die "No usable editor found (tried \$GIT_EDITOR, git config [core.editor], \$VISUAL, \$EDITOR)." fi [[ -s $msgfile ]] || die stat_busy 'Committing changes' diff --git a/src/lib/api/archweb.sh b/src/lib/api/archweb.sh new file mode 100644 index 0000000..34c7a9c --- /dev/null +++ b/src/lib/api/archweb.sh @@ -0,0 +1,57 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_API_ARCHWEB_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_API_ARCHWEB_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh + +set -e +set -o pipefail + + +archweb_query_all_packages() { + [[ -z ${WORKDIR:-} ]] && setup_workdir + + stat_busy "Query all released packages" + mapfile -t pkgbases < <( + curl --location --show-error --no-progress-meter --fail --retry 3 --retry-delay 3 \ + "${PKGBASE_MAINTAINER_URL}" 2> "${WORKDIR}/error" \ + | jq --raw-output --exit-status 'keys[]' 2> "${WORKDIR}/error" + ) + if ! wait $!; then + stat_failed + print_workdir_error + return 1 + fi + stat_done + + printf "%s\n" "${pkgbases[@]}" + return 0 +} + + +archweb_query_maintainer_packages() { + local maintainer=$1 + + [[ -z ${WORKDIR:-} ]] && setup_workdir + + stat_busy "Query maintainer packages" + mapfile -t pkgbases < <( + curl --location --show-error --no-progress-meter --fail --retry 3 --retry-delay 3 \ + "${PKGBASE_MAINTAINER_URL}" 2> "${WORKDIR}/error" \ + | jq --raw-output --exit-status '. as $parent | keys[] | select(. as $key | $parent[$key] | index("'"${maintainer}"'"))' 2> "${WORKDIR}/error" + ) + if ! wait $!; then + stat_failed + print_workdir_error + return 1 + fi + stat_done + + printf "%s\n" "${pkgbases[@]}" + return 0 +} diff --git a/src/lib/api/gitlab.sh b/src/lib/api/gitlab.sh index e5f4237..115e58c 100644 --- a/src/lib/api/gitlab.sh +++ b/src/lib/api/gitlab.sh @@ -13,13 +13,63 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/config.sh set -e +graphql_api_call() { + local outfile=$1 + local request=$2 + local node_type=$3 + local data=$4 + local hasNextPage cursor + + # empty token + if [[ -z "${GITLAB_TOKEN}" ]]; then + msg_error " api call failed: No token provided" + return 1 + fi + + [[ -z ${WORKDIR:-} ]] && setup_workdir + api_workdir=$(mktemp --tmpdir="${WORKDIR}" --directory pkgctl-gitlab-api.XXXXXXXXXX) + + # normalize graphql data and prepare query + data="${data//\"/\\\"}" + data='{ + "query": "'"${data}"'" + }' + data="${data//$'\t'/ }" + data="${data//$'\n'/}" + + cursor="" + hasNextPage=true + while [[ ${hasNextPage} == true ]]; do + data=$(sed -E 's|after: \\"[a-zA-Z0-9]*\\"|after: \\"'"${cursor}"'\\"|' <<< "${data}") + result="${api_workdir}/result.${cursor}" + + if ! curl --request "${request}" \ + --url "https://${GITLAB_HOST}/api/graphql" \ + --header "Authorization: Bearer ${GITLAB_TOKEN}" \ + --header "Content-Type: application/json" \ + --data "${data}" \ + --output "${result}" \ + --silent; then + msg_error " api call failed: $(cat "${outfile}")" + return 1 + fi + + hasNextPage=$(jq --raw-output ".data | .${node_type} | .pageInfo | .hasNextPage" < "${result}") + cursor=$(jq --raw-output ".data | .${node_type} | .pageInfo | .endCursor" < "${result}") + + cp "${result}" "${api_workdir}/tmp" + jq ".data.${node_type}.nodes" "${api_workdir}/tmp" > "${result}" + done + + jq --slurp add "${api_workdir}"/result.* > "${outfile}" + return 0 +} gitlab_api_call() { local outfile=$1 local request=$2 local endpoint=$3 local data=${4:-} - local error # empty token if [[ -z "${GITLAB_TOKEN}" ]]; then @@ -38,27 +88,113 @@ gitlab_api_call() { return 1 fi + if ! gitlab_check_api_errors "${outfile}"; then + return 1 + fi + + return 0 +} + +gitlab_api_call_paged() { + local outfile=$1 + local status_file=$2 + local request=$3 + local endpoint=$4 + local data=${5:-} + local result header + + # empty token + if [[ -z "${GITLAB_TOKEN}" ]]; then + msg_error " api call failed: No token provided" + return 1 + fi + + [[ -z ${WORKDIR:-} ]] && setup_workdir + api_workdir=$(mktemp --tmpdir="${WORKDIR}" --directory pkgctl-gitlab-api.XXXXXXXXXX) + tmp_file=$(mktemp --tmpdir="${api_workdir}" spinner.tmp.XXXXXXXXXX) + + local next_page=1 + local total_pages=1 + + while [[ -n "${next_page}" ]]; do + percentage=$(( 100 * next_page / total_pages )) + printf "📡 Querying GitLab: %s/%s [%s] %%spinner%%" \ + "${BOLD}${next_page}" "${total_pages}" "${percentage}%${ALL_OFF}" \ + > "${tmp_file}" + mv "${tmp_file}" "${status_file}" + + result="${api_workdir}/result.${next_page}" + header="${api_workdir}/header" + if ! curl --request "${request}" \ + --get \ + --url "https://${GITLAB_HOST}/api/v4/${endpoint}&per_page=100&page=${next_page}" \ + --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" \ + --header "Content-Type: application/json" \ + --data-urlencode "${data}" \ + --dump-header "${header}" \ + --output "${result}" \ + --silent; then + msg_error " api call failed: $(cat "${result}")" + return 1 + fi + + if ! gitlab_check_api_errors "${result}"; then + return 1 + fi + + next_page=$(grep "x-next-page" "${header}" | tr -d '\r' | awk '{ print $2 }') + total_pages=$(grep "x-total-pages" "${header}" | tr -d '\r' | awk '{ print $2 }') + done + + jq --slurp add "${api_workdir}"/result.* > "${outfile}" + return 0 +} + +gitlab_check_api_errors() { + local file=$1 + local error + + # search API only returns an array, no errors + if [[ $(jq --raw-output 'type' < "${file}") == "array" ]]; then + return 0 + fi + # check for general purpose api error - if error=$(jq --raw-output --exit-status '.error' < "${outfile}"); then + if error=$(jq --raw-output --exit-status '.error' < "${file}"); then msg_error " api call failed: ${error}" return 1 fi # check for api specific error messages - if ! jq --raw-output --exit-status '.id' < "${outfile}" >/dev/null; then - if jq --raw-output --exit-status '.message | keys[]' < "${outfile}" &>/dev/null; then + if ! jq --raw-output --exit-status '.id' < "${file}" >/dev/null; then + if jq --raw-output --exit-status '.message | keys[]' < "${file}" &>/dev/null; then while read -r error; do msg_error " api call failed: ${error}" - done < <(jq --raw-output --exit-status '.message|to_entries|map("\(.key) \(.value[])")[]' < "${outfile}") - elif error=$(jq --raw-output --exit-status '.message' < "${outfile}"); then + done < <(jq --raw-output --exit-status '.message|to_entries|map("\(.key) \(.value[])")[]' < "${file}") + elif error=$(jq --raw-output --exit-status '.message' < "${file}"); then msg_error " api call failed: ${error}" fi return 1 fi - return 0 } +graphql_check_api_errors() { + local file=$1 + local error + + # early exit if we do not have errors + if ! jq --raw-output --exit-status '.errors[]' < "${file}" &>/dev/null; then + return 0 + fi + + # check for api specific error messages + while read -r error; do + msg_error " api call failed: ${error}" + done < <(jq --raw-output --exit-status '.errors[].message' < "${file}") + return 1 +} + gitlab_api_get_user() { local outfile username @@ -81,6 +217,23 @@ gitlab_api_get_user() { return 0 } +gitlab_api_get_project_name_mapping() { + local query=$1 + local outfile + + [[ -z ${WORKDIR:-} ]] && setup_workdir + outfile=$(mktemp --tmpdir="${WORKDIR}" pkgctl-gitlab-api.XXXXXXXXXX) + + # query user details + if ! graphql_api_call "${outfile}" POST projects "${query}"; then + msg_warn " Invalid token provided?" + exit 1 + fi + + cat "${outfile}" + return 0 +} + # Convert arbitrary project names to GitLab valid path names. # # GitLab has several limitations on project and group names and also maintains @@ -130,3 +283,22 @@ gitlab_api_create_project() { printf "%s" "${path}" return 0 } + +# TODO: parallelize +# https://docs.gitlab.com/ee/api/search.html#scope-blobs +gitlab_api_search() { + local search=$1 + local status_file=$2 + local outfile + + [[ -z ${WORKDIR:-} ]] && setup_workdir + outfile=$(mktemp --tmpdir="${WORKDIR}" pkgctl-gitlab-api.XXXXXXXXXX) + + if ! gitlab_api_call_paged "${outfile}" "${status_file}" GET "/groups/archlinux%2fpackaging%2fpackages/search?scope=blobs" "search=${search}"; then + return 1 + fi + + cat "${outfile}" + + return 0 +} diff --git a/src/lib/aur.sh b/src/lib/aur.sh new file mode 100644 index 0000000..24cbb62 --- /dev/null +++ b/src/lib/aur.sh @@ -0,0 +1,65 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_AUR_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_AUR_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} + +set -eo pipefail + + +pkgctl_aur_usage() { + local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} + cat <<- _EOF_ + Usage: ${COMMAND} [COMMAND] [OPTIONS] + + Interact with the Arch User Repository (AUR). + + Provides a suite of tools designed for managing and interacting with the Arch + User Repository (AUR). It simplifies various tasks related to AUR, including + importing repositories, managing packages, and transitioning packages between + the official repositories and the AUR. + + COMMANDS + drop-from-repo Drop a package from the official repository to the AUR + + OPTIONS + -h, --help Show this help text + + EXAMPLES + $ ${COMMAND} drop-from-repo libfoo +_EOF_ +} + +pkgctl_aur() { + if (( $# < 1 )); then + pkgctl_aur_usage + exit 0 + fi + + # option checking + while (( $# )); do + case $1 in + -h|--help) + pkgctl_aur_usage + exit 0 + ;; + drop-from-repo) + _DEVTOOLS_COMMAND+=" $1" + shift + # shellcheck source=src/lib/aur/drop-from-repo.sh + source "${_DEVTOOLS_LIBRARY_DIR}"/lib/aur/drop-from-repo.sh + pkgctl_aur_drop_from_repo "$@" + exit 0 + ;; + -*) + die "invalid argument: %s" "$1" + ;; + *) + die "invalid command: %s" "$1" + ;; + esac + done +} diff --git a/src/lib/aur/drop-from-repo.sh b/src/lib/aur/drop-from-repo.sh new file mode 100644 index 0000000..6ebe12a --- /dev/null +++ b/src/lib/aur/drop-from-repo.sh @@ -0,0 +1,168 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_AUR_DROP_FROM_REPO_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_AUR_DROP_FROM_REPO_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} + +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/db/remove.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/db/remove.sh +# shellcheck source=src//lib/util/pacman.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/pacman.sh + +source /usr/share/makepkg/util/message.sh + +set -eo pipefail + + +pkgctl_aur_drop_from_repo_usage() { + local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} + cat <<- _EOF_ + Usage: ${COMMAND} [OPTIONS] [PATH]... + + Drops a specified package from the official repositories to the Arch + User Repository. + + This command requires a local Git clone of the package repository. It + reconfigures the repository for AUR compatibility and pushes it to the + AUR. Afterwards, the package is removed from the official repository. + + By default, the package is automatically disowned in the AUR. + + OPTIONS + --no-disown Do not disown the package on the AUR + -f, --force Force push to the AUR overwriting the remote repository + -h, --help Show this help text + + EXAMPLES + $ ${COMMAND} foo + $ ${COMMAND} --no-disown --force +_EOF_ +} + +pkgctl_aur_drop_from_repo() { + # options + local paths=() + local DISOWN=1 + local FORCE=0 + + # variables + local path realpath pkgbase pkgrepo remote_url + + while (( $# )); do + case $1 in + -h|--help) + pkgctl_aur_drop_from_repo_usage + exit 0 + ;; + --no-disown) + DISOWN=0 + shift + ;; + -f|--force) + FORCE=1 + shift + ;; + --) + shift + break + ;; + -*) + die "Invalid argument: %s" "$1" + ;; + *) + paths=("$@") + break + ;; + esac + done + + # check if invoked without any path from within a packaging repo + if (( ${#paths[@]} == 0 )); then + if [[ -f PKGBUILD ]]; then + paths=(".") + else + pkgctl_aur_drop_from_repo_usage + exit 1 + fi + fi + + for path in "${paths[@]}"; do + if ! realpath=$(realpath -e "${path}"); then + die "No such directory: ${path}" + fi + + pkgbase=$(basename "${realpath}") + pkgbase=${pkgbase%.git} + + if [[ ! -d "${path}/.git" ]]; then + die "Not a Git repository: ${path}" + fi + + pushd "${path}" >/dev/null + + if [[ ! -f PKGBUILD ]]; then + die 'PKGBUILD not found in %s' "${path}" + fi + + msg "Dropping ${pkgbase} to the AUR" + + remote_url="${AUR_URL_SSH}:${pkgbase}.git" + if ! git remote add origin "${remote_url}" &>/dev/null; then + git remote set-url origin "${remote_url}" + fi + + # move the main branch to master + if [[ $(git symbolic-ref --quiet --short HEAD) == main ]]; then + git branch --move master + git config branch.master.merge refs/heads/master + fi + + # auto generate .SRCINFO if not already present + if [[ -z "$(git ls-tree -r HEAD --name-only .SRCINFO)" ]]; then + stat_busy 'Generating .SRCINFO' + makepkg --printsrcinfo > .SRCINFO + stat_done + + git add --force -- .SRCINFO + git commit --quiet --message "Adding .SRCINFO" -- .SRCINFO + fi + + msg "Pushing ${pkgbase} to the AUR" + if (( FORCE )); then + AUR_OVERWRITE=1 \ + GIT_SSH_COMMAND="ssh -o SendEnv=AUR_OVERWRITE" \ + git push --force origin master + else + git push origin master + fi + + # update the local default branch in case this clone is used in the future + git remote set-head origin master + + if (( DISOWN )); then + msg "Disowning ${pkgbase} on the AUR" + # shellcheck disable=SC2029 + ssh "${AUR_URL_SSH}" disown "${pkgbase}" + fi + + # auto-detection of the repo to remove from + if ! pkgrepo=$(get_pacman_repo_from_pkgbuild PKGBUILD); then + die 'Failed to get pacman repo' + fi + + msg "Deleting ${pkgbase} from the official repository" + if [[ -z "${pkgrepo}" ]]; then + warning 'Did not find %s in any repository, please delete manually' "${pkgbase}" + else + msg2 " repo: ${pkgrepo}" + pkgctl_db_remove "${pkgrepo}" "${pkgbase}" + fi + + popd >/dev/null + done +} diff --git a/src/lib/build/build.sh b/src/lib/build/build.sh index 3394395..171bb9a 100644 --- a/src/lib/build/build.sh +++ b/src/lib/build/build.sh @@ -14,18 +14,25 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/db/update.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/release.sh # shellcheck source=src/lib/util/git.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh +# shellcheck source=src/lib/util/srcinfo.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/srcinfo.sh # shellcheck source=src/lib/util/pacman.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/pacman.sh +# shellcheck source=src/lib/util/pkgbuild.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/pkgbuild.sh +# shellcheck source=src/lib/valid-build-install.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-build-install.sh # shellcheck source=src/lib/valid-repos.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-repos.sh # shellcheck source=src/lib/valid-tags.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-tags.sh +# shellcheck source=src/lib/valid-inspect.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-inspect.sh source /usr/share/makepkg/util/config.sh source /usr/share/makepkg/util/message.sh -set -e -set -o pipefail +set -eo pipefail pkgctl_build_usage() { @@ -42,19 +49,24 @@ pkgctl_build_usage() { BUILD OPTIONS --arch ARCH Specify architectures to build for (disables auto-detection) - --repo REPO Specify a target repository (disables auto-detection) + --repo REPO Specify target repository for new packages not in any official repo -s, --staging Build against the staging counterpart of the auto-detected repo -t, --testing Build against the testing counterpart of the auto-detected repo -o, --offload Build on a remote server and transfer artifacts afterwards -c, --clean Recreate the chroot before building - -I, --install FILE Install a package into the working copy of the chroot + --inspect WHEN Spawn an interactive shell to inspect the chroot (never, always, failure) -w, --worker SLOT Name of the worker slot, useful for concurrent builds (disables automatic names) --nocheck Do not run the check() function in the PKGBUILD + INSTALL OPTIONS + -I, --install-to-chroot FILE Install a package to the working copy of the chroot + -i, --install-to-host MODE Install the built package to the host system, possible modes are 'all' and 'auto' + PKGBUILD OPTIONS --pkgver=PKGVER Set pkgver, reset pkgrel and update checksums --pkgrel=PKGREL Set pkgrel to a given value --rebuild Increment the current pkgrel variable + --update-checksums Force computation and update of the checksums (disables auto-detection) -e, --edit Edit the PKGBUILD before building RELEASE OPTIONS @@ -77,8 +89,7 @@ pkgctl_build_check_option_group_repo() { local repo=$2 local testing=$3 local staging=$4 - if ( (( testing )) && (( staging )) ) || - ( [[ $repo =~ ^.*-(staging|testing)$ ]] && ( (( testing )) || (( staging )) )); then + if [[ -n "${repo}" ]] || (( testing )) || (( staging )); then die "The argument '%s' cannot be used with one or more of the other specified arguments" "${option}" exit 1 fi @@ -104,7 +115,7 @@ pkgctl_build() { exit 1 fi - local UPDPKGSUMS=0 + local UPDATE_CHECKSUMS=0 local EDIT=0 local REBUILD=0 local OFFLOAD=0 @@ -112,6 +123,7 @@ pkgctl_build() { local TESTING=0 local RELEASE=0 local DB_UPDATE=0 + local INSTALL_TO_HOST=none local REPO= local PKGVER= @@ -124,12 +136,13 @@ pkgctl_build() { local MAKECHROOT_OPTIONS=() local RELEASE_OPTIONS=() local MAKEPKG_OPTIONS=() + local INSTALL_HOST_PACKAGES=() local WORKER= local WORKER_SLOT= # variables - local path pkgbase pkgrepo source + local _arch path pkgbase pkgrepo source pkgbuild_checksum current_checksum while (( $# )); do case $1 in @@ -139,18 +152,19 @@ pkgctl_build() { ;; --repo) (( $# <= 1 )) && die "missing argument for %s" "$1" - REPO="${2}" pkgctl_build_check_option_group_repo '--repo' "${REPO}" "${TESTING}" "${STAGING}" + REPO="${2}" + RELEASE_OPTIONS+=("--repo" "${REPO}") shift 2 ;; --arch) (( $# <= 1 )) && die "missing argument for %s" "$1" if [[ ${2} == all ]]; then - BUILD_ARCH=("${_arch[@]::${#_arch[@]}-1}") + BUILD_ARCH=("${DEVTOOLS_VALID_ARCHES[@]::${#DEVTOOLS_VALID_ARCHES[@]}-1}") elif [[ ${2} == any ]]; then - BUILD_ARCH=("${_arch[0]}") + BUILD_ARCH=("${DEVTOOLS_VALID_ARCHES[0]}") elif ! in_array "${2}" "${BUILD_ARCH[@]}"; then - if ! in_array "${2}" "${_arch[@]}"; then + if ! in_array "${2}" "${DEVTOOLS_VALID_ARCHES[@]}"; then die 'invalid architecture: %s' "${2}" fi BUILD_ARCH+=("${2}") @@ -161,7 +175,7 @@ pkgctl_build() { pkgctl_build_check_option_group_ver '--pkgver' "${PKGVER}" "${PKGREL}" "${REBUILD}" PKGVER="${1#*=}" PKGREL=1 - UPDPKGSUMS=1 + UPDATE_CHECKSUMS=1 shift ;; --pkgrel=*) @@ -169,6 +183,10 @@ pkgctl_build() { PKGREL="${1#*=}" shift ;; + --update-checksums) + UPDATE_CHECKSUMS=1 + shift + ;; --rebuild) # shellcheck source=src/lib/util/git.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh @@ -185,23 +203,37 @@ pkgctl_build() { shift ;; -s|--staging) - STAGING=1 pkgctl_build_check_option_group_repo '--staging' "${REPO}" "${TESTING}" "${STAGING}" + STAGING=1 + RELEASE_OPTIONS+=("--staging") shift ;; -t|--testing) - TESTING=1 pkgctl_build_check_option_group_repo '--testing' "${REPO}" "${TESTING}" "${STAGING}" + TESTING=1 + RELEASE_OPTIONS+=("--testing") shift ;; -c|--clean) BUILD_OPTIONS+=("-c") shift ;; - -I|--install) + -I|--install-to-chroot) + (( $# <= 1 )) && die "missing argument for %s" "$1" + if (( OFFLOAD )); then + MAKECHROOT_OPTIONS+=("-I" "$2") + else + MAKECHROOT_OPTIONS+=("-I" "$(realpath "$2")") + fi + warning 'installing packages to the chroot may break reproducible builds, use with caution!' + shift 2 + ;; + -i|--install-to-host) (( $# <= 1 )) && die "missing argument for %s" "$1" - MAKECHROOT_OPTIONS+=("-I" "$2") - warning 'installing packages into the chroot may break reproducible builds, use with caution!' + if ! in_array "$2" "${DEVTOOLS_VALID_BUILD_INSTALL[@]}"; then + die 'invalid install mode: %s' "${2}" + fi + INSTALL_TO_HOST=$2 shift 2 ;; --nocheck) @@ -209,6 +241,14 @@ pkgctl_build() { warning 'not running checks is disallowed for official packages, except for bootstrapping. Please rebuild after bootstrapping is completed!' shift ;; + --inspect) + (( $# <= 1 )) && die "missing argument for %s" "$1" + if ! in_array "${2}" "${DEVTOOLS_VALID_INSPECT_MODES[@]}"; then + die "Invalid inspect mode: %s" "${2}" + fi + MAKECHROOT_OPTIONS+=("-x" "${2}") + shift 2 + ;; -r|--release) # shellcheck source=src/lib/release.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/release.sh @@ -274,7 +314,7 @@ pkgctl_build() { if [[ -z ${REPO} ]]; then update_pacman_repo_cache # Check valid repos if not resolved dynamically - elif ! in_array "${REPO}" "${_repos[@]}"; then + elif ! in_array "${REPO}" "${DEVTOOLS_VALID_REPOS[@]}"; then die "Invalid repository target: %s" "${REPO}" fi @@ -290,18 +330,32 @@ pkgctl_build() { . ./PKGBUILD pkgbase=${pkgbase:-$pkgname} pkgrepo=${REPO} + pkgbuild_checksum=$(b2sum PKGBUILD | awk '{print $1}') msg "Building ${pkgbase}" - # auto-detection of build target - if [[ -z ${pkgrepo} ]]; then - if ! pkgrepo=$(get_pacman_repo_from_pkgbuild PKGBUILD); then - die 'failed to get pacman repo' - fi - if [[ -z "${pkgrepo}" ]]; then - die 'unknown repo, specify --repo for packages not currently in any official repo' + # auto-detect target repository + if ! repo=$(get_pacman_repo_from_pkgbuild PKGBUILD); then + die 'Failed to query pacman repo' + fi + + # fail if an existing package specifies --repo + if [[ -n "${repo}" ]] && [[ -n ${pkgrepo} ]]; then + # allow unstable to use --repo + if [[ ${pkgrepo} == *unstable ]]; then + repo=${pkgrepo} + else + die 'Using --repo for packages that exist in official repositories is disallowed' fi fi + # assign auto-detected target repository + if [[ -n ${repo} ]]; then + pkgrepo=${repo} + # fallback to extra for unreleased packages + elif [[ -z ${pkgrepo} ]]; then + pkgrepo=extra + fi + # special cases to resolve final build target if (( TESTING )); then pkgrepo="${pkgrepo}-testing" @@ -316,9 +370,15 @@ pkgctl_build() { BUILD_ARCH=("") elif (( ${#BUILD_ARCH[@]} == 0 )); then if in_array any "${arch[@]}"; then - BUILD_ARCH=("${_arch[0]}") + BUILD_ARCH=("${DEVTOOLS_VALID_ARCHES[0]}") else - BUILD_ARCH+=("${arch[@]}") + for _arch in "${arch[@]}"; do + if in_array "${_arch}" "${DEVTOOLS_VALID_ARCHES[@]}"; then + BUILD_ARCH+=("$_arch") + else + warning 'invalid architecture, not building for: %s' "${_arch}" + fi + done fi fi @@ -329,7 +389,7 @@ pkgctl_build() { # increment pkgrel on rebuild if (( REBUILD )); then - # try to figure out of pkgrel has been changed + # try to figure out if pkgrel has been changed if ! old_pkgrel=$(git_diff_tree HEAD PKGBUILD | grep --perl-regexp --only-matching --max-count=1 '^-pkgrel=\K\w+'); then old_pkgrel=${pkgrel} fi @@ -346,20 +406,14 @@ pkgctl_build() { # update pkgver if [[ -n ${PKGVER} ]]; then - if [[ $(type -t pkgver) == function ]]; then - # TODO: check if die or warn, if we provide _commit _gitcommit setter maybe? - warning 'setting pkgver variable has no effect if the PKGBUILD has a pkgver() function' - fi msg "Bumping pkgver to ${PKGVER}" - grep --extended-regexp --quiet --max-count=1 "^pkgver=${pkgver}$" PKGBUILD || die "Non-standard pkgver declaration" - sed --regexp-extended "s|^(pkgver=)${pkgver}$|\1${PKGVER}|g" -i PKGBUILD + pkgbuild_set_pkgver "${PKGVER}" fi # update pkgrel if [[ -n ${PKGREL} ]]; then msg "Bumping pkgrel to ${PKGREL}" - grep --extended-regexp --quiet --max-count=1 "^pkgrel=${pkgrel}$" PKGBUILD || die "Non-standard pkgrel declaration" - sed --regexp-extended "s|^(pkgrel=)${pkgrel}$|\1${PKGREL}|g" -i PKGBUILD + pkgbuild_set_pkgrel "${PKGREL}" fi # edit PKGBUILD @@ -381,10 +435,18 @@ pkgctl_build() { # update checksums if any sources are declared - if (( UPDPKGSUMS )) && (( ${#source[@]} >= 1 )); then + if (( UPDATE_CHECKSUMS )) && (( ${#source[@]} >= 1 )); then updpkgsums fi + # re-source the PKGBUILD if it changed + current_checksum="$(b2sum PKGBUILD | awk '{print $1}')" + if [[ ${pkgbuild_checksum} != "${current_checksum}" ]]; then + pkgbuild_checksum=${current_checksum} + # shellcheck source=contrib/makepkg/PKGBUILD.proto + . ./PKGBUILD + fi + # execute build for arch in "${BUILD_ARCH[@]}"; do if [[ -n $arch ]]; then @@ -402,9 +464,41 @@ pkgctl_build() { fi done + # re-source the PKGBUILD if it changed + current_checksum="$(b2sum PKGBUILD | awk '{print $1}')" + if [[ ${pkgbuild_checksum} != "${current_checksum}" ]]; then + pkgbuild_checksum=${current_checksum} + # shellcheck source=contrib/makepkg/PKGBUILD.proto + . ./PKGBUILD + fi + + # auto generate .SRCINFO + # shellcheck disable=SC2119 + write_srcinfo_file + + # test-install (some of) the produced packages + if [[ ${INSTALL_TO_HOST} == auto ]] || [[ ${INSTALL_TO_HOST} == all ]]; then + # shellcheck disable=2119 + load_makepkg_config + + # this is inspired by print_all_package_names from libmakepkg + local version pkg_architecture pkg pkgfile + version=$(get_full_version) + + for pkg in "${pkgname[@]}"; do + pkg_architecture=$(get_pkg_arch "$pkg") + pkgfile=$(realpath "$(printf "%s/%s-%s-%s%s\n" "${PKGDEST:-.}" "$pkg" "$version" "$pkg_architecture" "$PKGEXT")") + + # check if we install all packages or if the (split-)package is already installed + if [[ ${INSTALL_TO_HOST} == all ]] || ( [[ ${INSTALL_TO_HOST} == auto ]] && pacman -Qq -- "$pkg" &>/dev/null ); then + INSTALL_HOST_PACKAGES+=("$pkgfile") + fi + done + fi + # release the build if (( RELEASE )); then - pkgctl_release --repo "${pkgrepo}" "${RELEASE_OPTIONS[@]}" + pkgctl_release "${RELEASE_OPTIONS[@]}" fi # reset common PKGBUILD variables @@ -412,6 +506,12 @@ pkgctl_build() { popd >/dev/null done + # install all collected packages to the host system + if (( ${#INSTALL_HOST_PACKAGES[@]} )); then + msg "Installing built packages to the host system" + sudo pacman -U -- "${INSTALL_HOST_PACKAGES[@]}" + fi + # update the binary package repo db as last action if (( RELEASE )) && (( DB_UPDATE )); then # shellcheck disable=2119 diff --git a/src/lib/cache.sh b/src/lib/cache.sh new file mode 100644 index 0000000..24056fa --- /dev/null +++ b/src/lib/cache.sh @@ -0,0 +1,22 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_CACHE_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_CACHE_SH=1 + +set -e + +readonly XDG_DEVTOOLS_CACHE_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/devtools" + +get_cache_file() { + local filename=$1 + local path="${XDG_DEVTOOLS_CACHE_DIR}/${filename}" + + mkdir --parents -- "$(dirname -- "$path")" + if [[ ! -f ${path} ]]; then + touch -- "${path}" + fi + + printf '%s' "${path}" +} diff --git a/src/lib/common.sh b/src/lib/common.sh index 3d1ee56..ff767c6 100644 --- a/src/lib/common.sh +++ b/src/lib/common.sh @@ -13,7 +13,7 @@ set +u +o posix $DEVTOOLS_INCLUDE_COMMON_SH # Avoid any encoding problems -export LANG=C +export LANG=C.UTF-8 # Set buildtool properties export BUILDTOOL=devtools @@ -22,19 +22,33 @@ export BUILDTOOLVER=@buildtoolver@ # Set common properties export PACMAN_KEYRING_DIR=/etc/pacman.d/gnupg export GITLAB_HOST=gitlab.archlinux.org -export GIT_REPO_SPEC_VERSION=1 +export GIT_REPO_SPEC_VERSION=2 export GIT_PACKAGING_NAMESPACE=archlinux/packaging/packages export GIT_PACKAGING_NAMESPACE_ID=11323 export GIT_PACKAGING_URL_SSH="git@${GITLAB_HOST}:${GIT_PACKAGING_NAMESPACE}" export GIT_PACKAGING_URL_HTTPS="https://${GITLAB_HOST}/${GIT_PACKAGING_NAMESPACE}" export PACKAGING_REPO_RELEASE_HOST=repos.archlinux.org +export PKGBASE_MAINTAINER_URL=https://archlinux.org/packages/pkgbase-maintainer +export AUR_URL_SSH=aur@aur.archlinux.org + +# ensure TERM is set with a fallback to dumb +export TERM=${TERM:-dumb} # check if messages are to be printed using color if [[ -t 2 && "$TERM" != dumb ]] || [[ ${DEVTOOLS_COLOR} == always ]]; then colorize + if tput setaf 0 &>/dev/null; then + PURPLE="$(tput setaf 5)" + DARK_GREEN="$(tput setaf 2)" + UNDERLINE="$(tput smul)" + else + PURPLE="\e[35m" + DARK_GREEN="\e[32m" + UNDERLINE="\e[4m" + fi else # shellcheck disable=2034 - declare -gr ALL_OFF='' BOLD='' BLUE='' GREEN='' RED='' YELLOW='' + declare -gr ALL_OFF='' BOLD='' BLUE='' GREEN='' RED='' YELLOW='' PURPLE='' DARK_GREEN='' UNDERLINE='' fi stat_busy() { @@ -53,6 +67,11 @@ stat_done() { printf "${BOLD}done${ALL_OFF}\n" >&2 } +stat_failed() { + # shellcheck disable=2059 + printf "${BOLD}${RED}failed${ALL_OFF}\n" >&2 +} + msg_success() { local msg=$1 local padding @@ -77,6 +96,15 @@ msg_warn() { printf "%s %s\n" "${padding}${YELLOW}!${ALL_OFF}" "${msg}" >&2 } +print_workdir_error() { + if [[ ! -f "${WORKDIR}"/error ]]; then + return + fi + while read -r LINE; do + error '%s' "${LINE}" + done < "${WORKDIR}/error" +} + _setup_workdir=false setup_workdir() { [[ -z ${WORKDIR:-} ]] && WORKDIR=$(mktemp -d --tmpdir "${0##*/}.XXXXXXXXXX") @@ -89,6 +117,9 @@ cleanup() { if [[ -n ${WORKDIR:-} ]] && $_setup_workdir; then rm -rf "$WORKDIR" fi + if tput setaf 0 &>/dev/null; then + tput cnorm >&2 + fi exit "${1:-0}" } @@ -120,7 +151,7 @@ lock() { # Only reopen the FD if it wasn't handed to us if ! [[ "/dev/fd/$1" -ef "$2" ]]; then mkdir -p -- "$(dirname -- "$2")" - eval "exec $1>"'"$2"' + eval "exec $1>>"'"$2"' fi if ! flock -n "$1"; then diff --git a/src/lib/db.sh b/src/lib/db.sh index 397ff0d..91e4da5 100644 --- a/src/lib/db.sh +++ b/src/lib/db.sh @@ -15,7 +15,7 @@ pkgctl_db_usage() { cat <<- _EOF_ Usage: ${COMMAND} [COMMAND] [OPTIONS] - Pacman database modification for packge update, move etc + Pacman database modification for package update, move etc COMMANDS move Move packages between pacman repositories diff --git a/src/lib/release.sh b/src/lib/release.sh index aabbd35..acb3b54 100644 --- a/src/lib/release.sh +++ b/src/lib/release.sh @@ -35,7 +35,7 @@ pkgctl_release_usage() { OPTIONS -m, --message MSG Use the given <msg> as the commit message - -r, --repo REPO Specify a target repository (disables auto-detection) + -r, --repo REPO Specify target repository for new packages not in any official repo -s, --staging Release to the staging counterpart of the auto-detected repo -t, --testing Release to the testing counterpart of the auto-detected repo -u, --db-update Automatically update the pacman database after uploading @@ -43,8 +43,8 @@ pkgctl_release_usage() { EXAMPLES $ ${COMMAND} - $ ${COMMAND} --repo core-testing --message 'libyay 0.42 rebuild' libfoo libbar - $ ${COMMAND} --staging --db-update libfoo + $ ${COMMAND} --staging --message 'libyay 0.42 rebuild' libfoo libbar + $ ${COMMAND} --repo extra --db-update new-package _EOF_ } @@ -126,7 +126,7 @@ pkgctl_release() { if [[ -z ${REPO} ]]; then update_pacman_repo_cache # Check valid repos if not resolved dynamically - elif ! in_array "${REPO}" "${_repos[@]}"; then + elif ! in_array "${REPO}" "${DEVTOOLS_VALID_REPOS[@]}"; then die "Invalid repository target: %s" "${REPO}" fi @@ -134,15 +134,27 @@ pkgctl_release() { pushd "${path}" >/dev/null pkgbase=$(basename "${path}") - if [[ -n ${REPO} ]]; then - repo=${REPO} - else - if ! repo=$(get_pacman_repo_from_pkgbuild PKGBUILD); then - die 'Failed to get pacman repo' + # auto-detect target repository + if ! repo=$(get_pacman_repo_from_pkgbuild PKGBUILD); then + die 'Failed to query pacman repo' + fi + + # fail if an existing package specifies --repo + if [[ -n "${repo}" ]] && [[ -n ${REPO} ]]; then + # allow unstable to use --repo + if [[ ${REPO} == *unstable ]]; then + repo=${REPO} + else + die 'Using --repo for packages that exist in official repositories is disallowed' fi - if [[ -z "${repo}" ]]; then - die 'Unknown repo, please specify --repo for new packages' + fi + + # fail if a new package does not specify --repo + if [[ -z "${repo}" ]]; then + if [[ -z ${REPO} ]]; then + die 'Specify --repo for packages that do not yet exist in official repositories' fi + repo=${REPO} fi if (( TESTING )); then diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh index a8cf6f5..33a333f 100644 --- a/src/lib/repo/clone.sh +++ b/src/lib/repo/clone.sh @@ -8,16 +8,21 @@ DEVTOOLS_INCLUDE_REPO_CLONE_SH=1 _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} # shellcheck source=src/lib/common.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/api/archweb.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/archweb.sh # shellcheck source=src/lib/api/gitlab.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/gitlab.sh # shellcheck source=src/lib/repo/configure.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/repo/configure.sh +# shellcheck source=src/lib/util/git.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh # shellcheck source=src/lib/repo/arch32.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/repo/arch32.sh source /usr/share/makepkg/util/message.sh set -e +set -o pipefail pkgctl_repo_clone_usage() { @@ -55,6 +60,7 @@ pkgctl_repo_clone() { fi # options + local protocol=ssh local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_SSH} local CLONE_ALL=0 local MAINTAINER= @@ -76,6 +82,7 @@ pkgctl_repo_clone() { ;; --protocol=https) GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS} + protocol=https CONFIGURE_OPTIONS+=("$1") shift ;; @@ -86,6 +93,7 @@ pkgctl_repo_clone() { else die "unsupported protocol: %s" "$2" fi + protocol="$2" CONFIGURE_OPTIONS+=("$1" "$2") shift 2 ;; @@ -140,33 +148,18 @@ pkgctl_repo_clone() { # Query packages of a maintainer if [[ -n ${MAINTAINER} ]]; then - stat_busy "Query packages" - max_pages=$(curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&maintainer=${MAINTAINER}" | jq -r '.num_pages') - if [[ ! ${max_pages} =~ ([[:digit:]]) ]]; then - stat_done - warning "found no packages for maintainer ${MAINTAINER}" - exit 0 + mapfile -t pkgbases < <(archweb_query_maintainer_packages "${MAINTAINER}") + if ! wait $!; then + die "Failed to query maintainer packages" fi - mapfile -t pkgbases < <(for page in $(seq "${max_pages}"); do - curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&maintainer=${MAINTAINER}&page=${page}" | jq -r '.results[].pkgbase' - stat_progress - done | sort --unique) - stat_done fi # Query all released packages if (( CLONE_ALL )); then - stat_busy "Query all released packages" - max_pages=$(curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name" | jq -r '.num_pages') - if [[ ! ${max_pages} =~ ([[:digit:]]) ]]; then - stat_done - die "failed to query packages" + mapfile -t pkgbases < <(archweb_query_all_packages) + if ! wait $!; then + die "Failed to query all packages" fi - mapfile -t pkgbases < <(for page in $(seq "${max_pages}"); do - curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&page=${page}" | jq -r '.results[].pkgbase' - stat_progress - done | sort --unique) - stat_done fi # parallelization @@ -179,6 +172,12 @@ pkgctl_repo_clone() { if [[ -n "${VERSION}" ]]; then command+=" --switch '${VERSION}'" fi + + # warm up ssh connection as it may require user input (key unlock, hostkey verification etc) + if [[ ${protocol} == ssh ]]; then + git_warmup_ssh_connection + fi + if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${pkgbases[@]}"; then die 'Failed to clone some packages, please check the output' exit 1 diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh index 73300ae..b3c188c 100644 --- a/src/lib/repo/configure.sh +++ b/src/lib/repo/configure.sh @@ -10,11 +10,14 @@ _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh # shellcheck source=src/lib/api/gitlab.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/gitlab.sh +# shellcheck source=src/lib/util/git.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh source /usr/share/makepkg/util/config.sh source /usr/share/makepkg/util/message.sh set -e +shopt -s nullglob pkgctl_repo_configure_usage() { @@ -32,6 +35,8 @@ pkgctl_repo_configure_usage() { address by choosing SSH for all official packager identities and read-only HTTPS otherwise. + Git default excludes and hooks are applied to the configured repo. + OPTIONS --protocol https Configure remote url to use https -j, --jobs N Run up to N jobs in parallel (default: $(nproc)) @@ -102,7 +107,7 @@ pkgctl_repo_configure() { # variables local -r command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} - local path realpath pkgbase remote_url project_path + local path realpath pkgbase remote_url project_path hook local PACKAGER GPGKEY packager_name packager_email while (( $# )); do @@ -188,6 +193,12 @@ pkgctl_repo_configure() { if [[ -n ${BOLD} ]]; then export DEVTOOLS_COLOR=always fi + + # warm up ssh connection as it may require user input (key unlock, hostkey verification etc) + if [[ ${proto} == ssh ]]; then + git_warmup_ssh_connection + fi + if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${paths[@]}"; then die 'Failed to configure some packages, please check the output' exit 1 @@ -222,7 +233,15 @@ pkgctl_repo_configure() { git config branch.main.merge refs/heads/main fi + # configure spec version and variant to avoid using development hooks in production git config devtools.version "${GIT_REPO_SPEC_VERSION}" + if [[ ${_DEVTOOLS_LIBRARY_DIR} == /usr/share/devtools ]]; then + git config devtools.variant canonical + else + warning "Configuring with development version of pkgctl, do not use this repo in production" + git config devtools.variant development + fi + git config pull.rebase true git config branch.autoSetupRebase always git config branch.main.remote origin @@ -249,6 +268,18 @@ pkgctl_repo_configure() { git config user.signingKey "${GPGKEY}" fi + # set default git exclude + mkdir -p .git/info + ln -sf "${_DEVTOOLS_LIBRARY_DIR}/git.conf.d/template/info/exclude" \ + .git/info/exclude + + # set default git hooks + mkdir -p .git/hooks + rm -f .git/hooks/*.sample + for hook in "${_DEVTOOLS_LIBRARY_DIR}"/git.conf.d/template/hooks/*; do + ln -sf "${hook}" ".git/hooks/$(basename "${hook}")" + done + if ! git ls-remote origin &>/dev/null; then warning "configured remote origin may not exist, run:" msg2 "pkgctl repo create ${pkgbase}" diff --git a/src/lib/repo/web.sh b/src/lib/repo/web.sh index 45ea53b..ab3d8c7 100644 --- a/src/lib/repo/web.sh +++ b/src/lib/repo/web.sh @@ -23,6 +23,7 @@ pkgctl_repo_web_usage() { no arguments, open the package cloned in the current working directory. OPTIONS + --print Print the url instead of opening it with xdg-open -h, --help Show this help text EXAMPLES @@ -32,7 +33,8 @@ _EOF_ pkgctl_repo_web() { local pkgbases=() - local path giturl pkgbase + local path giturl pkgbase url + local mode=open # option checking while (( $# )); do @@ -41,6 +43,10 @@ pkgctl_repo_web() { pkgctl_repo_web_usage exit 0 ;; + --print) + mode=print + shift + ;; --) shift break @@ -56,7 +62,7 @@ pkgctl_repo_web() { done # Check if web mode has xdg-open - if ! command -v xdg-open &>/dev/null; then + if [[ ${mode} == open ]] && ! command -v xdg-open &>/dev/null; then die "The web command requires 'xdg-open'" fi @@ -78,7 +84,18 @@ pkgctl_repo_web() { fi for pkgbase in "${pkgbases[@]}"; do + pkgbase=$(basename "${pkgbase}") path=$(gitlab_project_name_to_path "${pkgbase}") - xdg-open "${GIT_PACKAGING_URL_HTTPS}/${path}" + url="${GIT_PACKAGING_URL_HTTPS}/${path}" + case ${mode} in + open) + xdg-open "${url}" + ;; + print) + printf "%s\n" "${url}" + ;; + *) + die "Unknown mode: ${mode}" + esac done } diff --git a/src/lib/search.sh b/src/lib/search.sh new file mode 100644 index 0000000..d3bad68 --- /dev/null +++ b/src/lib/search.sh @@ -0,0 +1,308 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_SEARCH_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_SEARCH_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/cache.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/cache.sh +# shellcheck source=src/lib/api/gitlab.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/gitlab.sh +# shellcheck source=src/lib/valid-search.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-search.sh +# shellcheck source=src/lib/util/term.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/term.sh + +source /usr/share/makepkg/util/util.sh +source /usr/share/makepkg/util/message.sh + +set -eo pipefail + + +pkgctl_search_usage() { + local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} + cat <<- _EOF_ + Usage: ${COMMAND} [OPTIONS] QUERY + + Search for an expression across the GitLab packaging group. + + To use a filter, include it in your query. You may use wildcards (*) to + use glob matching. + + Available filters for the blobs scope: path, extension + + Every usage of the search command must be authenticated. Consult the + 'pkgctl auth' command to authenticate with GitLab or view the + authentication status. + + SEARCH TIPS + Syntax Description Example + ─────────────────────────────────────── + " Exact search "gem sidekiq" + ~ Fuzzy search J~ Doe + | Or display | banner + + And display +banner + - Exclude display -banner + * Partial bug error 50* + \\ Escape \\*md + # Issue ID #23456 + ! Merge request !23456 + + OPTIONS + -h, --help Show this help text + + FILTER OPTIONS + --no-default-filter Do not apply default filter (like -path:keys/pgp/*.asc) + + OUTPUT OPTIONS + --json Enable printing in JSON; Shorthand for '--format json' + -F, --format FORMAT Controls the formatting of the results; FORMAT is 'pretty', + 'plain', or 'json' (default: pretty) + -N, --no-line-number Don't show line numbers when formatting results + + EXAMPLES + $ ${COMMAND} linux + $ ${COMMAND} --json '"pytest -v" +PYTHONPATH' +_EOF_ +} + +pkgctl_search_check_option_group_format() { + local option=$1 + local output_format=$2 + if [[ -n ${output_format} ]]; then + die "The argument '%s' cannot be used with one or more of the other specified arguments" "${option}" + exit 1 + fi + return 0 +} + +pkgctl_search() { + if (( $# < 1 )); then + pkgctl_search_usage + exit 0 + fi + + # options + local search + local output_format= + local use_default_filter=1 + local line_numbers=1 + + # variables + local bat_style="header,grid" + local default_filter="-path:keys/pgp/*.asc" + local graphql_lookup_batch=200 + local output result query entries from until length + local project_name_cache_file project_name_lookup project_ids project_id project_name project_slice + local mapping_output path startline currentline data line + + while (( $# )); do + case $1 in + -h|--help) + pkgctl_search_usage + exit 0 + ;; + --no-default-filter) + use_default_filter=0 + shift + ;; + --json) + pkgctl_search_check_option_group_format "$1" "${output_format}" + output_format=json + shift + ;; + -F|--format) + (( $# <= 1 )) && die "missing argument for %s" "$1" + pkgctl_search_check_option_group_format "$1" "${output_format}" + output_format="${2}" + if ! in_array "${output_format}" "${valid_search_output_format[@]}"; then + die "Unknown output format: %s" "${output_format}" + fi + shift 2 + ;; + -N|--no-line-number) + line_numbers=0 + shift + ;; + --) + shift + break + ;; + -*) + die "invalid argument: %s" "$1" + ;; + *) + break + ;; + esac + done + + if (( $# == 0 )); then + pkgctl_search_usage + exit 1 + fi + + # assign search parameter + search="${*}" + if (( use_default_filter )); then + search+=" ${default_filter}" + fi + + # assign default output format + if [[ -z ${output_format} ]]; then + output_format=pretty + fi + + # check for optional dependencies + if [[ ${output_format} == pretty ]] && ! command -v bat &>/dev/null; then + warning "Failed to find optional dependency 'bat': falling back to plain output" + output_format=plain + fi + + # populate line numbers option + if (( line_numbers )); then + bat_style="numbers,${bat_style}" + fi + + # call the gitlab search API + status_dir=$(mktemp --tmpdir="${WORKDIR}" --directory pkgctl-gitlab-api.XXXXXXXXXX) + printf "📡 Querying GitLab search API..." > "${status_dir}/status" + term_spinner_start "${status_dir}" + output=$(gitlab_api_search "${search}" "${status_dir}/status") + term_spinner_stop "${status_dir}" + msg_success "Querying GitLab search API" + + # collect project ids whose name needs to be looked up + project_name_cache_file=$(get_cache_file gitlab/project_id_to_name) + lock 11 "${project_name_cache_file}" "Locking project name cache" + mapfile -t project_ids < <( + jq --raw-output '[.[].project_id] | unique[]' <<< "${output}" | \ + grep --invert-match --file <(awk '{ print $1 }' < "${project_name_cache_file}" )) + + # look up project names + tmp_file=$(mktemp --tmpdir="${WORKDIR}" pkgctl-gitlab-api-spinner.tmp.XXXXXXXXXX) + printf "📡 Querying GitLab project names..." > "${status_dir}/status" + term_spinner_start "${status_dir}" + local entries="${#project_ids[@]}" + local until=0 + while (( until < entries )); do + from=${until} + until=$(( until + graphql_lookup_batch )) + if (( until > entries )); then + until=${entries} + fi + length=$(( until - from )) + + percentage=$(( 100 * until / entries )) + printf "📡 Querying GitLab project names: %s/%s [%s] %%spinner%%" \ + "${BOLD}${until}" "${entries}" "${percentage}%${ALL_OFF}" \ + > "${tmp_file}" + mv "${tmp_file}" "${status_dir}/status" + + project_slice=("${project_ids[@]:${from}:${length}}") + printf -v projects '"gid://gitlab/Project/%s",' "${project_slice[@]}" + query='{ + projects(after: "" ids: ['"${projects}"']) { + pageInfo { + startCursor + endCursor + hasNextPage + } + nodes { + id + name + } + } + }' + mapping_output=$(gitlab_api_get_project_name_mapping "${query}") + + # update cache + while read -r project_id project_name; do + printf "%s %s\n" "${project_id}" "${project_name}" >> "${project_name_cache_file}" + done < <(jq --raw-output \ + '.[] | "\(.id | rindex("/") as $lastSlash | .[$lastSlash+1:]) \(.name)"' \ + <<< "${mapping_output}") + done + term_spinner_stop "${status_dir}" + msg_success "Querying GitLab project names" + + # read project_id to name mapping from cache + declare -A project_name_lookup=() + while read -r project_id project_name; do + project_name_lookup[${project_id}]=${project_name} + done < "${project_name_cache_file}" + + # close project name cache lock + lock_close 11 + + # output mode JSON + if [[ ${output_format} == json ]]; then + jq --from-file <( + for project_id in $(jq '.[].project_id' <<< "${output}"); do + project_name=${project_name_lookup[${project_id}]} + printf 'map(if .project_id == %s then . + {"project_name": "%s"} else . end) | ' \ + "${project_id}" "${project_name}" + done + printf . + ) <<< "${output}" + exit 0 + fi + + # pretty print each result + while read -r result; do + # read properties from search result + mapfile -t data < <(jq --raw-output ".data" <<< "${result}") + { read -r project_id; read -r path; read -r startline; } < <( + jq --raw-output ".project_id, .path, .startline" <<< "${result}" + ) + project_name=${project_name_lookup[${project_id}]} + + # remove trailing newline for multiline results + if (( ${#data[@]} > 1 )) && [[ ${data[-1]} == "" ]]; then + unset "data[${#data[@]}-1]" + fi + + # output mode plain + if [[ ${output_format} == plain ]]; then + printf "%s%s%s\n" "${PURPLE}" "${project_name}/${path}" "${ALL_OFF}" + + currentline=${startline} + for line in "${data[@]}"; do + if (( line_numbers )); then + line="${DARK_GREEN}${currentline}${ALL_OFF}: ${line}" + currentline=$(( currentline + 1 )) + fi + printf "%s\n" "${line}" + done + printf "\n" + + continue + fi + + # prepend empty lines to match startline + if (( startline > 1 )); then + mapfile -t data < <( + printf '%.0s\n' $(seq 1 "$(( startline - 1 ))") + printf "%s\n" "${data[@]}" + ) + fi + + bat \ + --file-name="${project_name}/${path}" \ + --line-range "${startline}:" \ + --paging=never \ + --force-colorization \ + --style "${bat_style}" \ + --map-syntax "PKGBUILD:Bourne Again Shell (bash)" \ + --map-syntax ".SRCINFO:INI" \ + --map-syntax "*install:Bourne Again Shell (bash)" \ + --map-syntax "*sysusers*:Bourne Again Shell (bash)" \ + --map-syntax "*tmpfiles*:Bourne Again Shell (bash)" \ + --map-syntax "*.hook:INI" \ + <(printf "%s\n" "${data[@]}") + done < <(jq --compact-output '.[]' <<< "${output}") +} diff --git a/src/lib/util/git.sh b/src/lib/util/git.sh index c4af662..82e4beb 100644 --- a/src/lib/util/git.sh +++ b/src/lib/util/git.sh @@ -7,6 +7,9 @@ DEVTOOLS_INCLUDE_UTIL_GIT_SH=1 _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh + git_diff_tree() { local commit=$1 @@ -22,3 +25,10 @@ git_diff_tree() { "${commit}" \ -- "${path}" } + +git_warmup_ssh_connection() { + msg 'Establishing ssh connection to git@%s' "${GITLAB_HOST}" + if ! ssh -T "git@${GITLAB_HOST}" >/dev/null; then + die 'Failed to establish ssh connection to git@%s' "${GITLAB_HOST}" + fi +} diff --git a/src/lib/util/makepkg.sh b/src/lib/util/makepkg.sh new file mode 100644 index 0000000..22df247 --- /dev/null +++ b/src/lib/util/makepkg.sh @@ -0,0 +1,37 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_UTIL_MAKEPKG_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_UTIL_MAKEPKG_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/util/srcinfo.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/srcinfo.sh + + +set -e + +makepkg_source_package() { + if (( EUID != 0 )); then + [[ -z ${WORKDIR:-} ]] && setup_workdir + export WORKDIR DEVTOOLS_INCLUDE_COMMON_SH + fakeroot -- bash -$- -c "source '${BASH_SOURCE[0]}' && ${FUNCNAME[0]}" + return + fi + ( + export LIBMAKEPKG_LINT_PKGBUILD_SH=1 + lint_pkgbuild() { :; } + + export LIBMAKEPKG_SRCINFO_SH=1 + write_srcinfo() { print_srcinfo; } + + # explicitly instruct makepkg to not sign the source package, even when + # the BUILDENV array in makepkg.conf contains 'sign' + set +e -- -F --source --nosign + # shellcheck source=/usr/bin/makepkg + source "$(command -v makepkg)" + ) +} diff --git a/src/lib/util/pacman.sh b/src/lib/util/pacman.sh index f6c2d5f..620e1a8 100644 --- a/src/lib/util/pacman.sh +++ b/src/lib/util/pacman.sh @@ -38,13 +38,21 @@ get_pacman_repo_from_pkgbuild() { return fi + # update the pacman repo cache if it doesn't exist yet + if [[ ! -d "${_DEVTOOLS_PACMAN_CACHE_DIR}" ]]; then + update_pacman_repo_cache + fi + slock 10 "${_DEVTOOLS_PACMAN_CACHE_DIR}.lock" "Locking pacman database cache" + # query repo of passed pkgname, specify --nodeps twice to skip all dependency checks mapfile -t repos < <(pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/multilib.conf" \ --dbpath "${_DEVTOOLS_PACMAN_CACHE_DIR}" \ - -S \ + --sync \ + --nodeps \ + --nodeps \ --print \ --print-format '%n %r' \ - "${pkgnames[0]}" | grep -E "^${pkgnames[0]} " | awk '{print $2}' + "${pkgnames[0]}" 2>/dev/null | awk '$1=="'"${pkgnames[0]}"'"{print $2}' ) lock_close 10 diff --git a/src/lib/util/pkgbuild.sh b/src/lib/util/pkgbuild.sh new file mode 100644 index 0000000..ebf8e5f --- /dev/null +++ b/src/lib/util/pkgbuild.sh @@ -0,0 +1,43 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_UTIL_PKGBUILD_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_UTIL_PKGBUILD_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} + +source /usr/share/makepkg/util/message.sh + +set -e + + +# set the pkgver variable in a PKGBUILD +# assumes that the pkgbuild is sourced to detect the presence of a pkgver function +pkgbuild_set_pkgver() { + local new_pkgver=$1 + local pkgver=${pkgver} + + if [[ $(type -t pkgver) == function ]]; then + # TODO: check if die or warn, if we provide _commit _gitcommit setter maybe? + warning 'setting pkgver variable has no effect if the PKGBUILD has a pkgver() function' + fi + + if ! grep --extended-regexp --quiet --max-count=1 "^pkgver=${pkgver}$" PKGBUILD; then + die "Non-standard pkgver declaration" + fi + sed --regexp-extended "s|^(pkgver=)${pkgver}$|\1${new_pkgver}|g" --in-place PKGBUILD +} + +# set the pkgrel variable in a PKGBUILD +# assumes that the pkgbuild is sourced so pkgrel is present +pkgbuild_set_pkgrel() { + local new_pkgrel=$1 + local pkgrel=${pkgrel} + + if ! grep --extended-regexp --quiet --max-count=1 "^pkgrel=${pkgrel}$" PKGBUILD; then + die "Non-standard pkgrel declaration" + fi + sed --regexp-extended "s|^(pkgrel=)${pkgrel}$|\1${new_pkgrel}|g" --in-place PKGBUILD +} + diff --git a/src/lib/util/srcinfo.sh b/src/lib/util/srcinfo.sh new file mode 100644 index 0000000..b646dc3 --- /dev/null +++ b/src/lib/util/srcinfo.sh @@ -0,0 +1,69 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_UTIL_SRCINFO_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_UTIL_SRCINFO_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh + +source /usr/share/makepkg/util/util.sh +source /usr/share/makepkg/srcinfo.sh + +set -eo pipefail + + +print_srcinfo() { + local pkgpath=${1:-.} + local outdir pkg pid + local pids=() + + # source the PKGBUILD + # shellcheck source=contrib/makepkg/PKGBUILD.proto + . "${pkgpath}"/PKGBUILD + + # run without parallelization for single packages + if (( ${#pkgname[@]} == 1 )); then + write_srcinfo_content + return 0 + fi + + [[ -z ${WORKDIR:-} ]] && setup_workdir + outdir=$(mktemp --directory --tmpdir="${WORKDIR}" pkgctl-srcinfo.XXXXXXXXXX) + + # fork workload for each split pkgname + for pkg in "${pkgname[@]}"; do + ( + # deactivate errexit to avoid makepkg abort on grep_function + set +e + srcinfo_write_package "$pkg" > "${outdir}/${pkg}" + )& + pids+=($!) + done + + # join workload + for pid in "${pids[@]}"; do + if ! wait "${pid}"; then + return 1 + fi + done + + # collect output + srcinfo_write_global + for pkg in "${pkgname[@]}"; do + srcinfo_separate_section + cat "${outdir}/${pkg}" + done +} + +write_srcinfo_file() { + local pkgpath=${1:-.} + stat_busy 'Generating .SRCINFO' + if ! print_srcinfo "${pkgpath}" > "${pkgpath}"/.SRCINFO; then + error 'Failed to write .SRCINFO file' + return 1 + fi + stat_done +} diff --git a/src/lib/util/term.sh b/src/lib/util/term.sh new file mode 100644 index 0000000..853dccf --- /dev/null +++ b/src/lib/util/term.sh @@ -0,0 +1,182 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_UTIL_TERM_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_UTIL_TERM_SH=1 + +set -eo pipefail + + +readonly PKGCTL_TERM_SPINNER_DOTS=Dots +export PKGCTL_TERM_SPINNER_DOTS +readonly PKGCTL_TERM_SPINNER_DOTS12=Dots12 +export PKGCTL_TERM_SPINNER_DOTS12 +readonly PKGCTL_TERM_SPINNER_LINE=Line +export PKGCTL_TERM_SPINNER_LINE +readonly PKGCTL_TERM_SPINNER_SIMPLE_DOTS_SCROLLING=SimpleDotsScrolling +export PKGCTL_TERM_SPINNER_SIMPLE_DOTS_SCROLLING +readonly PKGCTL_TERM_SPINNER_TRIANGLE=Triangle +export PKGCTL_TERM_SPINNER_TRIANGLE +readonly PKGCTL_TERM_SPINNER_RANDOM=Random +export PKGCTL_TERM_SPINNER_RANDOM + +readonly PKGCTL_TERM_SPINNER_TYPES=( + "${PKGCTL_TERM_SPINNER_DOTS}" + "${PKGCTL_TERM_SPINNER_DOTS12}" + "${PKGCTL_TERM_SPINNER_LINE}" + "${PKGCTL_TERM_SPINNER_SIMPLE_DOTS_SCROLLING}" + "${PKGCTL_TERM_SPINNER_TRIANGLE}" +) +export PKGCTL_TERM_SPINNER_TYPES + + +term_cursor_hide() { + tput civis >&2 +} + +term_cursor_show() { + tput cnorm >&2 +} + +term_cursor_up() { + tput cuu1 +} + +term_carriage_return() { + tput cr +} + +term_erase_line() { + tput el +} + +term_erase_lines() { + local lines=$1 + + local cursor_up erase_line + cursor_up=$(term_cursor_up) + erase_line="$(term_carriage_return)$(term_erase_line)" + + local prefix='' + for _ in $(seq 1 "${lines}"); do + printf '%s' "${prefix}${erase_line}" + prefix="${cursor_up}" + done +} + +_pkgctl_spinner_type=${PKGCTL_TERM_SPINNER_RANDOM} +term_spinner_set_type() { + _pkgctl_spinner_type=$1 +} + +# takes a status directory that can be used to dynamically update the spinner +# by writing to the `status` file inside that directory atomically. +# replace the placeholder %spinner% with the currently configured spinner type +term_spinner_start() { + local status_dir=$1 + local parent_pid=$$ + ( + local spinner_type=${_pkgctl_spinner_type} + local spinner_offset=0 + local frame_buffer='' + local spinner status_message line + + local status_file="${status_dir}/status" + local next_file="${status_dir}/next" + local drawn_file="${status_dir}/drawn" + + # assign random spinner type + if [[ ${spinner_type} == "${PKGCTL_TERM_SPINNER_RANDOM}" ]]; then + spinner_type=${PKGCTL_TERM_SPINNER_TYPES[$((RANDOM % ${#PKGCTL_TERM_SPINNER_TYPES[@]}))]} + fi + + # select spinner based on the named type + case "${spinner_type}" in + "${PKGCTL_TERM_SPINNER_DOTS}") + spinner=("⠋" "⠙" "⠹" "⠸" "⠼" "⠴" "⠦" "⠧" "⠇" "⠏") + update_interval=0.08 + ;; + "${PKGCTL_TERM_SPINNER_DOTS12}") + spinner=("⢀⠀" "⡀⠀" "⠄⠀" "⢂⠀" "⡂⠀" "⠅⠀" "⢃⠀" "⡃⠀" "⠍⠀" "⢋⠀" "⡋⠀" "⠍⠁" "⢋⠁" "⡋⠁" "⠍⠉" "⠋⠉" "⠋⠉" "⠉⠙" "⠉⠙" "⠉⠩" "⠈⢙" "⠈⡙" "⢈⠩" "⡀⢙" "⠄⡙" "⢂⠩" "⡂⢘" "⠅⡘" "⢃⠨" "⡃⢐" "⠍⡐" "⢋⠠" "⡋⢀" "⠍⡁" "⢋⠁" "⡋⠁" "⠍⠉" "⠋⠉" "⠋⠉" "⠉⠙" "⠉⠙" "⠉⠩" "⠈⢙" "⠈⡙" "⠈⠩" "⠀⢙" "⠀⡙" "⠀⠩" "⠀⢘" "⠀⡘" "⠀⠨" "⠀⢐" "⠀⡐" "⠀⠠" "⠀⢀" "⠀⡀") + update_interval=0.08 + ;; + "${PKGCTL_TERM_SPINNER_LINE}") + spinner=("⎯" "\\" "|" "/") + update_interval=0.13 + ;; + "${PKGCTL_TERM_SPINNER_SIMPLE_DOTS_SCROLLING}") + spinner=(". " ".. " "..." " .." " ." " ") + update_interval=0.2 + ;; + "${PKGCTL_TERM_SPINNER_TRIANGLE}") + spinner=("◢" "◣" "◤" "◥") + update_interval=0.05 + ;; + esac + + # hide the cursor while spinning + term_cursor_hide + + # run the spinner as long as the parent process didn't terminate + while ps -p "${parent_pid}" &>/dev/null; do + # cache the new status template if it exists + if mv "${status_file}" "${next_file}" &>/dev/null; then + status_message="$(cat "$next_file")" + elif [[ -z "${status_message}" ]]; then + # wait until we either have a new or cached status + sleep 0.05 + fi + + # fill the frame buffer with the current status + local prefix='' + while IFS= read -r line; do + # replace spinner placeholder + line=${line//%spinner%/${spinner[spinner_offset%${#spinner[@]}]}} + + # append the current line to the frame buffer + frame_buffer+="${prefix}${line}" + prefix=$'\n' + done <<< "${status_message}" + + # print current frame buffer + echo -n "${frame_buffer}" >&2 + mv "${next_file}" "${drawn_file}" &>/dev/null ||: + + # setup next frame buffer to clear current content + frame_buffer=$(term_erase_lines "$(awk 'END {print NR}' <<< "${status_message}")") + + # advance the spinner animation offset + (( ++spinner_offset )) + + # sleep for the spinner update interval + sleep "${update_interval}" + done + )& + _pkgctl_spinner_pid=$! + disown +} + +term_spinner_stop() { + local status_dir=$1 + local frame_buffer status_file + + # kill the spinner process + if ! kill "${_pkgctl_spinner_pid}" > /dev/null 2>&1; then + return 1 + fi + unset _pkgctl_spinner_pid + + # acquire last drawn status + status_file="${status_dir}/drawn" + if [[ ! -f ${status_file} ]]; then + return 0 + fi + + # clear terminal based on last status line + frame_buffer=$(term_erase_lines "$(awk 'END {print NR}' < "${status_file}")") + echo -n "${frame_buffer}" >&2 + + # show the cursor after stopping the spinner + term_cursor_show +} diff --git a/src/lib/valid-build-install.sh b/src/lib/valid-build-install.sh new file mode 100644 index 0000000..9e98be2 --- /dev/null +++ b/src/lib/valid-build-install.sh @@ -0,0 +1,11 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later +: + +# shellcheck disable=2034 +DEVTOOLS_VALID_BUILD_INSTALL=( + none + auto + all +) diff --git a/src/lib/valid-inspect.sh b/src/lib/valid-inspect.sh new file mode 100644 index 0000000..3b5dcad --- /dev/null +++ b/src/lib/valid-inspect.sh @@ -0,0 +1,10 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +# shellcheck disable=2034 +DEVTOOLS_VALID_INSPECT_MODES=( + never + always + failure +) diff --git a/src/lib/valid-repos.sh b/src/lib/valid-repos.sh index 14f90ce..af8a552 100644 --- a/src/lib/valid-repos.sh +++ b/src/lib/valid-repos.sh @@ -4,7 +4,7 @@ : # shellcheck disable=2034 -_repos=( +DEVTOOLS_VALID_REPOS=( core core-staging core-testing extra extra-staging extra-testing multilib multilib-staging multilib-testing @@ -13,7 +13,7 @@ _repos=( ) # shellcheck disable=2034 -_build_repos=( +DEVTOOLS_VALID_BUILDREPOS=( core-staging core-testing extra extra-staging extra-testing multilib multilib-staging multilib-testing diff --git a/src/lib/valid-search.sh b/src/lib/valid-search.sh new file mode 100644 index 0000000..43799a5 --- /dev/null +++ b/src/lib/valid-search.sh @@ -0,0 +1,11 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later +: + +# shellcheck disable=2034 +valid_search_output_format=( + pretty + plain + json +) diff --git a/src/lib/valid-tags.sh b/src/lib/valid-tags.sh index 5382c5c..abca7ef 100644 --- a/src/lib/valid-tags.sh +++ b/src/lib/valid-tags.sh @@ -5,7 +5,9 @@ # shellcheck disable=2034 _arch=( + pentium4 i686 + i486 x86_64 any ) diff --git a/src/lib/version.sh b/src/lib/version.sh new file mode 100644 index 0000000..ac810ae --- /dev/null +++ b/src/lib/version.sh @@ -0,0 +1,65 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_VERSION_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_VERSION_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} + +set -e + + +pkgctl_version_usage() { + local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} + cat <<- _EOF_ + Usage: ${COMMAND} [COMMAND] [OPTIONS] + + Check and manage package versions against upstream. + + COMMANDS + check Compares local package versions against upstream + upgrade Adjust the PKGBUILD to match the latest upstream version + + OPTIONS + -h, --help Show this help text + + EXAMPLES + $ ${COMMAND} check libfoo linux libbar +_EOF_ +} + +pkgctl_version() { + if (( $# < 1 )); then + pkgctl_version_usage + exit 0 + fi + + while (( $# )); do + case $1 in + -h|--help) + pkgctl_version_usage + exit 0 + ;; + check) + _DEVTOOLS_COMMAND+=" $1" + shift + # shellcheck source=src/lib/version/check.sh + source "${_DEVTOOLS_LIBRARY_DIR}"/lib/version/check.sh + pkgctl_version_check "$@" + exit $? + ;; + upgrade) + _DEVTOOLS_COMMAND+=" $1" + shift + # shellcheck source=src/lib/version/upgrade.sh + source "${_DEVTOOLS_LIBRARY_DIR}"/lib/version/upgrade.sh + pkgctl_version_upgrade "$@" + exit $? + ;; + *) + die "invalid argument: %s" "$1" + ;; + esac + done +} diff --git a/src/lib/version/check.sh b/src/lib/version/check.sh new file mode 100644 index 0000000..35d07e2 --- /dev/null +++ b/src/lib/version/check.sh @@ -0,0 +1,340 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later +# +[[ -z ${DEVTOOLS_INCLUDE_VERSION_CHECK_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_VERSION_CHECK_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/util/term.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/term.sh + +source /usr/share/makepkg/util/message.sh + +set -eo pipefail + +readonly PKGCTL_VERSION_CHECK_EXIT_UP_TO_DATE=0 +export PKGCTL_VERSION_CHECK_EXIT_UP_TO_DATE +readonly PKGCTL_VERSION_CHECK_EXIT_OUT_OF_DATE=2 +export PKGCTL_VERSION_CHECK_EXIT_OUT_OF_DATE +readonly PKGCTL_VERSION_CHECK_EXIT_FAILURE=3 +export PKGCTL_VERSION_CHECK_EXIT_FAILURE + + +pkgctl_version_check_usage() { + local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} + cat <<- _EOF_ + Usage: ${COMMAND} [OPTIONS] [PKGBASE]... + + Compares the versions of packages in the local packaging repository against + their latest upstream versions. + + Upon execution, it generates a grouped list that provides detailed insights + into each package's status. For each package, it displays the current local + version alongside the latest version available upstream. + + Outputs a summary of up-to-date packages, out-of-date packages, and any + check failures. + + OPTIONS + -v, --verbose Display results including up-to-date versions + -h, --help Show this help text + + EXAMPLES + $ ${COMMAND} neovim vim +_EOF_ +} + +pkgctl_version_check() { + local pkgbases=() + local verbose=0 + + local path status_file path pkgbase upstream_version result + + local up_to_date=() + local out_of_date=() + local failure=() + local current_item=0 + local section_separator='' + local exit_code=${PKGCTL_VERSION_CHECK_EXIT_UP_TO_DATE} + + while (( $# )); do + case $1 in + -h|--help) + pkgctl_version_check_usage + exit 0 + ;; + -v|--verbose) + verbose=1 + shift + ;; + --) + shift + break + ;; + -*) + die "invalid argument: %s" "$1" + ;; + *) + pkgbases=("$@") + break + ;; + esac + done + + if ! command -v nvchecker &>/dev/null; then + die "The \"$_DEVTOOLS_COMMAND\" command requires 'nvchecker'" + fi + + # Check if used without pkgbases in a packaging directory + if (( ${#pkgbases[@]} == 0 )); then + if [[ -f PKGBUILD ]]; then + pkgbases=(".") + else + pkgctl_version_check_usage + exit 1 + fi + fi + + # enable verbose mode when we only have a single item to check + if (( ${#pkgbases[@]} == 1 )); then + verbose=1 + fi + + # start a terminal spinner as checking versions takes time + status_dir=$(mktemp --tmpdir="${WORKDIR}" --directory pkgctl-version-check-spinner.XXXXXXXXXX) + term_spinner_start "${status_dir}" + + for path in "${pkgbases[@]}"; do + pushd "${path}" >/dev/null + + if [[ ! -f "PKGBUILD" ]]; then + die "No PKGBUILD found for ${path}" + fi + + # update the current terminal spinner status + (( ++current_item )) + pkgctl_version_check_spinner \ + "${status_dir}" \ + "${#up_to_date[@]}" \ + "${#out_of_date[@]}" \ + "${#failure[@]}" \ + "${current_item}" \ + "${#pkgbases[@]}" + + # reset common PKGBUILD variables + unset pkgbase pkgname arch source pkgver pkgrel validpgpkeys + # shellcheck source=contrib/makepkg/PKGBUILD.proto + . ./PKGBUILD + pkgbase=${pkgbase:-$pkgname} + + if ! result=$(get_upstream_version); then + result="${BOLD}${pkgbase}${ALL_OFF}: ${result}" + failure+=("${result}") + popd >/dev/null + continue + fi + upstream_version=${result} + + if ! result=$(vercmp "${upstream_version}" "${pkgver}"); then + result="${BOLD}${pkgbase}${ALL_OFF}: failed to compare version ${upstream_version} against ${pkgver}" + failure+=("${result}") + popd >/dev/null + continue + fi + + if (( result == 0 )); then + result="${BOLD}${pkgbase}${ALL_OFF}: current version ${PURPLE}${pkgver}${ALL_OFF} is latest" + up_to_date+=("${result}") + elif (( result < 0 )); then + result="${BOLD}${pkgbase}${ALL_OFF}: current version ${PURPLE}${pkgver}${ALL_OFF} is newer than ${DARK_GREEN}${upstream_version}${ALL_OFF}" + up_to_date+=("${result}") + elif (( result > 0 )); then + result="${BOLD}${pkgbase}${ALL_OFF}: upgrade from version ${PURPLE}${pkgver}${ALL_OFF} to ${DARK_GREEN}${upstream_version}${ALL_OFF}" + out_of_date+=("${result}") + fi + + popd >/dev/null + done + + # stop the terminal spinner after all checks + term_spinner_stop "${status_dir}" + + if (( verbose )) && (( ${#up_to_date[@]} > 0 )); then + printf "%sUp-to-date%s\n" "${section_separator}${BOLD}${UNDERLINE}" "${ALL_OFF}" + section_separator=$'\n' + for result in "${up_to_date[@]}"; do + msg_success " ${result}" + done + fi + + if (( ${#failure[@]} > 0 )); then + exit_code=${PKGCTL_VERSION_CHECK_EXIT_FAILURE} + printf "%sFailure%s\n" "${section_separator}${BOLD}${UNDERLINE}" "${ALL_OFF}" + section_separator=$'\n' + for result in "${failure[@]}"; do + msg_error " ${result}" + done + fi + + if (( ${#out_of_date[@]} > 0 )); then + exit_code=${PKGCTL_VERSION_CHECK_EXIT_OUT_OF_DATE} + printf "%sOut-of-date%s\n" "${section_separator}${BOLD}${UNDERLINE}" "${ALL_OFF}" + section_separator=$'\n' + for result in "${out_of_date[@]}"; do + msg_warn " ${result}" + done + fi + + # Show summary when processing multiple packages + if (( ${#pkgbases[@]} > 1 )); then + printf '%s' "${section_separator}" + pkgctl_version_check_summary \ + "${#up_to_date[@]}" \ + "${#out_of_date[@]}" \ + "${#failure[@]}" + fi + + # return status based on results + return "${exit_code}" +} + +get_upstream_version() { + local config=.nvchecker.toml + local output errors upstream_version + local output + local opts=() + local keyfile="${XDG_CONFIG_HOME:-${HOME}/.config}/nvchecker/keyfile.toml" + + # check nvchecker config file + if ! errors=$(nvchecker_check_config "${config}"); then + printf "%s" "${errors}" + return 1 + fi + + # populate keyfile to nvchecker opts + if [[ -f ${keyfile} ]]; then + opts+=(--keyfile "${keyfile}") + fi + + if ! output=$(nvchecker --file "${config}" --logger json "${opts[@]}" 2>&1 | \ + jq --raw-output 'select(.level != "debug")'); then + printf "failed to run nvchecker: %s" "${output}" + return 1 + fi + + if ! errors=$(nvchecker_check_error "${output}"); then + printf "%s" "${errors}" + return 1 + fi + + if ! upstream_version=$(jq --raw-output --exit-status '.version' <<< "${output}"); then + printf "failed to select version from result" + return 1 + fi + + printf "%s" "${upstream_version}" + return 0 +} + +nvchecker_check_config() { + local config=$1 + + local restricted_properties=(keyfile httptoken token) + local property + + # check if the config file exists + if [[ ! -f ${config} ]]; then + printf "configuration file not found: %s" "${config}" + return 1 + fi + + # check if config contains any restricted properties like secrets + for property in "${restricted_properties[@]}"; do + if grep --max-count=1 --quiet "^${property}" < "${config}"; then + printf "restricted property in %s: %s" "${config}" "${property}" + return 1 + fi + done + + # check if the config contains a pkgbase section + if [[ -n ${pkgbase} ]] && ! grep --max-count=1 --extended-regexp --quiet "^\\[\"?${pkgbase}\"?\\]" < "${config}"; then + printf "missing pkgbase section in %s: %s" "${config}" "${pkgbase}" + return 1 + fi + + # check if the config contains any section other than pkgbase + if [[ -n ${pkgbase} ]] && property=$(grep --max-count=1 --perl-regexp "^\\[(?!\"?${pkgbase}\"?\\]).+\\]" < "${config}"); then + printf "non-pkgbase section not supported in %s: %s" "${config}" "${property}" + return 1 + fi +} + +nvchecker_check_error() { + local result=$1 + local errors + + if ! errors=$(jq --raw-output --exit-status \ + 'select(.level == "error") | "\(.event)" + if .error then ": \(.error)" else "" end' \ + <<< "${result}"); then + return 0 + fi + + mapfile -t errors <<< "${errors}" + printf "%s\n" "${errors[@]}" + return 1 +} + +pkgctl_version_check_summary() { + local up_to_date_count=$1 + local out_of_date_count=$2 + local failure_count=$3 + + # print nothing if all stats are zero + if (( up_to_date_count == 0 )) && \ + (( out_of_date_count == 0 )) && \ + (( failure_count == 0 )); then + return 0 + fi + + # print summary for all none zero stats + printf "%sSummary%s\n" "${BOLD}${UNDERLINE}" "${ALL_OFF}" + if (( up_to_date_count > 0 )); then + msg_success " Up-to-date: ${BOLD}${up_to_date_count}${ALL_OFF}" 2>&1 + fi + if (( failure_count > 0 )); then + msg_error " Failure: ${BOLD}${failure_count}${ALL_OFF}" 2>&1 + fi + if (( out_of_date_count > 0 )); then + msg_warn " Out-of-date: ${BOLD}${out_of_date_count}${ALL_OFF}" 2>&1 + fi +} + +pkgctl_version_check_spinner() { + local status_dir=$1 + local up_to_date_count=$2 + local out_of_date_count=$3 + local failure_count=$4 + local current=$5 + local total=$6 + + local percentage=$(( 100 * current / total )) + local tmp_file="${status_dir}/tmp" + local status_file="${status_dir}/status" + + # print the current summary + pkgctl_version_check_summary \ + "${up_to_date_count}" \ + "${out_of_date_count}" \ + "${failure_count}" > "${tmp_file}" + + # print the progress status + printf "📡 Checking: %s/%s [%s] %%spinner%%" \ + "${BOLD}${current}" "${total}" "${percentage}%${ALL_OFF}" \ + >> "${tmp_file}" + + # swap the status file + mv "${tmp_file}" "${status_file}" +} diff --git a/src/lib/version/upgrade.sh b/src/lib/version/upgrade.sh new file mode 100644 index 0000000..e217532 --- /dev/null +++ b/src/lib/version/upgrade.sh @@ -0,0 +1,248 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later +# +[[ -z ${DEVTOOLS_INCLUDE_VERSION_UPGRADE_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_VERSION_UPGRADE_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/version/check.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/version/check.sh +# shellcheck source=src/lib/util/pkgbuild.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/pkgbuild.sh +# shellcheck source=src/lib/util/term.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/term.sh + +source /usr/share/makepkg/util/message.sh + +set -e + +pkgctl_version_upgrade_usage() { + local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} + cat <<- _EOF_ + Usage: ${COMMAND} [OPTIONS] [PKGBASE]... + + Streamlines the process of keeping PKGBUILD files up-to-date with the latest + upstream versions. + + Upon execution, it automatically adjusts the PKGBUILD file, ensuring that the + pkgver field is set to match the latest version available from the upstream + source. In addition to updating the pkgver, this command also resets the pkgrel + to 1. + + Outputs a summary of upgraded packages, up-to-date packages, and any check + failures. + + OPTIONS + -v, --verbose Display results including up-to-date versions + -h, --help Show this help text + + EXAMPLES + $ ${COMMAND} neovim vim +_EOF_ +} + +pkgctl_version_upgrade() { + local path upstream_version result + local pkgbases=() + local verbose=0 + local exit_code=0 + local current_item=0 + + while (( $# )); do + case $1 in + -h|--help) + pkgctl_version_upgrade_usage + exit 0 + ;; + -v|--verbose) + verbose=1 + shift + ;; + --) + shift + break + ;; + -*) + die "invalid argument: %s" "$1" + ;; + *) + pkgbases=("$@") + break + ;; + esac + done + + if ! command -v nvchecker &>/dev/null; then + die "The \"$_DEVTOOLS_COMMAND\" command requires 'nvchecker'" + fi + + # Check if used without pkgbases in a packaging directory + if (( ${#pkgbases[@]} == 0 )); then + if [[ -f PKGBUILD ]]; then + pkgbases=(".") + else + pkgctl_version_upgrade_usage + exit 1 + fi + fi + + # enable verbose mode when we only have a single item to check + if (( ${#pkgbases[@]} == 1 )); then + verbose=1 + fi + + # start a terminal spinner as checking versions takes time + status_dir=$(mktemp --tmpdir="${WORKDIR}" --directory pkgctl-version-check-spinner.XXXXXXXXXX) + term_spinner_start "${status_dir}" + + for path in "${pkgbases[@]}"; do + pushd "${path}" >/dev/null + + if [[ ! -f "PKGBUILD" ]]; then + die "No PKGBUILD found for ${path}" + fi + + # update the current terminal spinner status + (( ++current_item )) + pkgctl_version_upgrade_spinner \ + "${status_dir}" \ + "${#up_to_date[@]}" \ + "${#out_of_date[@]}" \ + "${#failure[@]}" \ + "${current_item}" \ + "${#pkgbases[@]}" + + # reset common PKGBUILD variables + unset pkgbase pkgname arch source pkgver pkgrel validpgpkeys + # shellcheck source=contrib/makepkg/PKGBUILD.proto + . ./PKGBUILD + pkgbase=${pkgbase:-$pkgname} + + if ! result=$(get_upstream_version); then + result="${BOLD}${pkgbase}${ALL_OFF}: ${result}" + failure+=("${result}") + popd >/dev/null + continue + fi + upstream_version=${result} + + if ! result=$(vercmp "${upstream_version}" "${pkgver}"); then + result="${BOLD}${pkgbase}${ALL_OFF}: failed to compare version ${upstream_version} against ${pkgver}" + failure+=("${result}") + popd >/dev/null + continue + fi + + if (( result == 0 )); then + result="${BOLD}${pkgbase}${ALL_OFF}: current version ${PURPLE}${pkgver}${ALL_OFF} is latest" + up_to_date+=("${result}") + elif (( result < 0 )); then + result="${BOLD}${pkgbase}${ALL_OFF}: current version ${PURPLE}${pkgver}${ALL_OFF} is newer than ${DARK_GREEN}${upstream_version}${ALL_OFF}" + up_to_date+=("${result}") + elif (( result > 0 )); then + result="${BOLD}${pkgbase}${ALL_OFF}: upgraded from version ${PURPLE}${pkgver}${ALL_OFF} to ${DARK_GREEN}${upstream_version}${ALL_OFF}" + out_of_date+=("${result}") + + # change the PKGBUILD + pkgbuild_set_pkgver "${upstream_version}" + pkgbuild_set_pkgrel 1 + fi + + popd >/dev/null + done + + # stop the terminal spinner after all checks + term_spinner_stop "${status_dir}" + + if (( verbose )) && (( ${#up_to_date[@]} > 0 )); then + printf "%sUp-to-date%s\n" "${section_separator}${BOLD}${UNDERLINE}" "${ALL_OFF}" + section_separator=$'\n' + for result in "${up_to_date[@]}"; do + msg_success " ${result}" + done + fi + + if (( ${#failure[@]} > 0 )); then + exit_code=1 + printf "%sFailure%s\n" "${section_separator}${BOLD}${UNDERLINE}" "${ALL_OFF}" + section_separator=$'\n' + for result in "${failure[@]}"; do + msg_error " ${result}" + done + fi + + if (( ${#out_of_date[@]} > 0 )); then + printf "%sUpgraded%s\n" "${section_separator}${BOLD}${UNDERLINE}" "${ALL_OFF}" + section_separator=$'\n' + for result in "${out_of_date[@]}"; do + msg_warn " ${result}" + done + fi + + # Show summary when processing multiple packages + if (( ${#pkgbases[@]} > 1 )); then + printf '%s' "${section_separator}" + pkgctl_version_upgrade_summary \ + "${#up_to_date[@]}" \ + "${#out_of_date[@]}" \ + "${#failure[@]}" + fi + + # return status based on results + return "${exit_code}" +} + +pkgctl_version_upgrade_summary() { + local up_to_date_count=$1 + local out_of_date_count=$2 + local failure_count=$3 + + # print nothing if all stats are zero + if (( up_to_date_count == 0 )) && \ + (( out_of_date_count == 0 )) && \ + (( failure_count == 0 )); then + return 0 + fi + + # print summary for all none zero stats + printf "%sSummary%s\n" "${BOLD}${UNDERLINE}" "${ALL_OFF}" + if (( up_to_date_count > 0 )); then + msg_success " Up-to-date: ${BOLD}${up_to_date_count}${ALL_OFF}" 2>&1 + fi + if (( failure_count > 0 )); then + msg_error " Failure: ${BOLD}${failure_count}${ALL_OFF}" 2>&1 + fi + if (( out_of_date_count > 0 )); then + msg_warn " Upgraded: ${BOLD}${out_of_date_count}${ALL_OFF}" 2>&1 + fi +} + +pkgctl_version_upgrade_spinner() { + local status_dir=$1 + local up_to_date_count=$2 + local out_of_date_count=$3 + local failure_count=$4 + local current=$5 + local total=$6 + + local percentage=$(( 100 * current / total )) + local tmp_file="${status_dir}/tmp" + local status_file="${status_dir}/status" + + # print the current summary + pkgctl_version_upgrade_summary \ + "${up_to_date_count}" \ + "${out_of_date_count}" \ + "${failure_count}" > "${tmp_file}" + + # print the progress status + printf "📡 Upgrading: %s/%s [%s] %%spinner%%" \ + "${BOLD}${current}" "${total}" "${percentage}%${ALL_OFF}" \ + >> "${tmp_file}" + + # swap the status file + mv "${tmp_file}" "${status_file}" +} diff --git a/src/lib/version/version.sh b/src/lib/version/version.sh deleted file mode 100644 index d00a460..0000000 --- a/src/lib/version/version.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/hint/bash -# -# SPDX-License-Identifier: GPL-3.0-or-later - -[[ -z ${DEVTOOLS_INCLUDE_VERSION_SH:-} ]] || return 0 -DEVTOOLS_INCLUDE_VERSION_SH=1 - -_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} - -source /usr/share/makepkg/util/message.sh - -set -e - - -pkgctl_version_usage() { - local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} - cat <<- _EOF_ - Usage: ${COMMAND} [OPTIONS] - - Shows the current version information of pkgctl - - OPTIONS - -h, --help Show this help text -_EOF_ -} - -pkgctl_version_print() { - cat <<- _EOF_ - pkgctl @buildtoolver@ -_EOF_ -} - -pkgctl_version() { - while (( $# )); do - case $1 in - -h|--help) - pkgctl_version_usage - exit 0 - ;; - *) - die "invalid argument: %s" "$1" - ;; - esac - done - - pkgctl_version_print -} diff --git a/src/makechrootpkg.in b/src/makechrootpkg.in index c2e3daa..70eba3c 100644 --- a/src/makechrootpkg.in +++ b/src/makechrootpkg.in @@ -8,9 +8,12 @@ _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh # shellcheck source=src/lib/archroot.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/archroot.sh +# shellcheck source=src/lib/valid-inspect.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-inspect.sh source /usr/share/makepkg/util/config.sh +source /usr/share/makepkg/util/util.sh shopt -s nullglob @@ -31,6 +34,8 @@ run_checkpkg=0 temp_chroot=0 tmp_opts="nosuid,nodev,size=50%,nr_inodes=2m" +inspect=never + bindmounts_ro=() bindmounts_rw=() @@ -76,6 +81,7 @@ usage() { echo '-C Run checkpkg on the package' echo '-T Build in a temporary directory' echo '-U Run makepkg as a specified user' + echo '-x <when> Inspect chroot after build (never, always, failure)' exit 1 } @@ -289,7 +295,7 @@ move_products() { } # }}} -while getopts 'hcur:I:l:nCTD:d:U:' arg; do +while getopts 'hcur:I:l:nCTD:d:U:x:' arg; do case "$arg" in c) clean_first=1 ;; D) bindmounts_ro+=("--bind-ro=$OPTARG") ;; @@ -302,6 +308,7 @@ while getopts 'hcur:I:l:nCTD:d:U:' arg; do C) run_checkpkg=1 ;; T) temp_chroot=1; copy+="-$$" ;; U) makepkg_user="$OPTARG" ;; + x) inspect="$OPTARG" ;; h|*) usage ;; esac done @@ -323,6 +330,10 @@ else copydir="$chrootdir/$copy" fi +if ! in_array "${inspect}" "${DEVTOOLS_VALID_INSPECT_MODES[@]}"; then + die "Invalid inspect mode: %s" "${inspect}" +fi + # Pass all arguments after -- right to makepkg makepkg_args+=("${@:$OPTIND}") @@ -378,11 +389,16 @@ download_sources prepare_chroot +nspawn_build_args=( + --bind="${PWD//:/\\:}:/startdir" + --bind="${SRCDEST//:/\\:}:/srcdest" + --tmpfs="/tmp:${tmp_opts}" + "${bindmounts_ro[@]}" + "${bindmounts_rw[@]}" +) + if arch-nspawn "$copydir" \ - --bind="${PWD//:/\\:}:/startdir" \ - --bind="${SRCDEST//:/\\:}:/srcdest" \ - --tmpfs="/tmp:${tmp_opts}" \ - "${bindmounts_ro[@]}" "${bindmounts_rw[@]}" \ + "${nspawn_build_args[@]}" \ /chrootbuild "${makepkg_args[@]}" then mapfile -t pkgnames < <(sudo -u "$makepkg_user" bash -c 'source PKGBUILD; printf "%s\n" "${pkgname[@]}"') @@ -392,6 +408,18 @@ else move_logfiles fi +if [[ $inspect == always ]] || ( [[ $inspect == failure ]] && (( ret != 0 )) ); then + if (( ret == 0 )); then + msg "Build succeeded, inspecting %s" "$copydir" + else + error "Build failed, inspecting %s" "$copydir" + fi + arch-nspawn "$copydir" \ + "${nspawn_build_args[@]}" \ + --user=builduser \ + --chdir=/build +fi + (( temp_chroot )) && delete_chroot "$copydir" "$copy" if (( ret != 0 )); then diff --git a/src/makerepropkg.in b/src/makerepropkg.in index 398d4af..a31f8d5 100644 --- a/src/makerepropkg.in +++ b/src/makerepropkg.in @@ -22,6 +22,7 @@ declare -a buildenv buildopts installed installpkgs archiveurl='https://archive.archlinux.org/packages' buildroot=/var/lib/archbuild/reproducible diffoscope=0 +makepkg_options=() chroot=$USER [[ -n ${SUDO_USER:-} ]] && chroot=$SUDO_USER @@ -116,6 +117,7 @@ For more details see https://reproducible-builds.org/ OPTIONS -d Run diffoscope if the package is unreproducible + -n Do not run the check() function in the PKGBUILD -c <dir> Set pacman cache -M <file> Location of a makepkg config file -l <chroot> The directory name to use as the chroot namespace @@ -128,9 +130,10 @@ __EOF__ # save all args for check_root orig_args=("$@") -while getopts 'dM:c:l:h' arg; do +while getopts 'dnM:c:l:h' arg; do case "$arg" in d) diffoscope=1 ;; + n) makepkg_options+=(--nocheck) ;; M) archroot_args+=(-M "$OPTARG") ;; c) cache_dirs+=("$OPTARG") ;; l) chroot="$OPTARG" ;; @@ -254,7 +257,7 @@ install -d -o "${SUDO_UID:-$UID}" -g "$(id -g "${SUDO_UID:-$UID}")" "${namespace arch-nspawn "${namespace}/build" \ --bind="${PWD}:/startdir" \ --bind="${SRCDEST}:/srcdest" \ - /chrootbuild -C --noconfirm --log --holdver --skipinteg + /chrootbuild -C --noconfirm --log --holdver --skipinteg "${makepkg_options[@]}" ret=$? if (( ${ret} == 0 )); then diff --git a/src/offload-build.in b/src/offload-build.in index 027bad3..7a8c1fd 100644 --- a/src/offload-build.in +++ b/src/offload-build.in @@ -6,15 +6,24 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/util/makepkg.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/makepkg.sh + source /usr/share/makepkg/util/config.sh +# Deprecation warning +if [[ -z $_DEVTOOLS_COMMAND ]]; then + warning "${0##*/} is deprecated and will be removed. Use 'pkgctl build --offload' instead" +fi # global defaults suitable for use by Arch staff repo=extra arch=x86_64 server=build.archlinux.org - -die() { printf "error: $1\n" "${@:2}"; exit 1; } +rsyncopts=(-e ssh -c -h -L --progress --partial -y) usage() { cat <<- _EOF_ @@ -79,7 +88,7 @@ load_makepkg_config # transferred, including local sources, install scripts, and changelogs. export TEMPDIR=$(mktemp -d --tmpdir offload-build.XXXXXXXXXX) export SRCPKGDEST=${TEMPDIR} -makepkg --source || die "unable to make source package" +makepkg_source_package || die "unable to make source package" # Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else # but an empty src dir is created in PWD. Remove once fixed in makepkg. @@ -91,6 +100,7 @@ mapfile -t files < <( # shellcheck disable=SC2145 cat "$SRCPKGDEST"/*"$SRCEXT" | ssh $server ' + export TERM="'"${TERM}"'" temp="${XDG_CACHE_HOME:-$HOME/.cache}/offload-build" && mkdir -p "$temp" && temp=$(mktemp -d -p "$temp") && @@ -106,14 +116,16 @@ mapfile -t files < <( if [[ -f /usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf ]]; then makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf" fi && - makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist && + while read -r file; do + [[ -f "${file}" ]] && printf "%s\n" "${file}" ||: + done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist) && printf "%s\n" "${temp}/PKGBUILD" ') if (( ${#files[@]} )); then printf '%s\n' '' '-> copying files...' - scp "${files[@]/#/$server:}" "${TEMPDIR}/" + rsync "${rsyncopts[@]}" "${files[@]/#/$server:}" "${TEMPDIR}/" || die mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/" mv "${TEMPDIR}/PKGBUILD" "${PWD}/" else diff --git a/src/pkgctl.in b/src/pkgctl.in index 1797b32..50c14b5 100644 --- a/src/pkgctl.in +++ b/src/pkgctl.in @@ -19,13 +19,15 @@ usage() { Unified command-line frontend for devtools. COMMANDS + aur Interact with the Arch User Repository auth Authenticate with services like GitLab build Build packages inside a clean chroot - db Pacman database modification for packge update, move etc + db Pacman database modification for package update, move etc diff Compare package files using different modes release Release step to commit, tag and upload build artifacts repo Manage Git packaging repositories and their configuration - version Show pkgctl version information + search Search for an expression across the GitLab packaging group + version Check and manage package versions against upstream OPTIONS -h, --help Show this help text @@ -37,8 +39,16 @@ if (( $# < 1 )); then exit 1 fi +pkgctl_version_print() { + cat <<- _EOF_ + pkgctl @buildtoolver@ +_EOF_ +} + export _DEVTOOLS_COMMAND='pkgctl' +setup_workdir + load_devtools_config # command checking @@ -48,6 +58,14 @@ while (( $# )); do usage exit 0 ;; + aur) + _DEVTOOLS_COMMAND+=" $1" + shift + # shellcheck source=src/lib/aur.sh + source "${_DEVTOOLS_LIBRARY_DIR}"/lib/aur.sh + pkgctl_aur "$@" + exit 0 + ;; build) _DEVTOOLS_COMMAND+=" $1" shift @@ -94,14 +112,28 @@ while (( $# )); do pkgctl_release "$@" exit 0 ;; - version|--version|-V) + search) _DEVTOOLS_COMMAND+=" $1" shift - # shellcheck source=src/lib/version/version.sh - source "${_DEVTOOLS_LIBRARY_DIR}"/lib/version/version.sh + # shellcheck source=src/lib/release.sh + source "${_DEVTOOLS_LIBRARY_DIR}"/lib/search.sh + pkgctl_search "$@" + exit 0 + ;; + version) + _DEVTOOLS_COMMAND+=" $1" + shift + # shellcheck source=src/lib/version.sh + source "${_DEVTOOLS_LIBRARY_DIR}"/lib/version.sh pkgctl_version "$@" exit 0 ;; + --version|-V) + _DEVTOOLS_COMMAND+=" $1" + shift + pkgctl_version_print + exit 0 + ;; *) die "invalid command: %s" "$1" ;; diff --git a/src/sogrep.in b/src/sogrep.in index 0ee05cc..9f51e53 100644 --- a/src/sogrep.in +++ b/src/sogrep.in @@ -31,7 +31,7 @@ recache() { (( VERBOSE )) && verbosity=--progress-bar - for repo in "${_repos[@]}"; do + for repo in "${DEVTOOLS_VALID_REPOS[@]}"; do if [[ -n "$SOLINKS_MIRROR" ]]; then mirror="$SOLINKS_MIRROR" elif ! mirror="$(set -o pipefail; pacman-conf --repo "$repo" Server 2>/dev/null | head -n1)"; then @@ -72,7 +72,7 @@ is_outdated_cache() { # links databases are generated at about the same time every day; we should # attempt to check for new database files if any of them are over a day old - for repo in "${_repos[@]}"; do + for repo in "${DEVTOOLS_VALID_REPOS[@]}"; do for arch in "${arches[@]}"; do local dbpath=${SOCACHE_DIR}/${arch}/${repo}.links.tar.gz if [[ ! -f ${dbpath} ]] || [[ $(find "${dbpath}" -mtime +0) ]]; then @@ -85,10 +85,10 @@ is_outdated_cache() { } search() { - local repo=$1 arch lib=$2 srepos=("${_repos[@]}") + local repo=$1 arch lib=$2 srepos=("${DEVTOOLS_VALID_REPOS[@]}") if [[ $repo != all ]]; then - if ! in_array "${repo}" "${_repos[@]}"; then + if ! in_array "${repo}" "${DEVTOOLS_VALID_REPOS[@]}"; then echo "${BASH_SOURCE[0]##*/}: unrecognized repo '$repo'" echo "Try '${BASH_SOURCE[0]##*/} --help' for more information." exit 1 |