Send patches - preferably formatted by git format-patch - to patches at archlinux32 dot org.
summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorLevente Polyak <anthraxx@archlinux.org>2023-12-03 19:46:09 +0100
committerLevente Polyak <anthraxx@archlinux.org>2023-12-03 21:51:59 +0100
commitf2cafa3cb0941be8235025434620adbf5849a432 (patch)
tree908ea4208915dfbc0b3af59e16cb2dcc21de009b /src
parentc356995dc19b3dfe944920a5864f9d3ffd5f9b6f (diff)
feat(clone): speedup maintainer and universe clone query
There is a single endpoint now to list all pkgbases and their current maintainers. Use this endpoint for speeding up the clone of all packages of a maintainer by only issuing a single API call. Component: pkgctl repo clone Signed-off-by: Levente Polyak <anthraxx@archlinux.org>
Diffstat (limited to 'src')
-rw-r--r--src/lib/api/archweb.sh57
-rw-r--r--src/lib/common.sh15
-rw-r--r--src/lib/repo/clone.sh30
-rw-r--r--src/pkgctl.in2
4 files changed, 83 insertions, 21 deletions
diff --git a/src/lib/api/archweb.sh b/src/lib/api/archweb.sh
new file mode 100644
index 0000000..34c7a9c
--- /dev/null
+++ b/src/lib/api/archweb.sh
@@ -0,0 +1,57 @@
+#!/hint/bash
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+[[ -z ${DEVTOOLS_INCLUDE_API_ARCHWEB_SH:-} ]] || return 0
+DEVTOOLS_INCLUDE_API_ARCHWEB_SH=1
+
+_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
+# shellcheck source=src/lib/common.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
+
+set -e
+set -o pipefail
+
+
+archweb_query_all_packages() {
+ [[ -z ${WORKDIR:-} ]] && setup_workdir
+
+ stat_busy "Query all released packages"
+ mapfile -t pkgbases < <(
+ curl --location --show-error --no-progress-meter --fail --retry 3 --retry-delay 3 \
+ "${PKGBASE_MAINTAINER_URL}" 2> "${WORKDIR}/error" \
+ | jq --raw-output --exit-status 'keys[]' 2> "${WORKDIR}/error"
+ )
+ if ! wait $!; then
+ stat_failed
+ print_workdir_error
+ return 1
+ fi
+ stat_done
+
+ printf "%s\n" "${pkgbases[@]}"
+ return 0
+}
+
+
+archweb_query_maintainer_packages() {
+ local maintainer=$1
+
+ [[ -z ${WORKDIR:-} ]] && setup_workdir
+
+ stat_busy "Query maintainer packages"
+ mapfile -t pkgbases < <(
+ curl --location --show-error --no-progress-meter --fail --retry 3 --retry-delay 3 \
+ "${PKGBASE_MAINTAINER_URL}" 2> "${WORKDIR}/error" \
+ | jq --raw-output --exit-status '. as $parent | keys[] | select(. as $key | $parent[$key] | index("'"${maintainer}"'"))' 2> "${WORKDIR}/error"
+ )
+ if ! wait $!; then
+ stat_failed
+ print_workdir_error
+ return 1
+ fi
+ stat_done
+
+ printf "%s\n" "${pkgbases[@]}"
+ return 0
+}
diff --git a/src/lib/common.sh b/src/lib/common.sh
index 3d1ee56..7589120 100644
--- a/src/lib/common.sh
+++ b/src/lib/common.sh
@@ -28,6 +28,7 @@ export GIT_PACKAGING_NAMESPACE_ID=11323
export GIT_PACKAGING_URL_SSH="git@${GITLAB_HOST}:${GIT_PACKAGING_NAMESPACE}"
export GIT_PACKAGING_URL_HTTPS="https://${GITLAB_HOST}/${GIT_PACKAGING_NAMESPACE}"
export PACKAGING_REPO_RELEASE_HOST=repos.archlinux.org
+export PKGBASE_MAINTAINER_URL=https://archlinux.org/packages/pkgbase-maintainer
# check if messages are to be printed using color
if [[ -t 2 && "$TERM" != dumb ]] || [[ ${DEVTOOLS_COLOR} == always ]]; then
@@ -53,6 +54,11 @@ stat_done() {
printf "${BOLD}done${ALL_OFF}\n" >&2
}
+stat_failed() {
+ # shellcheck disable=2059
+ printf "${BOLD}${RED}failed${ALL_OFF}\n" >&2
+}
+
msg_success() {
local msg=$1
local padding
@@ -77,6 +83,15 @@ msg_warn() {
printf "%s %s\n" "${padding}${YELLOW}!${ALL_OFF}" "${msg}" >&2
}
+print_workdir_error() {
+ if [[ ! -f "${WORKDIR}"/error ]]; then
+ return
+ fi
+ while read -r LINE; do
+ error '%s' "${LINE}"
+ done < "${WORKDIR}/error"
+}
+
_setup_workdir=false
setup_workdir() {
[[ -z ${WORKDIR:-} ]] && WORKDIR=$(mktemp -d --tmpdir "${0##*/}.XXXXXXXXXX")
diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh
index fb927d2..ef6a0e2 100644
--- a/src/lib/repo/clone.sh
+++ b/src/lib/repo/clone.sh
@@ -8,6 +8,8 @@ DEVTOOLS_INCLUDE_REPO_CLONE_SH=1
_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
# shellcheck source=src/lib/common.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
+# shellcheck source=src/lib/api/archweb.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/archweb.sh
# shellcheck source=src/lib/api/gitlab.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/gitlab.sh
# shellcheck source=src/lib/repo/configure.sh
@@ -18,6 +20,7 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh
source /usr/share/makepkg/util/message.sh
set -e
+set -o pipefail
pkgctl_repo_clone_usage() {
@@ -137,33 +140,18 @@ pkgctl_repo_clone() {
# Query packages of a maintainer
if [[ -n ${MAINTAINER} ]]; then
- stat_busy "Query packages"
- max_pages=$(curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&maintainer=${MAINTAINER}" | jq -r '.num_pages')
- if [[ ! ${max_pages} =~ ([[:digit:]]) ]]; then
- stat_done
- warning "found no packages for maintainer ${MAINTAINER}"
- exit 0
+ mapfile -t pkgbases < <(archweb_query_maintainer_packages "${MAINTAINER}")
+ if ! wait $!; then
+ die "Failed to query maintainer packages"
fi
- mapfile -t pkgbases < <(for page in $(seq "${max_pages}"); do
- curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&maintainer=${MAINTAINER}&page=${page}" | jq -r '.results[].pkgbase'
- stat_progress
- done | sort --unique)
- stat_done
fi
# Query all released packages
if (( CLONE_ALL )); then
- stat_busy "Query all released packages"
- max_pages=$(curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name" | jq -r '.num_pages')
- if [[ ! ${max_pages} =~ ([[:digit:]]) ]]; then
- stat_done
- die "failed to query packages"
+ mapfile -t pkgbases < <(archweb_query_all_packages)
+ if ! wait $!; then
+ die "Failed to query all packages"
fi
- mapfile -t pkgbases < <(for page in $(seq "${max_pages}"); do
- curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&page=${page}" | jq -r '.results[].pkgbase'
- stat_progress
- done | sort --unique)
- stat_done
fi
# parallelization
diff --git a/src/pkgctl.in b/src/pkgctl.in
index 07242a3..ad215ac 100644
--- a/src/pkgctl.in
+++ b/src/pkgctl.in
@@ -39,6 +39,8 @@ fi
export _DEVTOOLS_COMMAND='pkgctl'
+setup_workdir
+
load_devtools_config
# command checking