Send patches - preferably formatted by git format-patch - to patches at archlinux32 dot org.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLevente Polyak <anthraxx@archlinux.org>2023-02-03 00:58:59 +0100
committerLevente Polyak <anthraxx@archlinux.org>2023-05-20 16:12:45 +0200
commit6ce666a1669235749c17d5c44d8a24dea4a135da (patch)
tree9a69904029bea07f5f21b965b8046c727d54cefc
parentbf61b8472a2e50a740229f007513dd663b9dddda (diff)
feature(parallel): run up to N jobs in parallel for repo clone/configure
Run up to N jobs in parallel. By default the number of jobs is equal to the number of available processing units. For sequential processing this option needs to be passed with 1. Signed-off-by: Levente Polyak <anthraxx@archlinux.org>
-rw-r--r--contrib/completion/bash/devtools.in6
-rw-r--r--contrib/completion/zsh/_devtools.in2
-rw-r--r--doc/man/pkgctl-repo-clone.1.asciidoc5
-rw-r--r--doc/man/pkgctl-repo-configure.1.asciidoc5
-rw-r--r--src/lib/common.sh2
-rw-r--r--src/lib/repo/clone.sh31
-rw-r--r--src/lib/repo/configure.sh27
7 files changed, 71 insertions, 7 deletions
diff --git a/contrib/completion/bash/devtools.in b/contrib/completion/bash/devtools.in
index b0a90e5..17f863f 100644
--- a/contrib/completion/bash/devtools.in
+++ b/contrib/completion/bash/devtools.in
@@ -268,17 +268,23 @@ _pkgctl_repo_clone_args=(
--switch
-u --unprivileged
--universe
+ -j --jobs
-h --help
)
_pkgctl_repo_clone_args__maintainer_opts() { :; }
_pkgctl_repo_clone_args_m_opts() { _pkgctl_repo_clone_args__maintainer_opts; }
_pkgctl_repo_clone_args__switch_opts() { :; }
+_pkgctl_repo_clone_args__jobs_opts() { :; }
+_pkgctl_repo_clone_args_j_opts() { _pkgctl_repo_clone_args__jobs_opts; }
_pkgctl_repo_clone_opts() { _devtools_completions_all_packages; }
_pkgctl_repo_configure_args=(
+ -j --jobs
-h --help
)
+_pkgctl_repo_configure_args__jobs_opts() { :; }
+_pkgctl_repo_configure_args_j_opts() { _pkgctl_repo_clone_args__jobs_opts; }
_pkgctl_repo_configure_opts() { _filedir -d; }
diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in
index 45ffde3..240f781 100644
--- a/contrib/completion/zsh/_devtools.in
+++ b/contrib/completion/zsh/_devtools.in
@@ -110,11 +110,13 @@ _pkgctl_repo_clone_args=(
'(-m --maintainer=)'{-m,--maintainer=}'[Clone all packages of the named maintainer]:maintainer:'
'--switch=[Switch the current working tree to a specified version]'
'--universe[Clone all existing packages, useful for cache warming]'
+ '(-j --jobs)'{-j,--jobs}'[Run up to N jobs in parallel (default: number of processing units)]:jobs:'
'(-h --help)'{-h,--help}'[Display usage]'
'*:packages:_devtools_completions_all_packages'
)
_pkgctl_repo_configure_args=(
+ '(-j --jobs)'{-j,--jobs}'[Run up to N jobs in parallel (default: number of processing units)]:jobs:'
'(-h --help)'{-h,--help}'[Display usage]'
'*:git_dir:_files -/'
)
diff --git a/doc/man/pkgctl-repo-clone.1.asciidoc b/doc/man/pkgctl-repo-clone.1.asciidoc
index 487ec35..d6da062 100644
--- a/doc/man/pkgctl-repo-clone.1.asciidoc
+++ b/doc/man/pkgctl-repo-clone.1.asciidoc
@@ -32,6 +32,11 @@ Options
Switch to a specified version. The working tree and the index are updated to
match the version.
+*-j, --jobs* 'N'::
+ Run up to N jobs in parallel. By default the number of jobs is equal to the
+ number of available processing units. For sequential processing this option
+ needs to be passed with 1.
+
*-h, --help*::
Show a help text
diff --git a/doc/man/pkgctl-repo-configure.1.asciidoc b/doc/man/pkgctl-repo-configure.1.asciidoc
index 4499ed6..1b07dc7 100644
--- a/doc/man/pkgctl-repo-configure.1.asciidoc
+++ b/doc/man/pkgctl-repo-configure.1.asciidoc
@@ -25,6 +25,11 @@ read-only HTTPS otherwise.
Options
-------
+*-j, --jobs* 'N'::
+ Run up to N jobs in parallel. By default the number of jobs is equal to the
+ number of available processing units. For sequential processing this option
+ needs to be passed with 1.
+
*-h, --help*::
Show a help text
diff --git a/src/lib/common.sh b/src/lib/common.sh
index 24479eb..3d1ee56 100644
--- a/src/lib/common.sh
+++ b/src/lib/common.sh
@@ -30,7 +30,7 @@ export GIT_PACKAGING_URL_HTTPS="https://${GITLAB_HOST}/${GIT_PACKAGING_NAMESPACE
export PACKAGING_REPO_RELEASE_HOST=repos.archlinux.org
# check if messages are to be printed using color
-if [[ -t 2 && "$TERM" != dumb ]]; then
+if [[ -t 2 && "$TERM" != dumb ]] || [[ ${DEVTOOLS_COLOR} == always ]]; then
colorize
else
# shellcheck disable=2034
diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh
index 340aa69..a02d799 100644
--- a/src/lib/repo/clone.sh
+++ b/src/lib/repo/clone.sh
@@ -34,6 +34,7 @@ pkgctl_repo_clone_usage() {
-m, --maintainer=NAME Clone all packages of the named maintainer
--switch VERSION Switch the current working tree to a specified version
--universe Clone all existing packages, useful for cache warming
+ -j, --jobs N Run up to N jobs in parallel (default: $(nproc))
-h, --help Show this help text
EXAMPLES
@@ -55,9 +56,11 @@ pkgctl_repo_clone() {
local MAINTAINER=
local VERSION=
local CONFIGURE_OPTIONS=()
- local pkgbases
+ local jobs=
+ jobs=$(nproc)
# variables
+ local command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
local project_path
while (( $# )); do
@@ -97,6 +100,11 @@ pkgctl_repo_clone() {
CLONE_ALL=1
shift
;;
+ -j|--jobs)
+ (( $# <= 1 )) && die "missing argument for %s" "$1"
+ jobs=$2
+ shift 2
+ ;;
--)
shift
break
@@ -142,12 +150,31 @@ pkgctl_repo_clone() {
stat_done
fi
+ # parallelization
+ if [[ ${jobs} != 1 ]] && (( ${#pkgbases[@]} > 1 )); then
+ # force colors in parallel if parent process is colorized
+ if [[ -n ${BOLD} ]]; then
+ export DEVTOOLS_COLOR=always
+ fi
+ # assign command options
+ if [[ -n "${VERSION}" ]]; then
+ command+=" --switch '${VERSION}'"
+ fi
+ if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${pkgbases[@]}"; then
+ die 'Failed to clone some packages, please check the output'
+ exit 1
+ fi
+ exit 0
+ fi
+
for pkgbase in "${pkgbases[@]}"; do
if [[ ! -d ${pkgbase} ]]; then
msg "Cloning ${pkgbase} ..."
project_path=$(gitlab_project_name_to_path "${pkgbase}")
remote_url="${GIT_REPO_BASE_URL}/${project_path}.git"
- git clone --origin origin "${remote_url}" "${pkgbase}"
+ if ! git clone --origin origin "${remote_url}" "${pkgbase}"; then
+ die 'failed to clone %s' "${pkgbase}"
+ fi
else
warning "Skip cloning ${pkgbase}: Directory exists"
fi
diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh
index 942876a..a5708a0 100644
--- a/src/lib/repo/configure.sh
+++ b/src/lib/repo/configure.sh
@@ -33,6 +33,7 @@ pkgctl_repo_configure_usage() {
read-only HTTPS otherwise.
OPTIONS
+ -j, --jobs N Run up to N jobs in parallel (default: $(nproc))
-h, --help Show this help text
EXAMPLES
@@ -93,9 +94,12 @@ pkgctl_repo_configure() {
local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS}
local official=0
local proto=https
+ local jobs=
+ jobs=$(nproc)
local paths=()
# variables
+ local -r command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
local path realpath pkgbase remote_url project_path
local PACKAGER GPGKEY packager_name packager_email
@@ -105,6 +109,11 @@ pkgctl_repo_configure() {
pkgctl_repo_configure_usage
exit 0
;;
+ -j|--jobs)
+ (( $# <= 1 )) && die "missing argument for %s" "$1"
+ jobs=$2
+ shift 2
+ ;;
--)
shift
break
@@ -157,10 +166,21 @@ pkgctl_repo_configure() {
msg2 "protocol: ${YELLOW}${proto}${ALL_OFF}"
fi
+ # parallelization
+ if [[ ${jobs} != 1 ]] && (( ${#paths[@]} > 1 )); then
+ if [[ -n ${BOLD} ]]; then
+ export DEVTOOLS_COLOR=always
+ fi
+ if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${paths[@]}"; then
+ die 'Failed to configure some packages, please check the output'
+ exit 1
+ fi
+ exit 0
+ fi
+
for path in "${paths[@]}"; do
if ! realpath=$(realpath -e "${path}"); then
- error "No such directory: ${path}"
- continue
+ die "No such directory: ${path}"
fi
pkgbase=$(basename "${realpath}")
@@ -168,8 +188,7 @@ pkgctl_repo_configure() {
msg "Configuring ${pkgbase}"
if [[ ! -d "${path}/.git" ]]; then
- error "Not a Git repository: ${path}"
- continue
+ die "Not a Git repository: ${path}"
fi
pushd "${path}" >/dev/null