Send patches - preferably formatted by git format-patch - to patches at archlinux32 dot org.
summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorLevente Polyak <anthraxx@archlinux.org>2023-02-03 00:58:59 +0100
committerLevente Polyak <anthraxx@archlinux.org>2023-05-20 16:12:45 +0200
commit6ce666a1669235749c17d5c44d8a24dea4a135da (patch)
tree9a69904029bea07f5f21b965b8046c727d54cefc /src
parentbf61b8472a2e50a740229f007513dd663b9dddda (diff)
feature(parallel): run up to N jobs in parallel for repo clone/configure
Run up to N jobs in parallel. By default the number of jobs is equal to the number of available processing units. For sequential processing this option needs to be passed with 1. Signed-off-by: Levente Polyak <anthraxx@archlinux.org>
Diffstat (limited to 'src')
-rw-r--r--src/lib/common.sh2
-rw-r--r--src/lib/repo/clone.sh31
-rw-r--r--src/lib/repo/configure.sh27
3 files changed, 53 insertions, 7 deletions
diff --git a/src/lib/common.sh b/src/lib/common.sh
index 24479eb..3d1ee56 100644
--- a/src/lib/common.sh
+++ b/src/lib/common.sh
@@ -30,7 +30,7 @@ export GIT_PACKAGING_URL_HTTPS="https://${GITLAB_HOST}/${GIT_PACKAGING_NAMESPACE
export PACKAGING_REPO_RELEASE_HOST=repos.archlinux.org
# check if messages are to be printed using color
-if [[ -t 2 && "$TERM" != dumb ]]; then
+if [[ -t 2 && "$TERM" != dumb ]] || [[ ${DEVTOOLS_COLOR} == always ]]; then
colorize
else
# shellcheck disable=2034
diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh
index 340aa69..a02d799 100644
--- a/src/lib/repo/clone.sh
+++ b/src/lib/repo/clone.sh
@@ -34,6 +34,7 @@ pkgctl_repo_clone_usage() {
-m, --maintainer=NAME Clone all packages of the named maintainer
--switch VERSION Switch the current working tree to a specified version
--universe Clone all existing packages, useful for cache warming
+ -j, --jobs N Run up to N jobs in parallel (default: $(nproc))
-h, --help Show this help text
EXAMPLES
@@ -55,9 +56,11 @@ pkgctl_repo_clone() {
local MAINTAINER=
local VERSION=
local CONFIGURE_OPTIONS=()
- local pkgbases
+ local jobs=
+ jobs=$(nproc)
# variables
+ local command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
local project_path
while (( $# )); do
@@ -97,6 +100,11 @@ pkgctl_repo_clone() {
CLONE_ALL=1
shift
;;
+ -j|--jobs)
+ (( $# <= 1 )) && die "missing argument for %s" "$1"
+ jobs=$2
+ shift 2
+ ;;
--)
shift
break
@@ -142,12 +150,31 @@ pkgctl_repo_clone() {
stat_done
fi
+ # parallelization
+ if [[ ${jobs} != 1 ]] && (( ${#pkgbases[@]} > 1 )); then
+ # force colors in parallel if parent process is colorized
+ if [[ -n ${BOLD} ]]; then
+ export DEVTOOLS_COLOR=always
+ fi
+ # assign command options
+ if [[ -n "${VERSION}" ]]; then
+ command+=" --switch '${VERSION}'"
+ fi
+ if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${pkgbases[@]}"; then
+ die 'Failed to clone some packages, please check the output'
+ exit 1
+ fi
+ exit 0
+ fi
+
for pkgbase in "${pkgbases[@]}"; do
if [[ ! -d ${pkgbase} ]]; then
msg "Cloning ${pkgbase} ..."
project_path=$(gitlab_project_name_to_path "${pkgbase}")
remote_url="${GIT_REPO_BASE_URL}/${project_path}.git"
- git clone --origin origin "${remote_url}" "${pkgbase}"
+ if ! git clone --origin origin "${remote_url}" "${pkgbase}"; then
+ die 'failed to clone %s' "${pkgbase}"
+ fi
else
warning "Skip cloning ${pkgbase}: Directory exists"
fi
diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh
index 942876a..a5708a0 100644
--- a/src/lib/repo/configure.sh
+++ b/src/lib/repo/configure.sh
@@ -33,6 +33,7 @@ pkgctl_repo_configure_usage() {
read-only HTTPS otherwise.
OPTIONS
+ -j, --jobs N Run up to N jobs in parallel (default: $(nproc))
-h, --help Show this help text
EXAMPLES
@@ -93,9 +94,12 @@ pkgctl_repo_configure() {
local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS}
local official=0
local proto=https
+ local jobs=
+ jobs=$(nproc)
local paths=()
# variables
+ local -r command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
local path realpath pkgbase remote_url project_path
local PACKAGER GPGKEY packager_name packager_email
@@ -105,6 +109,11 @@ pkgctl_repo_configure() {
pkgctl_repo_configure_usage
exit 0
;;
+ -j|--jobs)
+ (( $# <= 1 )) && die "missing argument for %s" "$1"
+ jobs=$2
+ shift 2
+ ;;
--)
shift
break
@@ -157,10 +166,21 @@ pkgctl_repo_configure() {
msg2 "protocol: ${YELLOW}${proto}${ALL_OFF}"
fi
+ # parallelization
+ if [[ ${jobs} != 1 ]] && (( ${#paths[@]} > 1 )); then
+ if [[ -n ${BOLD} ]]; then
+ export DEVTOOLS_COLOR=always
+ fi
+ if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${paths[@]}"; then
+ die 'Failed to configure some packages, please check the output'
+ exit 1
+ fi
+ exit 0
+ fi
+
for path in "${paths[@]}"; do
if ! realpath=$(realpath -e "${path}"); then
- error "No such directory: ${path}"
- continue
+ die "No such directory: ${path}"
fi
pkgbase=$(basename "${realpath}")
@@ -168,8 +188,7 @@ pkgctl_repo_configure() {
msg "Configuring ${pkgbase}"
if [[ ! -d "${path}/.git" ]]; then
- error "Not a Git repository: ${path}"
- continue
+ die "Not a Git repository: ${path}"
fi
pushd "${path}" >/dev/null