feature(parallel): run up to N jobs in parallel for repo clone/configure

Run up to N jobs in parallel. By default the number of jobs is equal to the
number of available processing units. For sequential processing this option
needs to be passed with 1.

Signed-off-by: Levente Polyak <anthraxx@archlinux.org>

3 files changed, 53 insertions, 7 deletions

diff --git a/src/lib/common.sh b/src/lib/common.sh
index 24479eb..3d1ee56 100644
--- a/src/lib/common.sh
+++ b/src/lib/common.sh
@@ -30,7 +30,7 @@ export GIT_PACKAGING_URL_HTTPS="https://${GITLAB_HOST}/${GIT_PACKAGING_NAMESPACE
 export PACKAGING_REPO_RELEASE_HOST=repos.archlinux.org
 
 # check if messages are to be printed using color
-if [[ -t 2 && "$TERM" != dumb ]]; then
+if [[ -t 2 && "$TERM" != dumb ]] || [[ ${DEVTOOLS_COLOR} == always ]]; then
 	colorize
 else
 	# shellcheck disable=2034
diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh
index 340aa69..a02d799 100644
--- a/src/lib/repo/clone.sh
+++ b/src/lib/repo/clone.sh
@@ -34,6 +34,7 @@ pkgctl_repo_clone_usage() {
 		    -m, --maintainer=NAME  Clone all packages of the named maintainer
 		    --switch VERSION       Switch the current working tree to a specified version
 		    --universe             Clone all existing packages, useful for cache warming
+		    -j, --jobs N           Run up to N jobs in parallel (default: $(nproc))
 		    -h, --help             Show this help text
 
 		EXAMPLES
@@ -55,9 +56,11 @@ pkgctl_repo_clone() {
 	local MAINTAINER=
 	local VERSION=
 	local CONFIGURE_OPTIONS=()
-	local pkgbases
+	local jobs=
+	jobs=$(nproc)
 
 	# variables
+	local command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
 	local project_path
 
 	while (( $# )); do
@@ -97,6 +100,11 @@ pkgctl_repo_clone() {
 				CLONE_ALL=1
 				shift
 				;;
+			-j|--jobs)
+				(( $# <= 1 )) && die "missing argument for %s" "$1"
+				jobs=$2
+				shift 2
+				;;
 			--)
 				shift
 				break
@@ -142,12 +150,31 @@ pkgctl_repo_clone() {
 		stat_done
 	fi
 
+	# parallelization
+	if [[ ${jobs} != 1 ]] && (( ${#pkgbases[@]} > 1 )); then
+		# force colors in parallel if parent process is colorized
+		if [[ -n ${BOLD} ]]; then
+			export DEVTOOLS_COLOR=always
+		fi
+		# assign command options
+		if [[ -n "${VERSION}" ]]; then
+			command+=" --switch '${VERSION}'"
+		fi
+		if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${pkgbases[@]}"; then
+			die 'Failed to clone some packages, please check the output'
+			exit 1
+		fi
+		exit 0
+	fi
+
 	for pkgbase in "${pkgbases[@]}"; do
 		if [[ ! -d ${pkgbase} ]]; then
 			msg "Cloning ${pkgbase} ..."
 			project_path=$(gitlab_project_name_to_path "${pkgbase}")
 			remote_url="${GIT_REPO_BASE_URL}/${project_path}.git"
-			git clone --origin origin "${remote_url}" "${pkgbase}"
+			if ! git clone --origin origin "${remote_url}" "${pkgbase}"; then
+				die 'failed to clone %s' "${pkgbase}"
+			fi
 		else
 			warning "Skip cloning ${pkgbase}: Directory exists"
 		fi
diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh
index 942876a..a5708a0 100644
--- a/src/lib/repo/configure.sh
+++ b/src/lib/repo/configure.sh
@@ -33,6 +33,7 @@ pkgctl_repo_configure_usage() {
 		read-only HTTPS otherwise.
 
 		OPTIONS
+		    -j, --jobs N         Run up to N jobs in parallel (default: $(nproc))
 		    -h, --help           Show this help text
 
 		EXAMPLES
@@ -93,9 +94,12 @@ pkgctl_repo_configure() {
 	local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS}
 	local official=0
 	local proto=https
+	local jobs=
+	jobs=$(nproc)
 	local paths=()
 
 	# variables
+	local -r command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
 	local path realpath pkgbase remote_url project_path
 	local PACKAGER GPGKEY packager_name packager_email
 
@@ -105,6 +109,11 @@ pkgctl_repo_configure() {
 				pkgctl_repo_configure_usage
 				exit 0
 				;;
+			-j|--jobs)
+				(( $# <= 1 )) && die "missing argument for %s" "$1"
+				jobs=$2
+				shift 2
+				;;
 			--)
 				shift
 				break
@@ -157,10 +166,21 @@ pkgctl_repo_configure() {
 		msg2 "protocol: ${YELLOW}${proto}${ALL_OFF}"
 	fi
 
+	# parallelization
+	if [[ ${jobs} != 1 ]] && (( ${#paths[@]} > 1 )); then
+		if [[ -n ${BOLD} ]]; then
+			export DEVTOOLS_COLOR=always
+		fi
+		if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${paths[@]}"; then
+			die 'Failed to configure some packages, please check the output'
+			exit 1
+		fi
+		exit 0
+	fi
+
 	for path in "${paths[@]}"; do
 		if ! realpath=$(realpath -e "${path}"); then
-			error "No such directory: ${path}"
-			continue
+			die "No such directory: ${path}"
 		fi
 
 		pkgbase=$(basename "${realpath}")
@@ -168,8 +188,7 @@ pkgctl_repo_configure() {
 		msg "Configuring ${pkgbase}"
 
 		if [[ ! -d "${path}/.git" ]]; then
-			error "Not a Git repository: ${path}"
-			continue
+			die "Not a Git repository: ${path}"
 		fi
 
 		pushd "${path}" >/dev/null