From 6ce666a1669235749c17d5c44d8a24dea4a135da Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Fri, 3 Feb 2023 00:58:59 +0100 Subject: feature(parallel): run up to N jobs in parallel for repo clone/configure Run up to N jobs in parallel. By default the number of jobs is equal to the number of available processing units. For sequential processing this option needs to be passed with 1. Signed-off-by: Levente Polyak --- src/lib/common.sh | 2 +- src/lib/repo/clone.sh | 31 +++++++++++++++++++++++++++++-- src/lib/repo/configure.sh | 27 +++++++++++++++++++++++---- 3 files changed, 53 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/lib/common.sh b/src/lib/common.sh index 24479eb..3d1ee56 100644 --- a/src/lib/common.sh +++ b/src/lib/common.sh @@ -30,7 +30,7 @@ export GIT_PACKAGING_URL_HTTPS="https://${GITLAB_HOST}/${GIT_PACKAGING_NAMESPACE export PACKAGING_REPO_RELEASE_HOST=repos.archlinux.org # check if messages are to be printed using color -if [[ -t 2 && "$TERM" != dumb ]]; then +if [[ -t 2 && "$TERM" != dumb ]] || [[ ${DEVTOOLS_COLOR} == always ]]; then colorize else # shellcheck disable=2034 diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh index 340aa69..a02d799 100644 --- a/src/lib/repo/clone.sh +++ b/src/lib/repo/clone.sh @@ -34,6 +34,7 @@ pkgctl_repo_clone_usage() { -m, --maintainer=NAME Clone all packages of the named maintainer --switch VERSION Switch the current working tree to a specified version --universe Clone all existing packages, useful for cache warming + -j, --jobs N Run up to N jobs in parallel (default: $(nproc)) -h, --help Show this help text EXAMPLES @@ -55,9 +56,11 @@ pkgctl_repo_clone() { local MAINTAINER= local VERSION= local CONFIGURE_OPTIONS=() - local pkgbases + local jobs= + jobs=$(nproc) # variables + local command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} local project_path while (( $# )); do @@ -97,6 +100,11 @@ pkgctl_repo_clone() { CLONE_ALL=1 shift ;; + -j|--jobs) + (( $# <= 1 )) && die "missing argument for %s" "$1" + jobs=$2 + shift 2 + ;; --) shift break @@ -142,12 +150,31 @@ pkgctl_repo_clone() { stat_done fi + # parallelization + if [[ ${jobs} != 1 ]] && (( ${#pkgbases[@]} > 1 )); then + # force colors in parallel if parent process is colorized + if [[ -n ${BOLD} ]]; then + export DEVTOOLS_COLOR=always + fi + # assign command options + if [[ -n "${VERSION}" ]]; then + command+=" --switch '${VERSION}'" + fi + if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${pkgbases[@]}"; then + die 'Failed to clone some packages, please check the output' + exit 1 + fi + exit 0 + fi + for pkgbase in "${pkgbases[@]}"; do if [[ ! -d ${pkgbase} ]]; then msg "Cloning ${pkgbase} ..." project_path=$(gitlab_project_name_to_path "${pkgbase}") remote_url="${GIT_REPO_BASE_URL}/${project_path}.git" - git clone --origin origin "${remote_url}" "${pkgbase}" + if ! git clone --origin origin "${remote_url}" "${pkgbase}"; then + die 'failed to clone %s' "${pkgbase}" + fi else warning "Skip cloning ${pkgbase}: Directory exists" fi diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh index 942876a..a5708a0 100644 --- a/src/lib/repo/configure.sh +++ b/src/lib/repo/configure.sh @@ -33,6 +33,7 @@ pkgctl_repo_configure_usage() { read-only HTTPS otherwise. OPTIONS + -j, --jobs N Run up to N jobs in parallel (default: $(nproc)) -h, --help Show this help text EXAMPLES @@ -93,9 +94,12 @@ pkgctl_repo_configure() { local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS} local official=0 local proto=https + local jobs= + jobs=$(nproc) local paths=() # variables + local -r command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} local path realpath pkgbase remote_url project_path local PACKAGER GPGKEY packager_name packager_email @@ -105,6 +109,11 @@ pkgctl_repo_configure() { pkgctl_repo_configure_usage exit 0 ;; + -j|--jobs) + (( $# <= 1 )) && die "missing argument for %s" "$1" + jobs=$2 + shift 2 + ;; --) shift break @@ -157,10 +166,21 @@ pkgctl_repo_configure() { msg2 "protocol: ${YELLOW}${proto}${ALL_OFF}" fi + # parallelization + if [[ ${jobs} != 1 ]] && (( ${#paths[@]} > 1 )); then + if [[ -n ${BOLD} ]]; then + export DEVTOOLS_COLOR=always + fi + if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${paths[@]}"; then + die 'Failed to configure some packages, please check the output' + exit 1 + fi + exit 0 + fi + for path in "${paths[@]}"; do if ! realpath=$(realpath -e "${path}"); then - error "No such directory: ${path}" - continue + die "No such directory: ${path}" fi pkgbase=$(basename "${realpath}") @@ -168,8 +188,7 @@ pkgctl_repo_configure() { msg "Configuring ${pkgbase}" if [[ ! -d "${path}/.git" ]]; then - error "Not a Git repository: ${path}" - continue + die "Not a Git repository: ${path}" fi pushd "${path}" >/dev/null -- cgit v1.2.3-70-g09d2