From b5d5402e439f5edfd642fb4f680d596f5992e874 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Mon, 10 Oct 2022 00:37:51 +0200 Subject: src: modularize repo layout into a library This will greatly help us to structure the functionality and commands in a more sane way. We will distribute the sources as actual libraries and reuse code with imports instead of processing everything with m4 and duplicating a lot of code. --- src/lib/repo/clone.sh | 139 ++++++++++++++++++++++++++++++++++++++ src/lib/repo/configure.sh | 169 ++++++++++++++++++++++++++++++++++++++++++++++ src/lib/repo/web.sh | 84 +++++++++++++++++++++++ 3 files changed, 392 insertions(+) create mode 100644 src/lib/repo/clone.sh create mode 100644 src/lib/repo/configure.sh create mode 100644 src/lib/repo/web.sh (limited to 'src/lib/repo') diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh new file mode 100644 index 0000000..42dc383 --- /dev/null +++ b/src/lib/repo/clone.sh @@ -0,0 +1,139 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_REPO_CLONE_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_REPO_CLONE_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/api/gitlab.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/gitlab.sh +# shellcheck source=src/lib/repo/configure.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/repo/configure.sh + +source /usr/share/makepkg/util/message.sh + +set -e + + +pkgctl_repo_clone_usage() { + local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} + cat <<- _EOF_ + Usage: ${COMMAND} [OPTIONS] [PKGBASE]... + + Clone Git packaging repositories from the canonical namespace. + + The configure command is subsequently invoked to synchronize the distro + specs and makepkg.conf settings. The unprivileged option can be used + for cloning packaging repositories without SSH access using read-only + HTTPS. + + OPTIONS + -m, --maintainer=NAME Clone all packages of the named maintainer + -u, --unprivileged Clone package with read-only access and without + packager info as Git author + --universe Clone all existing packages, useful for cache warming + -h, --help Show this help text + + EXAMPLES + $ ${COMMAND} libfoo linux libbar + $ ${COMMAND} --maintainer mynickname +_EOF_ +} + +pkgctl_repo_clone() { + if (( $# < 1 )); then + pkgctl_repo_clone_usage + exit 0 + fi + + # options + local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_SSH} + local CLONE_ALL=0 + local MAINTAINER= + local CONFIGURE_OPTIONS=() + local pkgbases + + while (( $# )); do + case $1 in + -h|--help) + pkgctl_repo_clone_usage + exit 0 + ;; + -u|--unprivileged) + GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS} + CONFIGURE_OPTIONS+=("$1") + shift + ;; + -m|--maintainer) + (( $# <= 1 )) && die "missing argument for %s" "$1" + MAINTAINER="$2" + shift 2 + ;; + --maintainer=*) + MAINTAINER="${1#*=}" + shift + ;; + --universe) + CLONE_ALL=1 + shift + ;; + --) + shift + break + ;; + -*) + die "invalid argument: %s" "$1" + ;; + *) + pkgbases=("$@") + break + ;; + esac + done + + # Query packages of a maintainer + if [[ -n ${MAINTAINER} ]]; then + stat_busy "Query packages" + max_pages=$(curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&maintainer=${MAINTAINER}" | jq -r '.num_pages') + if [[ ! ${max_pages} =~ ([[:digit:]]) ]]; then + stat_done + warning "found no packages for maintainer ${MAINTAINER}" + exit 0 + fi + mapfile -t pkgbases < <(for page in $(seq "${max_pages}"); do + curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&maintainer=${MAINTAINER}&page=${page}" | jq -r '.results[].pkgbase' + stat_progress + done | sort --unique) + stat_done + fi + + # Query all released packages + if (( CLONE_ALL )); then + stat_busy "Query all released packages" + max_pages=$(curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name" | jq -r '.num_pages') + if [[ ! ${max_pages} =~ ([[:digit:]]) ]]; then + stat_done + die "failed to query packages" + fi + mapfile -t pkgbases < <(for page in $(seq "${max_pages}"); do + curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&page=${page}" | jq -r '.results[].pkgbase' + stat_progress + done | sort --unique) + stat_done + fi + + for pkgbase in "${pkgbases[@]}"; do + if [[ ! -d ${pkgbase} ]]; then + msg "Cloning ${pkgbase} ..." + remote_url="${GIT_REPO_BASE_URL}/${pkgbase}.git" + git clone --origin origin "${remote_url}" "${pkgbase}" + else + warning "Skip cloning ${pkgbase}: Directory exists" + fi + + pkgctl_repo_configure "${CONFIGURE_OPTIONS[@]}" "${pkgbase}" + done +} diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh new file mode 100644 index 0000000..5c99562 --- /dev/null +++ b/src/lib/repo/configure.sh @@ -0,0 +1,169 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_REPO_CONFIGURE_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_REPO_CONFIGURE_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/api/gitlab.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/gitlab.sh + +source /usr/share/makepkg/util/config.sh +source /usr/share/makepkg/util/message.sh + +set -e + + +pkgctl_repo_configure_usage() { + local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} + cat <<- _EOF_ + Usage: ${COMMAND} [OPTIONS] [PATH]... + + Configure Git packaging repositories according to distro specs and + makepkg.conf settings. + + Git author information and the used signing key is set up from + makepkg.conf read from any valid location like /etc or XDG_CONFIG_HOME. + The unprivileged option can be used for cloning packaging repositories + without SSH access using read-only HTTPS. + + OPTIONS + -u, --unprivileged Configure read-only repo without packager info as Git author + -h, --help Show this help text + + EXAMPLES + $ ${COMMAND} configure * +_EOF_ +} + +pkgctl_repo_configure() { + # options + local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_SSH} + local UNPRIVILEGED=0 + local PACKAGER_NAME= + local PACKAGER_EMAIL= + local paths=() + + # variables + local path realpath pkgbase remote_url + + while (( $# )); do + case $1 in + -h|--help) + pkgctl_repo_configure_usage + exit 0 + ;; + -u|--unprivileged) + GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS} + UNPRIVILEGED=1 + shift + ;; + --) + shift + break + ;; + -*) + die "invalid argument: %s" "$1" + ;; + *) + paths=("$@") + break + ;; + esac + done + + # check if invoked without any path from within a packaging repo + if (( ${#paths[@]} == 0 )); then + if [[ -f PKGBUILD ]]; then + paths=(".") + else + pkgctl_repo_configure_usage + exit 1 + fi + fi + + # Load makepkg.conf variables to be available + # shellcheck disable=2119 + load_makepkg_config + + # Check official packaging identity before setting Git author + if (( ! UNPRIVILEGED )); then + if [[ $PACKAGER == *"Unknown Packager"* ]]; then + die "Packager must be set in makepkg.conf" + fi + packager_pattern="(.+) <(.+@.+)>" + if [[ ! $PACKAGER =~ $packager_pattern ]]; then + die "Invalid Packager format '${PACKAGER}' in makepkg.conf" + fi + + PACKAGER_NAME=$(echo "${PACKAGER}"|sed -E "s/${packager_pattern}/\1/") + PACKAGER_EMAIL=$(echo "${PACKAGER}"|sed -E "s/${packager_pattern}/\2/") + + if [[ ! $PACKAGER_EMAIL =~ .+@archlinux.org ]]; then + die "Packager email '${PACKAGER_EMAIL}' is not an @archlinux.org address" + fi + fi + + msg "Collected packager settings" + msg2 "name : ${PACKAGER_NAME}" + msg2 "email : ${PACKAGER_EMAIL}" + msg2 "gpg-key : ${GPGKEY:-undefined}" + + # TODO: print which protocol got auto detected, ssh https + + for path in "${paths[@]}"; do + if ! realpath=$(realpath -e "${path}"); then + error "No such directory: ${path}" + continue + fi + + pkgbase=$(basename "${realpath}") + pkgbase=${pkgbase%.git} + msg "Configuring ${pkgbase}" + + if [[ ! -d "${path}/.git" ]]; then + error "Not a Git repository: ${path}" + continue + fi + + remote_url="${GIT_REPO_BASE_URL}/${pkgbase}.git" + if ! git -C "${path}" remote add origin "${remote_url}" &>/dev/null; then + git -C "${path}" remote set-url origin "${remote_url}" + fi + + # move the master branch to main + if [[ $(git -C "${path}" symbolic-ref --short HEAD) == master ]]; then + git -C "${path}" branch --move main + git -C "${path}" config branch.main.merge refs/heads/main + fi + + git -C "${path}" config devtools.version "${GIT_REPO_SPEC_VERSION}" + git -C "${path}" config pull.rebase true + git -C "${path}" config branch.autoSetupRebase always + git -C "${path}" config branch.main.remote origin + git -C "${path}" config branch.main.rebase true + + git -C "${path}" config transfer.fsckobjects true + git -C "${path}" config fetch.fsckobjects true + git -C "${path}" config receive.fsckobjects true + + if (( ! UNPRIVILEGED )); then + git -C "${path}" config user.name "${PACKAGER_NAME}" + git -C "${path}" config user.email "${PACKAGER_EMAIL}" + git -C "${path}" config commit.gpgsign true + if [[ -n $GPGKEY ]]; then + git -C "${path}" config user.signingKey "${GPGKEY}" + else + warning "Missing makepkg.conf configuration: GPGKEY" + fi + fi + + if ! git ls-remote origin &>/dev/null; then + warning "configured remote origin may not exist, run:" + msg2 "pkgctl repo create ${pkgbase}" + fi + done +} diff --git a/src/lib/repo/web.sh b/src/lib/repo/web.sh new file mode 100644 index 0000000..3fa214d --- /dev/null +++ b/src/lib/repo/web.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_REPO_WEB_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_REPO_WEB_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/api/gitlab.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/gitlab.sh + +set -e + + +pkgctl_repo_web_usage() { + local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} + cat <<- _EOF_ + Usage: ${COMMAND} [OPTIONS] [PKGBASE]... + + Open the packaging repository's website via xdg-open. If called with + no arguments, open the package cloned in the current working directory. + + OPTIONS + -h, --help Show this help text + + EXAMPLES + $ ${COMMAND} web linux +_EOF_ +} + +pkgctl_repo_web() { + local pkgbases=() + local path giturl pkgbase + + # option checking + while (( $# )); do + case $1 in + -h|--help) + pkgctl_repo_web_usage + exit 0 + ;; + --) + shift + break + ;; + -*) + die "invalid argument: %s" "$1" + ;; + *) + pkgbases=("$@") + break + ;; + esac + done + + # Check if web mode has xdg-open + if ! command -v xdg-open &>/dev/null; then + die "The web command requires 'xdg-open'" + fi + + # Check if used without pkgnames in a packaging directory + if (( ! $# )); then + path=${PWD} + if [[ ! -d "${path}/.git" ]]; then + die "Not a Git repository: ${path}" + fi + + giturl=$(git -C "${path}" remote get-url origin) + if [[ ${giturl} != *${GIT_PACKAGING_NAMESPACE}* ]]; then + die "Not a packaging repository: ${path}" + fi + + pkgbase=$(basename "${giturl}") + pkgbase=${pkgbase%.git} + pkgbases=("${pkgbase}") + fi + + for pkgbase in "${pkgbases[@]}"; do + path=$(gitlab_project_name_to_path "${pkgbase}") + xdg-open "${GIT_PACKAGING_URL_HTTPS}/${path}" + done +} -- cgit v1.2.3-70-g09d2 From 2a59c32bf4ff117bd02d58a4e3f322b709259f1e Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sun, 30 Oct 2022 14:44:06 +0100 Subject: repo: added command to create a new packaging repository --- contrib/completion/zsh/_devtools.in | 7 +++ doc/man/pkgctl-repo-create.1.asciidoc | 40 ++++++++++++ doc/man/pkgctl-repo.1.asciidoc | 4 ++ src/lib/repo.sh | 10 +++ src/lib/repo/create.sh | 113 ++++++++++++++++++++++++++++++++++ 5 files changed, 174 insertions(+) create mode 100644 doc/man/pkgctl-repo-create.1.asciidoc create mode 100644 src/lib/repo/create.sh (limited to 'src/lib/repo') diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in index 6ff6cad..3395338 100644 --- a/contrib/completion/zsh/_devtools.in +++ b/contrib/completion/zsh/_devtools.in @@ -38,6 +38,7 @@ _pkgctl_repo_cmds=( "pkgctl repo command" "clone[Clone a package repository]" "configure[Configure a clone according to distro specs]" + "create[Create a new GitLab package repository]" "web[Open the packaging repository's website]" ) @@ -55,6 +56,12 @@ _pkgctl_repo_configure_args=( '*:git_dir:_files -/' ) +_pkgctl_repo_create_args=( + '(-c --clone)'{-c,--clone}'[Clone the Git repository after creation]' + '(-h --help)'{-h,--help}'[Display usage]' + '1:pkgbase' +) + _pkgctl_repo_web_args=( '(-h --help)'{-h,--help}'[Display usage]' '*:git_dir:_files -/' diff --git a/doc/man/pkgctl-repo-create.1.asciidoc b/doc/man/pkgctl-repo-create.1.asciidoc new file mode 100644 index 0000000..b9d980b --- /dev/null +++ b/doc/man/pkgctl-repo-create.1.asciidoc @@ -0,0 +1,40 @@ +pkgctl-repo-create(1) +====================== + +Name +---- +pkgctl-repo-create - Create a new GitLab package repository + +Synopsis +-------- +pkgctl repo create [OPTIONS] [PKGBASE...] + +Description +----------- + +Create a new Git packaging repository in the canonical GitLab namespace. + +This command requires a valid GitLab API authentication. To setup a new +GitLab token or check the currently configured one please consult the +'auth' subcommand for further instructions. + +If invoked without a parameter, try to create a packaging repository +based on the 'PKGBUILD' from the current working directory. + +Options +------- + +*-c, --clone*:: + Clone the Git repository after creation + +*-h, --help*:: + Show a help text + +See Also +-------- + +linkman:pkgctl-auth[1] +linkman:pkgctl-repo-clone[1] +linkman:pkgctl-repo-configure[1] + +include::include/footer.asciidoc[] diff --git a/doc/man/pkgctl-repo.1.asciidoc b/doc/man/pkgctl-repo.1.asciidoc index 564679a..630afd8 100644 --- a/doc/man/pkgctl-repo.1.asciidoc +++ b/doc/man/pkgctl-repo.1.asciidoc @@ -38,6 +38,9 @@ pkgctl repo clone:: pkgctl repo configure:: Configure a clone according to distro specs +pkgctl repo create:: + Create a new GitLab package repository + pkgctl repo web:: Open the packaging repository's website @@ -46,6 +49,7 @@ See Also linkman:pkgctl-repo-clone[1] linkman:pkgctl-repo-configure[1] +linkman:pkgctl-repo-create[1] linkman:pkgctl-repo-web[1] include::include/footer.asciidoc[] diff --git a/src/lib/repo.sh b/src/lib/repo.sh index 8b8df11..6b3817a 100644 --- a/src/lib/repo.sh +++ b/src/lib/repo.sh @@ -29,6 +29,7 @@ pkgctl_repo_usage() { COMMANDS clone Clone a package repository configure Configure a clone according to distro specs + create Create a new GitLab package repository web Open the packaging repository's website OPTIONS @@ -38,6 +39,7 @@ pkgctl_repo_usage() { $ ${COMMAND} clone libfoo linux libbar $ ${COMMAND} clone --maintainer mynickname $ ${COMMAND} configure * + $ ${COMMAND} create libfoo $ ${COMMAND} web linux _EOF_ } @@ -71,6 +73,14 @@ pkgctl_repo() { pkgctl_repo_configure "$@" exit 0 ;; + create) + _DEVTOOLS_COMMAND+=" $1" + shift + # shellcheck source=src/lib/repo/create.sh + source "${_DEVTOOLS_LIBRARY_DIR}"/lib/repo/create.sh + pkgctl_repo_create "$@" + exit 0 + ;; web) _DEVTOOLS_COMMAND+=" $1" shift diff --git a/src/lib/repo/create.sh b/src/lib/repo/create.sh new file mode 100644 index 0000000..31b46e1 --- /dev/null +++ b/src/lib/repo/create.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_REPO_CREATE_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_REPO_CREATE_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/api/gitlab.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/gitlab.sh +# shellcheck source=src/lib/repo/clone.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/repo/clone.sh +# shellcheck source=src/lib/repo/configure.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/repo/configure.sh + +set -e + + +pkgctl_repo_create_usage() { + local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} + cat <<- _EOF_ + Usage: ${COMMAND} [OPTIONS] [PKGBASE]... + + Create a new Git packaging repository in the canonical GitLab namespace. + + This command requires a valid GitLab API authentication. To setup a new + GitLab token or check the currently configured one please consult the + 'auth' subcommand for further instructions. + + If invoked without a parameter, try to create a packaging repository + based on the PKGBUILD from the current working directory and configure + the local repository afterwards. + + OPTIONS + -c, --clone Clone the Git repository after creation + -h, --help Show this help text + + EXAMPLES + $ ${COMMAND} libfoo +_EOF_ +} + +pkgctl_repo_create() { + # options + local pkgbases=() + local pkgbase + local clone=0 + local configure=0 + + # variables + local path + + while (( $# )); do + case $1 in + -h|--help) + pkgctl_repo_create_usage + exit 0 + ;; + -c|--clone) + clone=1 + shift + ;; + -*) + die "invalid argument: %s" "$1" + ;; + *) + pkgbases=("$@") + break + ;; + esac + done + + # check if invoked without any path from within a packaging repo + if (( ${#pkgbases[@]} == 0 )); then + if [[ -f PKGBUILD ]]; then + if ! path=$(realpath -e .); then + die "failed to read path from current directory" + fi + pkgbases=("$(basename "${path}")") + clone=0 + configure=1 + else + pkgctl_repo_create_usage + exit 1 + fi + fi + + # create projects + for pkgbase in "${pkgbases[@]}"; do + if ! gitlab_api_create_project "${pkgbase}" >/dev/null; then + die "failed to create project: ${pkgbase}" + fi + msg_success "Successfully created ${pkgbase}" + if (( clone )); then + pkgctl_repo_clone "${pkgbase}" + elif (( configure )); then + pkgctl_repo_configure + fi + done + + # some convenience hints if not in auto clone/configure mode + if (( ! clone )) && (( ! configure )); then + cat <<- _EOF_ + + For new clones: + $(msg2 "pkgctl repo clone ${pkgbases[*]}") + For existing clones: + $(msg2 "pkgctl repo configure ${pkgbases[*]}") + _EOF_ + fi +} -- cgit v1.2.3-70-g09d2 From f669a71e847da88fefc1296dd2dd9ba82549a8c6 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Fri, 20 Jan 2023 18:03:31 +0100 Subject: repo-configure: automatically determine protocol from packager identity The remote protocol is automatically determined from the author email address by choosing SSH for all official packager identities and read-only HTTPS otherwise. Signed-off-by: Levente Polyak --- contrib/completion/zsh/_devtools.in | 2 - doc/man/pkgctl-repo-clone.1.asciidoc | 3 - doc/man/pkgctl-repo-configure.1.asciidoc | 9 +- src/lib/repo/clone.sh | 2 - src/lib/repo/configure.sh | 167 ++++++++++++++++++++----------- 5 files changed, 114 insertions(+), 69 deletions(-) (limited to 'src/lib/repo') diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in index c743667..5908bd6 100644 --- a/contrib/completion/zsh/_devtools.in +++ b/contrib/completion/zsh/_devtools.in @@ -98,14 +98,12 @@ _pkgctl_repo_cmds=( _pkgctl_repo_clone_args=( '(-m --maintainer=)'{-m,--maintainer=}'[Clone all packages of the named maintainer]:maintainer:' - '(-u --unprivileged)'{-u,--unprivileged}'[Read-only access without packager info as Git author]' '--universe[Clone all existing packages, useful for cache warming]' '(-h --help)'{-h,--help}'[Display usage]' '*:packages:_devtools_completions_all_packages' ) _pkgctl_repo_configure_args=( - '(-u --unprivileged)'{-u,--unprivileged}'[Configure read-only repo without packager info as Git author]' '(-h --help)'{-h,--help}'[Display usage]' '*:git_dir:_files -/' ) diff --git a/doc/man/pkgctl-repo-clone.1.asciidoc b/doc/man/pkgctl-repo-clone.1.asciidoc index 9770ef8..a39fb37 100644 --- a/doc/man/pkgctl-repo-clone.1.asciidoc +++ b/doc/man/pkgctl-repo-clone.1.asciidoc @@ -25,9 +25,6 @@ Options *-m, --maintainer* 'NAME':: Clone all packages of the named maintainer -*-u, --unprivileged*:: - Clone package with read-only access and without packager info as Git author - *--universe*:: Clone all existing packages, useful for cache warming diff --git a/doc/man/pkgctl-repo-configure.1.asciidoc b/doc/man/pkgctl-repo-configure.1.asciidoc index c3a14de..4499ed6 100644 --- a/doc/man/pkgctl-repo-configure.1.asciidoc +++ b/doc/man/pkgctl-repo-configure.1.asciidoc @@ -17,15 +17,14 @@ Configure Git packaging repositories according to distro specs and Git author information and the used signing key is set up from 'makepkg.conf' read from any valid location like '/etc' or 'XDG_CONFIG_HOME'. -The unprivileged option can be used for cloning packaging repositories -without SSH access using read-only HTTPS. + +The remote protocol is automatically determined from the author email +address by choosing SSH for all official packager identities and +read-only HTTPS otherwise. Options ------- -*-u, --unprivileged*:: - Configure read-only repo without packager info as Git author - *-h, --help*:: Show a help text diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh index 42dc383..4b26fcf 100644 --- a/src/lib/repo/clone.sh +++ b/src/lib/repo/clone.sh @@ -32,8 +32,6 @@ pkgctl_repo_clone_usage() { OPTIONS -m, --maintainer=NAME Clone all packages of the named maintainer - -u, --unprivileged Clone package with read-only access and without - packager info as Git author --universe Clone all existing packages, useful for cache warming -h, --help Show this help text diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh index 5c99562..d6262b7 100644 --- a/src/lib/repo/configure.sh +++ b/src/lib/repo/configure.sh @@ -27,11 +27,12 @@ pkgctl_repo_configure_usage() { Git author information and the used signing key is set up from makepkg.conf read from any valid location like /etc or XDG_CONFIG_HOME. - The unprivileged option can be used for cloning packaging repositories - without SSH access using read-only HTTPS. + + The remote protocol is automatically determined from the author email + address by choosing SSH for all official packager identities and + read-only HTTPS otherwise. OPTIONS - -u, --unprivileged Configure read-only repo without packager info as Git author -h, --help Show this help text EXAMPLES @@ -39,16 +40,64 @@ pkgctl_repo_configure_usage() { _EOF_ } +get_packager_name() { + local packager=$1 + local packager_pattern="(.+) <(.+@.+)>" + local name + + if [[ ! $packager =~ $packager_pattern ]]; then + return 1 + fi + + name=$(echo "${packager}"|sed -E "s/${packager_pattern}/\1/") + printf "%s" "${name}" +} + +get_packager_email() { + local packager=$1 + local packager_pattern="(.+) <(.+@.+)>" + local email + + if [[ ! $packager =~ $packager_pattern ]]; then + return 1 + fi + + email=$(echo "${packager}"|sed -E "s/${packager_pattern}/\2/") + printf "%s" "${email}" +} + +is_packager_name_valid() { + local packager_name=$1 + if [[ -z ${packager_name} ]]; then + return 1 + elif [[ ${packager_name} == "John Doe" ]]; then + return 1 + elif [[ ${packager_name} == "Unknown Packager" ]]; then + return 1 + fi + return 0 +} + +is_packager_email_official() { + local packager_email=$1 + if [[ -z ${packager_email} ]]; then + return 1 + elif [[ $packager_email =~ .+@archlinux.org ]]; then + return 0 + fi + return 1 +} + pkgctl_repo_configure() { # options - local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_SSH} - local UNPRIVILEGED=0 - local PACKAGER_NAME= - local PACKAGER_EMAIL= + local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS} + local official=0 + local proto=https local paths=() # variables local path realpath pkgbase remote_url + local PACKAGER GPGKEY packager_name packager_email while (( $# )); do case $1 in @@ -56,11 +105,6 @@ pkgctl_repo_configure() { pkgctl_repo_configure_usage exit 0 ;; - -u|--unprivileged) - GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS} - UNPRIVILEGED=1 - shift - ;; --) shift break @@ -85,34 +129,33 @@ pkgctl_repo_configure() { fi fi - # Load makepkg.conf variables to be available + # Load makepkg.conf variables to be available for packager identity + msg "Collecting packager identity from makepkg.conf" # shellcheck disable=2119 load_makepkg_config - - # Check official packaging identity before setting Git author - if (( ! UNPRIVILEGED )); then - if [[ $PACKAGER == *"Unknown Packager"* ]]; then - die "Packager must be set in makepkg.conf" + if [[ -n ${PACKAGER} ]]; then + if ! packager_name=$(get_packager_name "${PACKAGER}") || \ + ! packager_email=$(get_packager_email "${PACKAGER}"); then + die "invalid PACKAGER format '${PACKAGER}' in makepkg.conf" fi - packager_pattern="(.+) <(.+@.+)>" - if [[ ! $PACKAGER =~ $packager_pattern ]]; then - die "Invalid Packager format '${PACKAGER}' in makepkg.conf" + if ! is_packager_name_valid "${packager_name}"; then + die "invalid PACKAGER '${PACKAGER}' in makepkg.conf" fi - - PACKAGER_NAME=$(echo "${PACKAGER}"|sed -E "s/${packager_pattern}/\1/") - PACKAGER_EMAIL=$(echo "${PACKAGER}"|sed -E "s/${packager_pattern}/\2/") - - if [[ ! $PACKAGER_EMAIL =~ .+@archlinux.org ]]; then - die "Packager email '${PACKAGER_EMAIL}' is not an @archlinux.org address" + if is_packager_email_official "${packager_email}"; then + official=1 + proto=ssh + GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_SSH} fi fi - msg "Collected packager settings" - msg2 "name : ${PACKAGER_NAME}" - msg2 "email : ${PACKAGER_EMAIL}" - msg2 "gpg-key : ${GPGKEY:-undefined}" - - # TODO: print which protocol got auto detected, ssh https + msg2 "name : ${packager_name:-${YELLOW}undefined${ALL_OFF}}" + msg2 "email : ${packager_email:-${YELLOW}undefined${ALL_OFF}}" + msg2 "gpg-key : ${GPGKEY:-${YELLOW}undefined${ALL_OFF}}" + if [[ ${proto} == ssh ]]; then + msg2 "protocol: ${GREEN}${proto}${ALL_OFF}" + else + msg2 "protocol: ${YELLOW}${proto}${ALL_OFF}" + fi for path in "${paths[@]}"; do if ! realpath=$(realpath -e "${path}"); then @@ -129,41 +172,51 @@ pkgctl_repo_configure() { continue fi + pushd "${path}" >/dev/null + remote_url="${GIT_REPO_BASE_URL}/${pkgbase}.git" - if ! git -C "${path}" remote add origin "${remote_url}" &>/dev/null; then - git -C "${path}" remote set-url origin "${remote_url}" + if ! git remote add origin "${remote_url}" &>/dev/null; then + git remote set-url origin "${remote_url}" fi # move the master branch to main - if [[ $(git -C "${path}" symbolic-ref --short HEAD) == master ]]; then - git -C "${path}" branch --move main - git -C "${path}" config branch.main.merge refs/heads/main + if [[ $(git symbolic-ref --quiet --short HEAD) == master ]]; then + git branch --move main + git config branch.main.merge refs/heads/main + fi + + git config devtools.version "${GIT_REPO_SPEC_VERSION}" + git config pull.rebase true + git config branch.autoSetupRebase always + git config branch.main.remote origin + git config branch.main.rebase true + + git config transfer.fsckobjects true + git config fetch.fsckobjects true + git config receive.fsckobjects true + + # setup author identity + if [[ -n ${packager_name} ]]; then + git config user.name "${packager_name}" + git config user.email "${packager_email}" + fi + + # force gpg for official packagers + if (( official )); then + git config commit.gpgsign true fi - git -C "${path}" config devtools.version "${GIT_REPO_SPEC_VERSION}" - git -C "${path}" config pull.rebase true - git -C "${path}" config branch.autoSetupRebase always - git -C "${path}" config branch.main.remote origin - git -C "${path}" config branch.main.rebase true - - git -C "${path}" config transfer.fsckobjects true - git -C "${path}" config fetch.fsckobjects true - git -C "${path}" config receive.fsckobjects true - - if (( ! UNPRIVILEGED )); then - git -C "${path}" config user.name "${PACKAGER_NAME}" - git -C "${path}" config user.email "${PACKAGER_EMAIL}" - git -C "${path}" config commit.gpgsign true - if [[ -n $GPGKEY ]]; then - git -C "${path}" config user.signingKey "${GPGKEY}" - else - warning "Missing makepkg.conf configuration: GPGKEY" - fi + # set custom pgp key from makepkg.conf + if [[ -n $GPGKEY ]]; then + git config commit.gpgsign true + git config user.signingKey "${GPGKEY}" fi if ! git ls-remote origin &>/dev/null; then warning "configured remote origin may not exist, run:" msg2 "pkgctl repo create ${pkgbase}" fi + + popd >/dev/null done } -- cgit v1.2.3-70-g09d2 From eda3a4aea00803d14400beff819d3b8be81774ce Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Mon, 30 Jan 2023 21:15:50 +0100 Subject: gitlab: add project path function to map special chars Automatic path conversion is limited to GitLab API v4 and will be removed in the future. It's expected that the caller does the path conversion on caller side and only passes a valid path to the API within its limitations. Hence convert project names to valid paths: 1. replace single '+' between word boundaries with '-' 2. replace any other '+' with literal 'plus' 3. replace any special chars other than '_', '-' and '.' with '-' 4. replace consecutive '_-' chars with a single '-' 5. replace 'tree' with 'unix-tree' due to GitLab reserved keyword Signed-off-by: Levente Polyak --- src/lib/api/gitlab.sh | 26 +++++++++++++++++++++++++- src/lib/repo/clone.sh | 6 +++++- src/lib/repo/configure.sh | 5 +++-- 3 files changed, 33 insertions(+), 4 deletions(-) (limited to 'src/lib/repo') diff --git a/src/lib/api/gitlab.sh b/src/lib/api/gitlab.sh index 649e205..e5f4237 100644 --- a/src/lib/api/gitlab.sh +++ b/src/lib/api/gitlab.sh @@ -81,16 +81,40 @@ gitlab_api_get_user() { return 0 } +# Convert arbitrary project names to GitLab valid path names. +# +# GitLab has several limitations on project and group names and also maintains +# a list of reserved keywords as documented on their docs. +# https://docs.gitlab.com/ee/user/reserved_names.html +# +# 1. replace single '+' between word boundaries with '-' +# 2. replace any other '+' with literal 'plus' +# 3. replace any special chars other than '_', '-' and '.' with '-' +# 4. replace consecutive '_-' chars with a single '-' +# 5. replace 'tree' with 'unix-tree' due to GitLab reserved keyword +gitlab_project_name_to_path() { + local name=$1 + printf "%s" "${name}" \ + | sed -E 's/([a-zA-Z0-9]+)\+([a-zA-Z]+)/\1-\2/g' \ + | sed -E 's/\+/plus/g' \ + | sed -E 's/[^a-zA-Z0-9_\-\.]/-/g' \ + | sed -E 's/[_\-]{2,}/-/g' \ + | sed -E 's/^tree$/unix-tree/g' +} + gitlab_api_create_project() { local pkgbase=$1 - local outfile data path + local outfile data path project_path [[ -z ${WORKDIR:-} ]] && setup_workdir outfile=$(mktemp --tmpdir="${WORKDIR}" pkgctl-gitlab-api.XXXXXXXXXX) + project_path=$(gitlab_project_name_to_path "${pkgbase}") + # create GitLab project data='{ "name": "'"${pkgbase}"'", + "path": "'"${project_path}"'", "namespace_id": "'"${GIT_PACKAGING_NAMESPACE_ID}"'", "request_access_enabled": "false" }' diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh index 4b26fcf..dee4870 100644 --- a/src/lib/repo/clone.sh +++ b/src/lib/repo/clone.sh @@ -54,6 +54,9 @@ pkgctl_repo_clone() { local CONFIGURE_OPTIONS=() local pkgbases + # variables + local project_path + while (( $# )); do case $1 in -h|--help) @@ -126,7 +129,8 @@ pkgctl_repo_clone() { for pkgbase in "${pkgbases[@]}"; do if [[ ! -d ${pkgbase} ]]; then msg "Cloning ${pkgbase} ..." - remote_url="${GIT_REPO_BASE_URL}/${pkgbase}.git" + project_path=$(gitlab_project_name_to_path "${pkgbase}") + remote_url="${GIT_REPO_BASE_URL}/${project_path}.git" git clone --origin origin "${remote_url}" "${pkgbase}" else warning "Skip cloning ${pkgbase}: Directory exists" diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh index d6262b7..942876a 100644 --- a/src/lib/repo/configure.sh +++ b/src/lib/repo/configure.sh @@ -96,7 +96,7 @@ pkgctl_repo_configure() { local paths=() # variables - local path realpath pkgbase remote_url + local path realpath pkgbase remote_url project_path local PACKAGER GPGKEY packager_name packager_email while (( $# )); do @@ -174,7 +174,8 @@ pkgctl_repo_configure() { pushd "${path}" >/dev/null - remote_url="${GIT_REPO_BASE_URL}/${pkgbase}.git" + project_path=$(gitlab_project_name_to_path "${pkgbase}") + remote_url="${GIT_REPO_BASE_URL}/${project_path}.git" if ! git remote add origin "${remote_url}" &>/dev/null; then git remote set-url origin "${remote_url}" fi -- cgit v1.2.3-70-g09d2 From ed966351410b39bfcec749df59dbc434a5dade1e Mon Sep 17 00:00:00 2001 From: Christian Heusel Date: Sat, 15 Apr 2023 19:44:22 +0200 Subject: pkgctl repo: introduce the switch subcommand Signed-off-by: Christian Heusel Co-Authored-By: Levente Polyak --- contrib/completion/bash/devtools.in | 19 ++++++ contrib/completion/zsh/_devtools.in | 8 +++ doc/man/pkgctl-repo-switch.1.asciidoc | 36 ++++++++++ doc/man/pkgctl-repo.1.asciidoc | 4 ++ src/lib/repo.sh | 10 +++ src/lib/repo/switch.sh | 119 ++++++++++++++++++++++++++++++++++ 6 files changed, 196 insertions(+) create mode 100644 doc/man/pkgctl-repo-switch.1.asciidoc create mode 100644 src/lib/repo/switch.sh (limited to 'src/lib/repo') diff --git a/contrib/completion/bash/devtools.in b/contrib/completion/bash/devtools.in index e3ae023..e79b862 100644 --- a/contrib/completion/bash/devtools.in +++ b/contrib/completion/bash/devtools.in @@ -255,6 +255,7 @@ _pkgctl_repo_cmds=( clone configure create + switch web ) @@ -282,6 +283,24 @@ _pkgctl_repo_create_args=( ) +_pkgctl_repo_switch_args=( + --discard-changes + -f --force + -h --help +) +_pkgctl_repo_switch_opts() { + local subcommand args + subcommand=(repo switch) + args=$(__pkgctl_word_count_after_subcommand "${subcommand[@]}") + + if (( args == 0 )); then + : + elif (( args >= 1 )); then + _filedir -d; + fi +} + + _pkgctl_repo_web_args=( -h --help ) diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in index 20c37ce..5760458 100644 --- a/contrib/completion/zsh/_devtools.in +++ b/contrib/completion/zsh/_devtools.in @@ -94,9 +94,17 @@ _pkgctl_repo_cmds=( "clone[Clone a package repository]" "configure[Configure a clone according to distro specs]" "create[Create a new GitLab package repository]" + "switch[Switch a package repository to a specified version]" "web[Open the packaging repository's website]" ) +_pkgctl_repo_switch_args=( + '(-f --force --discard-changes)'{-f,--force,--discard-changes}'[Discard changes if index or working tree is dirty]' + '(-h --help)'{-h,--help}'[Display usage]' + '1:version' + '*:git_dir:_files -/' +) + _pkgctl_repo_clone_args=( '(-m --maintainer=)'{-m,--maintainer=}'[Clone all packages of the named maintainer]:maintainer:' '--universe[Clone all existing packages, useful for cache warming]' diff --git a/doc/man/pkgctl-repo-switch.1.asciidoc b/doc/man/pkgctl-repo-switch.1.asciidoc new file mode 100644 index 0000000..ac12019 --- /dev/null +++ b/doc/man/pkgctl-repo-switch.1.asciidoc @@ -0,0 +1,36 @@ +pkgctl-repo-switch(1) +===================== + +Name +---- +pkgctl-repo-switch - Switch a package repository to a specified version + +Synopsis +-------- +pkgctl repo switch [OPTIONS] [VERSION] [PKGBASE]... + +Description +----------- + +Switch a package source repository to a specified version, tag or branch. +The working tree and the index are updated to match the specified ref. + +If a version identifier is specified in the pacman version format, that +identifier is automatically translated to the Git tag name accordingly. + +The current working directory is used if no PKGBASE is specified. + +Options +------- + +*--discard-changes*:: + Proceed even if the index or the working tree differs from HEAD. Both the + index and working tree are restored to match the switching target. + +*-f, --force*:: + An alias for '--discard-changes'. + +*-h, --help*:: + Show a help text + +include::include/footer.asciidoc[] diff --git a/doc/man/pkgctl-repo.1.asciidoc b/doc/man/pkgctl-repo.1.asciidoc index 630afd8..713b474 100644 --- a/doc/man/pkgctl-repo.1.asciidoc +++ b/doc/man/pkgctl-repo.1.asciidoc @@ -41,6 +41,9 @@ pkgctl repo configure:: pkgctl repo create:: Create a new GitLab package repository +pkgctl repo switch:: + Switch a package repository to a specified version + pkgctl repo web:: Open the packaging repository's website @@ -50,6 +53,7 @@ See Also linkman:pkgctl-repo-clone[1] linkman:pkgctl-repo-configure[1] linkman:pkgctl-repo-create[1] +linkman:pkgctl-repo-switch[1] linkman:pkgctl-repo-web[1] include::include/footer.asciidoc[] diff --git a/src/lib/repo.sh b/src/lib/repo.sh index 6b3817a..9f545e9 100644 --- a/src/lib/repo.sh +++ b/src/lib/repo.sh @@ -30,6 +30,7 @@ pkgctl_repo_usage() { clone Clone a package repository configure Configure a clone according to distro specs create Create a new GitLab package repository + switch Switch a package repository to a specified version web Open the packaging repository's website OPTIONS @@ -40,6 +41,7 @@ pkgctl_repo_usage() { $ ${COMMAND} clone --maintainer mynickname $ ${COMMAND} configure * $ ${COMMAND} create libfoo + $ ${COMMAND} switch 2:1.19.5-1 libfoo $ ${COMMAND} web linux _EOF_ } @@ -81,6 +83,14 @@ pkgctl_repo() { pkgctl_repo_create "$@" exit 0 ;; + switch) + _DEVTOOLS_COMMAND+=" $1" + shift + # shellcheck source=src/lib/repo/switch.sh + source "${_DEVTOOLS_LIBRARY_DIR}"/lib/repo/switch.sh + pkgctl_repo_switch "$@" + exit 0 + ;; web) _DEVTOOLS_COMMAND+=" $1" shift diff --git a/src/lib/repo/switch.sh b/src/lib/repo/switch.sh new file mode 100644 index 0000000..f411ac2 --- /dev/null +++ b/src/lib/repo/switch.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_REPO_SWITCH_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_REPO_SWITCH_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh + +source /usr/share/makepkg/util/message.sh + +set -e + + +pkgctl_repo_switch_usage() { + local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} + cat <<- _EOF_ + Usage: ${COMMAND} [OPTIONS] [VERSION] [PKGBASE]... + + Switch a package source repository to a specified version, tag or + branch. The working tree and the index are updated to match the + specified ref. + + If a version identifier is specified in the pacman version format, that + identifier is automatically translated to the Git tag name accordingly. + + The current working directory is used if no PKGBASE is specified. + + OPTIONS + --discard-changes Discard changes if index or working tree is dirty + -f, --force An alias for --discard-changes + -h, --help Show this help text + + EXAMPLES + $ ${COMMAND} 1.14.6-1 gopass gopass-jsonapi + $ ${COMMAND} --force 2:1.19.5-1 + $ ${COMMAND} main +_EOF_ +} + +pkgctl_repo_switch() { + if (( $# < 1 )); then + pkgctl_repo_switch_usage + exit 0 + fi + + # options + local VERSION + local GIT_REF + local GIT_CHECKOUT_OPTIONS=() + local paths path realpath pkgbase + + while (( $# )); do + case $1 in + -h|--help) + pkgctl_repo_switch_usage + exit 0 + ;; + -f|--force|--discard-changes) + GIT_CHECKOUT_OPTIONS+=("--force") + shift + ;; + --) + shift + break + ;; + -*) + # - is special to switch back to previous version + if [[ $1 != - ]]; then + die "invalid argument: %s" "$1" + fi + ;;& + *) + if [[ -n ${VERSION} ]]; then + break + fi + VERSION=$1 + shift + ;; + esac + done + + if [[ -z ${VERSION} ]]; then + error "missing positional argument 'VERSION'" + pkgctl_repo_switch_usage + exit 1 + fi + + GIT_REF="$(get_tag_from_pkgver "${VERSION}")" + paths=("$@") + + # check if invoked without any path from within a packaging repo + if (( ${#paths[@]} == 0 )); then + if [[ -f PKGBUILD ]]; then + paths=(".") + else + die "Not a package repository: $(realpath -- .)" + fi + fi + + for path in "${paths[@]}"; do + if ! realpath=$(realpath -e -- "${path}"); then + die "No such directory: ${path}" + fi + pkgbase=$(basename "${realpath}") + + if [[ ! -d "${path}/.git" ]]; then + error "Not a Git repository: ${path}" + continue + fi + + if ! git -C "${path}" checkout "${GIT_CHECKOUT_OPTIONS[@]}" "${GIT_REF}"; then + die "Failed to switch ${pkgbase} to version ${VERSION}" + fi + msg "Successfully switched ${pkgbase} to version ${VERSION}" + done +} -- cgit v1.2.3-70-g09d2 From 8e3b6bcc5b82b270f8d310865f14f2b0405eddd7 Mon Sep 17 00:00:00 2001 From: Christian Heusel Date: Fri, 14 Apr 2023 17:14:05 +0200 Subject: pkgctl repo clone: add option to switch working tree Add an option to call the switch command after clone. Switch to a specified version. The working tree and the index are updated to match the version. Signed-off-by: Christian Heusel Signed-off-by: Levente Polyak --- contrib/completion/bash/devtools.in | 2 ++ contrib/completion/zsh/_devtools.in | 1 + doc/man/pkgctl-repo-clone.1.asciidoc | 5 +++++ src/lib/repo/clone.sh | 20 ++++++++++++++++++++ 4 files changed, 28 insertions(+) (limited to 'src/lib/repo') diff --git a/contrib/completion/bash/devtools.in b/contrib/completion/bash/devtools.in index e79b862..31269dd 100644 --- a/contrib/completion/bash/devtools.in +++ b/contrib/completion/bash/devtools.in @@ -262,12 +262,14 @@ _pkgctl_repo_cmds=( _pkgctl_repo_clone_args=( -m --maintainer + --switch -u --unprivileged --universe -h --help ) _pkgctl_repo_clone_args__maintainer_opts() { :; } _pkgctl_repo_clone_args_m_opts() { _pkgctl_repo_clone_args__maintainer_opts; } +_pkgctl_repo_clone_args__switch_opts() { :; } _pkgctl_repo_clone_opts() { _devtools_completions_all_packages; } diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in index 5760458..ed52a22 100644 --- a/contrib/completion/zsh/_devtools.in +++ b/contrib/completion/zsh/_devtools.in @@ -107,6 +107,7 @@ _pkgctl_repo_switch_args=( _pkgctl_repo_clone_args=( '(-m --maintainer=)'{-m,--maintainer=}'[Clone all packages of the named maintainer]:maintainer:' + '--switch=[Switch the current working tree to a specified version]' '--universe[Clone all existing packages, useful for cache warming]' '(-h --help)'{-h,--help}'[Display usage]' '*:packages:_devtools_completions_all_packages' diff --git a/doc/man/pkgctl-repo-clone.1.asciidoc b/doc/man/pkgctl-repo-clone.1.asciidoc index a39fb37..8f3aef7 100644 --- a/doc/man/pkgctl-repo-clone.1.asciidoc +++ b/doc/man/pkgctl-repo-clone.1.asciidoc @@ -28,6 +28,10 @@ Options *--universe*:: Clone all existing packages, useful for cache warming +*--switch* 'VERSION':: + Switch to a specified version. The working tree and the index are updated to + match the version. + *-h, --help*:: Show a help text @@ -35,5 +39,6 @@ See Also -------- linkman:pkgctl-repo-configure[1] +linkman:pkgctl-repo-switch[1] include::include/footer.asciidoc[] diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh index dee4870..340aa69 100644 --- a/src/lib/repo/clone.sh +++ b/src/lib/repo/clone.sh @@ -32,12 +32,14 @@ pkgctl_repo_clone_usage() { OPTIONS -m, --maintainer=NAME Clone all packages of the named maintainer + --switch VERSION Switch the current working tree to a specified version --universe Clone all existing packages, useful for cache warming -h, --help Show this help text EXAMPLES $ ${COMMAND} libfoo linux libbar $ ${COMMAND} --maintainer mynickname + $ ${COMMAND} --switch 1:1.0-2 libfoo _EOF_ } @@ -51,6 +53,7 @@ pkgctl_repo_clone() { local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_SSH} local CLONE_ALL=0 local MAINTAINER= + local VERSION= local CONFIGURE_OPTIONS=() local pkgbases @@ -77,6 +80,19 @@ pkgctl_repo_clone() { MAINTAINER="${1#*=}" shift ;; + --switch) + (( $# <= 1 )) && die "missing argument for %s" "$1" + # shellcheck source=src/lib/repo/switch.sh + source "${_DEVTOOLS_LIBRARY_DIR}"/lib/repo/switch.sh + VERSION="$2" + shift 2 + ;; + --switch=*) + # shellcheck source=src/lib/repo/switch.sh + source "${_DEVTOOLS_LIBRARY_DIR}"/lib/repo/switch.sh + VERSION="${1#*=}" + shift + ;; --universe) CLONE_ALL=1 shift @@ -137,5 +153,9 @@ pkgctl_repo_clone() { fi pkgctl_repo_configure "${CONFIGURE_OPTIONS[@]}" "${pkgbase}" + + if [[ -n "${VERSION}" ]]; then + pkgctl_repo_switch "${VERSION}" "${pkgbase}" + fi done } -- cgit v1.2.3-70-g09d2 From 6ce666a1669235749c17d5c44d8a24dea4a135da Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Fri, 3 Feb 2023 00:58:59 +0100 Subject: feature(parallel): run up to N jobs in parallel for repo clone/configure Run up to N jobs in parallel. By default the number of jobs is equal to the number of available processing units. For sequential processing this option needs to be passed with 1. Signed-off-by: Levente Polyak --- contrib/completion/bash/devtools.in | 6 ++++++ contrib/completion/zsh/_devtools.in | 2 ++ doc/man/pkgctl-repo-clone.1.asciidoc | 5 +++++ doc/man/pkgctl-repo-configure.1.asciidoc | 5 +++++ src/lib/common.sh | 2 +- src/lib/repo/clone.sh | 31 +++++++++++++++++++++++++++++-- src/lib/repo/configure.sh | 27 +++++++++++++++++++++++---- 7 files changed, 71 insertions(+), 7 deletions(-) (limited to 'src/lib/repo') diff --git a/contrib/completion/bash/devtools.in b/contrib/completion/bash/devtools.in index b0a90e5..17f863f 100644 --- a/contrib/completion/bash/devtools.in +++ b/contrib/completion/bash/devtools.in @@ -268,17 +268,23 @@ _pkgctl_repo_clone_args=( --switch -u --unprivileged --universe + -j --jobs -h --help ) _pkgctl_repo_clone_args__maintainer_opts() { :; } _pkgctl_repo_clone_args_m_opts() { _pkgctl_repo_clone_args__maintainer_opts; } _pkgctl_repo_clone_args__switch_opts() { :; } +_pkgctl_repo_clone_args__jobs_opts() { :; } +_pkgctl_repo_clone_args_j_opts() { _pkgctl_repo_clone_args__jobs_opts; } _pkgctl_repo_clone_opts() { _devtools_completions_all_packages; } _pkgctl_repo_configure_args=( + -j --jobs -h --help ) +_pkgctl_repo_configure_args__jobs_opts() { :; } +_pkgctl_repo_configure_args_j_opts() { _pkgctl_repo_clone_args__jobs_opts; } _pkgctl_repo_configure_opts() { _filedir -d; } diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in index 45ffde3..240f781 100644 --- a/contrib/completion/zsh/_devtools.in +++ b/contrib/completion/zsh/_devtools.in @@ -110,11 +110,13 @@ _pkgctl_repo_clone_args=( '(-m --maintainer=)'{-m,--maintainer=}'[Clone all packages of the named maintainer]:maintainer:' '--switch=[Switch the current working tree to a specified version]' '--universe[Clone all existing packages, useful for cache warming]' + '(-j --jobs)'{-j,--jobs}'[Run up to N jobs in parallel (default: number of processing units)]:jobs:' '(-h --help)'{-h,--help}'[Display usage]' '*:packages:_devtools_completions_all_packages' ) _pkgctl_repo_configure_args=( + '(-j --jobs)'{-j,--jobs}'[Run up to N jobs in parallel (default: number of processing units)]:jobs:' '(-h --help)'{-h,--help}'[Display usage]' '*:git_dir:_files -/' ) diff --git a/doc/man/pkgctl-repo-clone.1.asciidoc b/doc/man/pkgctl-repo-clone.1.asciidoc index 487ec35..d6da062 100644 --- a/doc/man/pkgctl-repo-clone.1.asciidoc +++ b/doc/man/pkgctl-repo-clone.1.asciidoc @@ -32,6 +32,11 @@ Options Switch to a specified version. The working tree and the index are updated to match the version. +*-j, --jobs* 'N':: + Run up to N jobs in parallel. By default the number of jobs is equal to the + number of available processing units. For sequential processing this option + needs to be passed with 1. + *-h, --help*:: Show a help text diff --git a/doc/man/pkgctl-repo-configure.1.asciidoc b/doc/man/pkgctl-repo-configure.1.asciidoc index 4499ed6..1b07dc7 100644 --- a/doc/man/pkgctl-repo-configure.1.asciidoc +++ b/doc/man/pkgctl-repo-configure.1.asciidoc @@ -25,6 +25,11 @@ read-only HTTPS otherwise. Options ------- +*-j, --jobs* 'N':: + Run up to N jobs in parallel. By default the number of jobs is equal to the + number of available processing units. For sequential processing this option + needs to be passed with 1. + *-h, --help*:: Show a help text diff --git a/src/lib/common.sh b/src/lib/common.sh index 24479eb..3d1ee56 100644 --- a/src/lib/common.sh +++ b/src/lib/common.sh @@ -30,7 +30,7 @@ export GIT_PACKAGING_URL_HTTPS="https://${GITLAB_HOST}/${GIT_PACKAGING_NAMESPACE export PACKAGING_REPO_RELEASE_HOST=repos.archlinux.org # check if messages are to be printed using color -if [[ -t 2 && "$TERM" != dumb ]]; then +if [[ -t 2 && "$TERM" != dumb ]] || [[ ${DEVTOOLS_COLOR} == always ]]; then colorize else # shellcheck disable=2034 diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh index 340aa69..a02d799 100644 --- a/src/lib/repo/clone.sh +++ b/src/lib/repo/clone.sh @@ -34,6 +34,7 @@ pkgctl_repo_clone_usage() { -m, --maintainer=NAME Clone all packages of the named maintainer --switch VERSION Switch the current working tree to a specified version --universe Clone all existing packages, useful for cache warming + -j, --jobs N Run up to N jobs in parallel (default: $(nproc)) -h, --help Show this help text EXAMPLES @@ -55,9 +56,11 @@ pkgctl_repo_clone() { local MAINTAINER= local VERSION= local CONFIGURE_OPTIONS=() - local pkgbases + local jobs= + jobs=$(nproc) # variables + local command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} local project_path while (( $# )); do @@ -97,6 +100,11 @@ pkgctl_repo_clone() { CLONE_ALL=1 shift ;; + -j|--jobs) + (( $# <= 1 )) && die "missing argument for %s" "$1" + jobs=$2 + shift 2 + ;; --) shift break @@ -142,12 +150,31 @@ pkgctl_repo_clone() { stat_done fi + # parallelization + if [[ ${jobs} != 1 ]] && (( ${#pkgbases[@]} > 1 )); then + # force colors in parallel if parent process is colorized + if [[ -n ${BOLD} ]]; then + export DEVTOOLS_COLOR=always + fi + # assign command options + if [[ -n "${VERSION}" ]]; then + command+=" --switch '${VERSION}'" + fi + if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${pkgbases[@]}"; then + die 'Failed to clone some packages, please check the output' + exit 1 + fi + exit 0 + fi + for pkgbase in "${pkgbases[@]}"; do if [[ ! -d ${pkgbase} ]]; then msg "Cloning ${pkgbase} ..." project_path=$(gitlab_project_name_to_path "${pkgbase}") remote_url="${GIT_REPO_BASE_URL}/${project_path}.git" - git clone --origin origin "${remote_url}" "${pkgbase}" + if ! git clone --origin origin "${remote_url}" "${pkgbase}"; then + die 'failed to clone %s' "${pkgbase}" + fi else warning "Skip cloning ${pkgbase}: Directory exists" fi diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh index 942876a..a5708a0 100644 --- a/src/lib/repo/configure.sh +++ b/src/lib/repo/configure.sh @@ -33,6 +33,7 @@ pkgctl_repo_configure_usage() { read-only HTTPS otherwise. OPTIONS + -j, --jobs N Run up to N jobs in parallel (default: $(nproc)) -h, --help Show this help text EXAMPLES @@ -93,9 +94,12 @@ pkgctl_repo_configure() { local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS} local official=0 local proto=https + local jobs= + jobs=$(nproc) local paths=() # variables + local -r command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} local path realpath pkgbase remote_url project_path local PACKAGER GPGKEY packager_name packager_email @@ -105,6 +109,11 @@ pkgctl_repo_configure() { pkgctl_repo_configure_usage exit 0 ;; + -j|--jobs) + (( $# <= 1 )) && die "missing argument for %s" "$1" + jobs=$2 + shift 2 + ;; --) shift break @@ -157,10 +166,21 @@ pkgctl_repo_configure() { msg2 "protocol: ${YELLOW}${proto}${ALL_OFF}" fi + # parallelization + if [[ ${jobs} != 1 ]] && (( ${#paths[@]} > 1 )); then + if [[ -n ${BOLD} ]]; then + export DEVTOOLS_COLOR=always + fi + if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${paths[@]}"; then + die 'Failed to configure some packages, please check the output' + exit 1 + fi + exit 0 + fi + for path in "${paths[@]}"; do if ! realpath=$(realpath -e "${path}"); then - error "No such directory: ${path}" - continue + die "No such directory: ${path}" fi pkgbase=$(basename "${realpath}") @@ -168,8 +188,7 @@ pkgctl_repo_configure() { msg "Configuring ${pkgbase}" if [[ ! -d "${path}/.git" ]]; then - error "Not a Git repository: ${path}" - continue + die "Not a Git repository: ${path}" fi pushd "${path}" >/dev/null -- cgit v1.2.3-70-g09d2 From a08bc2acf49c68061284c7991d41dc78c46ae2b4 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Wed, 24 May 2023 02:40:52 +0200 Subject: feature(clone): add protocol option to force cloning over HTTPS This is a rather quick and simple implementation to override the current logic and force clone with HTTPS. Allowing to explicitly clone over HTTPS is currently required to unblock reproducible builds where no ssh keys and GitLab user accounts are set up as of now. Hence this quick solution comes into play to mitigate the regression on reproducible builds builders. Revisit the overall auto detection and protocol logic approach for a later release related to some ideas floating around in pending merge-requests. Signed-off-by: Levente Polyak --- contrib/completion/bash/devtools.in | 7 +++++++ contrib/completion/zsh/_devtools.in | 2 ++ doc/man/pkgctl-repo-clone.1.asciidoc | 3 +++ doc/man/pkgctl-repo-configure.1.asciidoc | 3 +++ src/lib/repo/clone.sh | 15 +++++++++++++-- src/lib/repo/configure.sh | 21 +++++++++++++++++++-- 6 files changed, 47 insertions(+), 4 deletions(-) (limited to 'src/lib/repo') diff --git a/contrib/completion/bash/devtools.in b/contrib/completion/bash/devtools.in index 17f863f..3faad27 100644 --- a/contrib/completion/bash/devtools.in +++ b/contrib/completion/bash/devtools.in @@ -265,6 +265,7 @@ _pkgctl_repo_cmds=( _pkgctl_repo_clone_args=( -m --maintainer + --protocol --switch -u --unprivileged --universe @@ -273,6 +274,7 @@ _pkgctl_repo_clone_args=( ) _pkgctl_repo_clone_args__maintainer_opts() { :; } _pkgctl_repo_clone_args_m_opts() { _pkgctl_repo_clone_args__maintainer_opts; } +_pkgctl_repo_clone_args__protocol_opts() { _devtools_completions_protocol; } _pkgctl_repo_clone_args__switch_opts() { :; } _pkgctl_repo_clone_args__jobs_opts() { :; } _pkgctl_repo_clone_args_j_opts() { _pkgctl_repo_clone_args__jobs_opts; } @@ -280,9 +282,11 @@ _pkgctl_repo_clone_opts() { _devtools_completions_all_packages; } _pkgctl_repo_configure_args=( + --protocol -j --jobs -h --help ) +_pkgctl_repo_configure_args__protocol_opts() { _devtools_completions_protocol; } _pkgctl_repo_configure_args__jobs_opts() { :; } _pkgctl_repo_configure_args_j_opts() { _pkgctl_repo_clone_args__jobs_opts; } _pkgctl_repo_configure_opts() { _filedir -d; } @@ -363,6 +367,9 @@ _devtools_completions_build_repo() { _devtools_completions_all_packages() { mapfile -t COMPREPLY < <(compgen -W "$(pacman -Sql)" -- "$cur") } +_devtools_completions_protocol() { + mapfile -t COMPREPLY < <(compgen -W "https" -- "$cur") +} __devtools_complete() { local service=$1 diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in index 240f781..a473bc2 100644 --- a/contrib/completion/zsh/_devtools.in +++ b/contrib/completion/zsh/_devtools.in @@ -108,6 +108,7 @@ _pkgctl_repo_switch_args=( _pkgctl_repo_clone_args=( '(-m --maintainer=)'{-m,--maintainer=}'[Clone all packages of the named maintainer]:maintainer:' + '--protocol[Clone the repository over https]:proto:(https)' '--switch=[Switch the current working tree to a specified version]' '--universe[Clone all existing packages, useful for cache warming]' '(-j --jobs)'{-j,--jobs}'[Run up to N jobs in parallel (default: number of processing units)]:jobs:' @@ -116,6 +117,7 @@ _pkgctl_repo_clone_args=( ) _pkgctl_repo_configure_args=( + '--protocol[Configure remote url to use https]:proto:(https)' '(-j --jobs)'{-j,--jobs}'[Run up to N jobs in parallel (default: number of processing units)]:jobs:' '(-h --help)'{-h,--help}'[Display usage]' '*:git_dir:_files -/' diff --git a/doc/man/pkgctl-repo-clone.1.asciidoc b/doc/man/pkgctl-repo-clone.1.asciidoc index d6da062..421c71f 100644 --- a/doc/man/pkgctl-repo-clone.1.asciidoc +++ b/doc/man/pkgctl-repo-clone.1.asciidoc @@ -25,6 +25,9 @@ Options *-m, --maintainer* 'NAME':: Clone all packages of the named maintainer +*--protocol* 'https':: + Clone the repository over https + *--universe*:: Clone all existing packages, useful for cache warming diff --git a/doc/man/pkgctl-repo-configure.1.asciidoc b/doc/man/pkgctl-repo-configure.1.asciidoc index 1b07dc7..6bdea93 100644 --- a/doc/man/pkgctl-repo-configure.1.asciidoc +++ b/doc/man/pkgctl-repo-configure.1.asciidoc @@ -25,6 +25,9 @@ read-only HTTPS otherwise. Options ------- +*--protocol* 'https':: + Configure remote url to use https + *-j, --jobs* 'N':: Run up to N jobs in parallel. By default the number of jobs is equal to the number of available processing units. For sequential processing this option diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh index a02d799..08bded4 100644 --- a/src/lib/repo/clone.sh +++ b/src/lib/repo/clone.sh @@ -26,12 +26,13 @@ pkgctl_repo_clone_usage() { Clone Git packaging repositories from the canonical namespace. The configure command is subsequently invoked to synchronize the distro - specs and makepkg.conf settings. The unprivileged option can be used + specs and makepkg.conf settings. The protocol option can be used for cloning packaging repositories without SSH access using read-only HTTPS. OPTIONS -m, --maintainer=NAME Clone all packages of the named maintainer + --protocol https Clone the repository over https --switch VERSION Switch the current working tree to a specified version --universe Clone all existing packages, useful for cache warming -j, --jobs N Run up to N jobs in parallel (default: $(nproc)) @@ -69,11 +70,21 @@ pkgctl_repo_clone() { pkgctl_repo_clone_usage exit 0 ;; - -u|--unprivileged) + --protocol=https) GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS} CONFIGURE_OPTIONS+=("$1") shift ;; + --protocol) + (( $# <= 1 )) && die "missing argument for %s" "$1" + if [[ $2 == https ]]; then + GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS} + else + die "unsupported protocol: %s" "$2" + fi + CONFIGURE_OPTIONS+=("$1" "$2") + shift 2 + ;; -m|--maintainer) (( $# <= 1 )) && die "missing argument for %s" "$1" MAINTAINER="$2" diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh index a5708a0..81b7d19 100644 --- a/src/lib/repo/configure.sh +++ b/src/lib/repo/configure.sh @@ -33,6 +33,7 @@ pkgctl_repo_configure_usage() { read-only HTTPS otherwise. OPTIONS + --protocol https Configure remote url to use https -j, --jobs N Run up to N jobs in parallel (default: $(nproc)) -h, --help Show this help text @@ -94,6 +95,7 @@ pkgctl_repo_configure() { local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS} local official=0 local proto=https + local proto_force=0 local jobs= jobs=$(nproc) local paths=() @@ -109,6 +111,19 @@ pkgctl_repo_configure() { pkgctl_repo_configure_usage exit 0 ;; + --protocol=https) + proto_force=1 + shift + ;; + --protocol) + (( $# <= 1 )) && die "missing argument for %s" "$1" + if [[ $2 == https ]]; then + proto_force=1 + else + die "unsupported protocol: %s" "$2" + fi + shift 2 + ;; -j|--jobs) (( $# <= 1 )) && die "missing argument for %s" "$1" jobs=$2 @@ -152,8 +167,10 @@ pkgctl_repo_configure() { fi if is_packager_email_official "${packager_email}"; then official=1 - proto=ssh - GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_SSH} + if (( ! proto_force )); then + proto=ssh + GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_SSH} + fi fi fi -- cgit v1.2.3-70-g09d2 From 9b11b16a7e6cdfdff142ff1c5184a760673988d2 Mon Sep 17 00:00:00 2001 From: "Daniel M. Capella" Date: Fri, 26 May 2023 22:41:37 -0400 Subject: chore(doc): remove duplicate subcommand from example help text --- src/lib/repo/configure.sh | 2 +- src/lib/repo/web.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/repo') diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh index 81b7d19..73300ae 100644 --- a/src/lib/repo/configure.sh +++ b/src/lib/repo/configure.sh @@ -38,7 +38,7 @@ pkgctl_repo_configure_usage() { -h, --help Show this help text EXAMPLES - $ ${COMMAND} configure * + $ ${COMMAND} * _EOF_ } diff --git a/src/lib/repo/web.sh b/src/lib/repo/web.sh index 3fa214d..45ea53b 100644 --- a/src/lib/repo/web.sh +++ b/src/lib/repo/web.sh @@ -26,7 +26,7 @@ pkgctl_repo_web_usage() { -h, --help Show this help text EXAMPLES - $ ${COMMAND} web linux + $ ${COMMAND} linux _EOF_ } -- cgit v1.2.3-70-g09d2