From be5f54c95cbbcf46598e23aa456075cbb26806c0 Mon Sep 17 00:00:00 2001 From: Christian Heusel Date: Wed, 14 Jun 2023 00:25:03 +0200 Subject: fix(commitpkg): ensure the PKGBUILD is version controlled Early exit in case the PKGBUILD is not yet properly under version control, which can happen for freshly initialized repositories. Furthermore print an appropriate error message including a hint how to resolve this. Fixes #154 Fixes #167 Signed-off-by: Christian Heusel Signed-off-by: Levente Polyak --- src/commitpkg.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/commitpkg.in') diff --git a/src/commitpkg.in b/src/commitpkg.in index 8a8087a..108fe81 100644 --- a/src/commitpkg.in +++ b/src/commitpkg.in @@ -111,7 +111,7 @@ if (( ${#validpgpkeys[@]} != 0 )); then fi # find files which should be under source control -needsversioning=() +needsversioning=(PKGBUILD) for s in "${source[@]}"; do [[ $s != *://* ]] && needsversioning+=("$s") done -- cgit v1.2.3-54-g00ecf From 8190a22480ac6d808b9d648f87e89be4a01926ef Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Fri, 21 Jul 2023 16:04:17 +0200 Subject: feat(release): always generate a .SRCINFO file on release When releasing a package always generate a .SRCINFO file so we can build tooling based upon these files instead of having to parse bash in a non-bash language. Some tests have shown around 30 seconds or generating the metadata on packages like thunderbird. The implementation in makepkg becomes semi slow for some split packages that provide a huge number of individual packages, but it seems reasonably fast so we can instead have a uniform state of always providing .SRCINFO files. Signed-off-by: Levente Polyak --- src/commitpkg.in | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'src/commitpkg.in') diff --git a/src/commitpkg.in b/src/commitpkg.in index 108fe81..f979d61 100644 --- a/src/commitpkg.in +++ b/src/commitpkg.in @@ -7,6 +7,7 @@ _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh source /usr/share/makepkg/util/util.sh +source /usr/share/makepkg/srcinfo.sh check_pkgbuild_validity() { @@ -177,13 +178,11 @@ done # check for PKGBUILD standards check_pkgbuild_validity -# auto generate .SRCINFO if present -if [[ -f .SRCINFO ]]; then - stat_busy 'Generating .SRCINFO' - makepkg --printsrcinfo > .SRCINFO - git add .SRCINFO - stat_done -fi +# auto generate .SRCINFO +stat_busy 'Generating .SRCINFO' +write_srcinfo_content > .SRCINFO +git add --force .SRCINFO +stat_done if [[ -n $(git status --porcelain --untracked-files=no) ]]; then stat_busy 'Staging files' -- cgit v1.2.3-54-g00ecf From c2d73d73aec9c21ffdb0677a7e3ef96a42c1ba47 Mon Sep 17 00:00:00 2001 From: Christian Heusel Date: Mon, 9 Oct 2023 16:24:50 +0200 Subject: fix(commitpkg): prefer core.editor over $EDITOR and $VISUAL This is done to closer mimick the behaviour of git here, as it prefers core.editor setting over the other editor options as per git-var(1): > The order of preference is the $GIT_EDITOR environment variable, then > core.editor configuration, then $VISUAL, then $EDITOR, and then the > default chosen at compile time, which is usually vi. Fixes #192 Signed-off-by: Christian Heusel --- src/commitpkg.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/commitpkg.in') diff --git a/src/commitpkg.in b/src/commitpkg.in index f979d61..6d4b014 100644 --- a/src/commitpkg.in +++ b/src/commitpkg.in @@ -205,14 +205,14 @@ if [[ -n $(git status --porcelain --untracked-files=no) ]]; then echo "$msgtemplate" > "$msgfile" if [[ -n $GIT_EDITOR ]]; then $GIT_EDITOR "$msgfile" || die + elif giteditor=$(git config --get core.editor); then + $giteditor "$msgfile" || die elif [[ -n $VISUAL ]]; then $VISUAL "$msgfile" || die elif [[ -n $EDITOR ]]; then $EDITOR "$msgfile" || die - elif giteditor=$(git config --get core.editor); then - $giteditor "$msgfile" || die else - die "No usable editor found (tried \$GIT_EDITOR, \$VISUAL, \$EDITOR, git config [core.editor])." + die "No usable editor found (tried \$GIT_EDITOR, git config [core.editor], \$VISUAL, \$EDITOR)." fi [[ -s $msgfile ]] || die stat_busy 'Committing changes' -- cgit v1.2.3-54-g00ecf From d00a2b989044fe452792e108a15b6c901a1a1d4c Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sun, 11 Jun 2023 23:31:18 +0200 Subject: feat(configure): ship default ignore via exclude BREAKING CHANGE: Increments the repo spec version which requires to reconfigure all existing packaging repo clones. Fixes #129 Component: pkgctl repo configure Signed-off-by: Levente Polyak --- Makefile | 8 +++++++- config/git/template/hooks/applypatch-msg | 1 + config/git/template/hooks/commit-msg | 1 + config/git/template/hooks/post-applypatch | 1 + config/git/template/hooks/post-checkout | 1 + config/git/template/hooks/post-commit | 1 + config/git/template/hooks/post-merge | 1 + config/git/template/hooks/pre-applypatch | 1 + config/git/template/hooks/pre-commit | 1 + config/git/template/hooks/pre-merge-commit | 1 + config/git/template/hooks/pre-push | 1 + config/git/template/hooks/pre-rebase | 1 + config/git/template/hooks/prepare-commit-msg | 1 + config/git/template/info/exclude | 28 ++++++++++++++++++++++++++++ doc/man/pkgctl-repo-configure.1.asciidoc | 2 ++ src/commitpkg.in | 6 ++++++ src/lib/common.sh | 2 +- src/lib/repo/configure.sh | 25 ++++++++++++++++++++++++- 18 files changed, 80 insertions(+), 3 deletions(-) create mode 100755 config/git/template/hooks/applypatch-msg create mode 100755 config/git/template/hooks/commit-msg create mode 100755 config/git/template/hooks/post-applypatch create mode 100755 config/git/template/hooks/post-checkout create mode 100755 config/git/template/hooks/post-commit create mode 100755 config/git/template/hooks/post-merge create mode 100755 config/git/template/hooks/pre-applypatch create mode 100755 config/git/template/hooks/pre-commit create mode 100755 config/git/template/hooks/pre-merge-commit create mode 100755 config/git/template/hooks/pre-push create mode 100755 config/git/template/hooks/pre-rebase create mode 100755 config/git/template/hooks/prepare-commit-msg create mode 100644 config/git/template/info/exclude (limited to 'src/commitpkg.in') diff --git a/Makefile b/Makefile index 36f1b9b..afdb602 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,7 @@ LIBRARY_SRC = $(call rwildcard,src/lib,*.sh) LIBRARY = $(addprefix $(BUILDDIR)/,$(patsubst src/%,%,$(patsubst %.in,%,$(LIBRARY_SRC)))) MAKEPKG_CONFIGS=$(wildcard config/makepkg/*) PACMAN_CONFIGS=$(wildcard config/pacman/*) +GIT_CONFIGS = $(wildcard config/git/*) SETARCH_ALIASES = $(wildcard config/setarch-aliases.d/*) MANS = $(addprefix $(BUILDDIR)/,$(patsubst %.asciidoc,%,$(wildcard doc/man/*.asciidoc))) @@ -97,9 +98,12 @@ $(BUILDDIR)/doc/man/%: doc/man/%.asciidoc doc/asciidoc.conf doc/man/include/foot @a2x --no-xmllint --asciidoc-opts="-f doc/asciidoc.conf" -d manpage -f manpage --destination-dir=$(BUILDDIR)/doc/man -a pkgdatadir=$(DATADIR) $< conf: - @install -d $(BUILDDIR)/makepkg.conf.d $(BUILDDIR)/pacman.conf.d + @install -d $(BUILDDIR)/makepkg.conf.d @cp -a $(MAKEPKG_CONFIGS) $(BUILDDIR)/makepkg.conf.d + @install -d $(BUILDDIR)/pacman.conf.d @cp -a $(PACMAN_CONFIGS) $(BUILDDIR)/pacman.conf.d + @install -d $(BUILDDIR)/git.conf.d + cp -a $(GIT_CONFIGS) $(BUILDDIR)/git.conf.d clean: rm -rf $(BUILDDIR) @@ -112,6 +116,7 @@ install: all install -m0755 ${BINPROGS} $(DESTDIR)$(PREFIX)/bin install -dm0755 $(DESTDIR)$(DATADIR)/lib cp -ra $(BUILDDIR)/lib/* $(DESTDIR)$(DATADIR)/lib + cp -a $(BUILDDIR)/git.conf.d -t $(DESTDIR)$(DATADIR) for conf in $(notdir $(MAKEPKG_CONFIGS)); do install -Dm0644 $(BUILDDIR)/makepkg.conf.d/$$conf $(DESTDIR)$(DATADIR)/makepkg.conf.d/$${conf##*/}; done for conf in $(notdir $(PACMAN_CONFIGS)); do install -Dm0644 $(BUILDDIR)/pacman.conf.d/$$conf $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done for a in ${SETARCH_ALIASES}; do install -m0644 $$a -t $(DESTDIR)$(DATADIR)/setarch-aliases.d; done @@ -129,6 +134,7 @@ uninstall: for f in $(notdir $(BINPROGS)); do rm -f $(DESTDIR)$(PREFIX)/bin/$$f; done for f in $(notdir $(LIBRARY)); do rm -f $(DESTDIR)$(DATADIR)/lib/$$f; done rm -rf $(DESTDIR)$(DATADIR)/lib + rm -rf $(DESTDIR)$(DATADIR)/git.conf.d for conf in $(notdir $(MAKEPKG_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/makepkg.conf.d/$${conf##*/}; done for conf in $(notdir $(PACMAN_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done for f in $(notdir $(SETARCH_ALIASES)); do rm -f $(DESTDIR)$(DATADIR)/setarch-aliases.d/$$f; done diff --git a/config/git/template/hooks/applypatch-msg b/config/git/template/hooks/applypatch-msg new file mode 100755 index 0000000..1a24852 --- /dev/null +++ b/config/git/template/hooks/applypatch-msg @@ -0,0 +1 @@ +#!/bin/sh diff --git a/config/git/template/hooks/commit-msg b/config/git/template/hooks/commit-msg new file mode 100755 index 0000000..1a24852 --- /dev/null +++ b/config/git/template/hooks/commit-msg @@ -0,0 +1 @@ +#!/bin/sh diff --git a/config/git/template/hooks/post-applypatch b/config/git/template/hooks/post-applypatch new file mode 100755 index 0000000..1a24852 --- /dev/null +++ b/config/git/template/hooks/post-applypatch @@ -0,0 +1 @@ +#!/bin/sh diff --git a/config/git/template/hooks/post-checkout b/config/git/template/hooks/post-checkout new file mode 100755 index 0000000..1a24852 --- /dev/null +++ b/config/git/template/hooks/post-checkout @@ -0,0 +1 @@ +#!/bin/sh diff --git a/config/git/template/hooks/post-commit b/config/git/template/hooks/post-commit new file mode 100755 index 0000000..1a24852 --- /dev/null +++ b/config/git/template/hooks/post-commit @@ -0,0 +1 @@ +#!/bin/sh diff --git a/config/git/template/hooks/post-merge b/config/git/template/hooks/post-merge new file mode 100755 index 0000000..1a24852 --- /dev/null +++ b/config/git/template/hooks/post-merge @@ -0,0 +1 @@ +#!/bin/sh diff --git a/config/git/template/hooks/pre-applypatch b/config/git/template/hooks/pre-applypatch new file mode 100755 index 0000000..1a24852 --- /dev/null +++ b/config/git/template/hooks/pre-applypatch @@ -0,0 +1 @@ +#!/bin/sh diff --git a/config/git/template/hooks/pre-commit b/config/git/template/hooks/pre-commit new file mode 100755 index 0000000..1a24852 --- /dev/null +++ b/config/git/template/hooks/pre-commit @@ -0,0 +1 @@ +#!/bin/sh diff --git a/config/git/template/hooks/pre-merge-commit b/config/git/template/hooks/pre-merge-commit new file mode 100755 index 0000000..1a24852 --- /dev/null +++ b/config/git/template/hooks/pre-merge-commit @@ -0,0 +1 @@ +#!/bin/sh diff --git a/config/git/template/hooks/pre-push b/config/git/template/hooks/pre-push new file mode 100755 index 0000000..1a24852 --- /dev/null +++ b/config/git/template/hooks/pre-push @@ -0,0 +1 @@ +#!/bin/sh diff --git a/config/git/template/hooks/pre-rebase b/config/git/template/hooks/pre-rebase new file mode 100755 index 0000000..1a24852 --- /dev/null +++ b/config/git/template/hooks/pre-rebase @@ -0,0 +1 @@ +#!/bin/sh diff --git a/config/git/template/hooks/prepare-commit-msg b/config/git/template/hooks/prepare-commit-msg new file mode 100755 index 0000000..1a24852 --- /dev/null +++ b/config/git/template/hooks/prepare-commit-msg @@ -0,0 +1 @@ +#!/bin/sh diff --git a/config/git/template/info/exclude b/config/git/template/info/exclude new file mode 100644 index 0000000..ba46529 --- /dev/null +++ b/config/git/template/info/exclude @@ -0,0 +1,28 @@ +/pkg +/src +/*/ +!/keys/ + +/*.log +/*.tar.* +/*.tar +/*.tgz +/*.zst +/*.gz +/*.xz +/*.bz2 +/*.zip +/*.xpi +/*.jar +/*.whl +/*.war +/*.deb +/*.ttf +/*.dat +/*.iso +/*.asc +/*.sig +/*.signature +/*.sign +/*.SHA256SUMS +/*.sha256 diff --git a/doc/man/pkgctl-repo-configure.1.asciidoc b/doc/man/pkgctl-repo-configure.1.asciidoc index 6bdea93..12d879c 100644 --- a/doc/man/pkgctl-repo-configure.1.asciidoc +++ b/doc/man/pkgctl-repo-configure.1.asciidoc @@ -22,6 +22,8 @@ The remote protocol is automatically determined from the author email address by choosing SSH for all official packager identities and read-only HTTPS otherwise. +Git default excludes and hooks are applied to the configured repo. + Options ------- diff --git a/src/commitpkg.in b/src/commitpkg.in index 6d4b014..016ab22 100644 --- a/src/commitpkg.in +++ b/src/commitpkg.in @@ -71,6 +71,12 @@ if ! repo_spec=$(git config --local devtools.version) || [[ ${repo_spec} != "${G exit 1 fi +if ! repo_variant=$(git config --local devtools.variant) || [[ ${repo_variant} != canonical ]]; then + error "cannot release from a repository with none canonical specs (%s), try:" "${repo_variant:-development}" + msg2 'pkgctl repo configure' + exit 1 +fi + if [[ "$(git symbolic-ref --short HEAD)" != main ]]; then die 'must be run from the main branch' fi diff --git a/src/lib/common.sh b/src/lib/common.sh index 7589120..82fdd2e 100644 --- a/src/lib/common.sh +++ b/src/lib/common.sh @@ -22,7 +22,7 @@ export BUILDTOOLVER=@buildtoolver@ # Set common properties export PACMAN_KEYRING_DIR=/etc/pacman.d/gnupg export GITLAB_HOST=gitlab.archlinux.org -export GIT_REPO_SPEC_VERSION=1 +export GIT_REPO_SPEC_VERSION=2 export GIT_PACKAGING_NAMESPACE=archlinux/packaging/packages export GIT_PACKAGING_NAMESPACE_ID=11323 export GIT_PACKAGING_URL_SSH="git@${GITLAB_HOST}:${GIT_PACKAGING_NAMESPACE}" diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh index 1ddb1b8..b3c188c 100644 --- a/src/lib/repo/configure.sh +++ b/src/lib/repo/configure.sh @@ -17,6 +17,7 @@ source /usr/share/makepkg/util/config.sh source /usr/share/makepkg/util/message.sh set -e +shopt -s nullglob pkgctl_repo_configure_usage() { @@ -34,6 +35,8 @@ pkgctl_repo_configure_usage() { address by choosing SSH for all official packager identities and read-only HTTPS otherwise. + Git default excludes and hooks are applied to the configured repo. + OPTIONS --protocol https Configure remote url to use https -j, --jobs N Run up to N jobs in parallel (default: $(nproc)) @@ -104,7 +107,7 @@ pkgctl_repo_configure() { # variables local -r command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} - local path realpath pkgbase remote_url project_path + local path realpath pkgbase remote_url project_path hook local PACKAGER GPGKEY packager_name packager_email while (( $# )); do @@ -230,7 +233,15 @@ pkgctl_repo_configure() { git config branch.main.merge refs/heads/main fi + # configure spec version and variant to avoid using development hooks in production git config devtools.version "${GIT_REPO_SPEC_VERSION}" + if [[ ${_DEVTOOLS_LIBRARY_DIR} == /usr/share/devtools ]]; then + git config devtools.variant canonical + else + warning "Configuring with development version of pkgctl, do not use this repo in production" + git config devtools.variant development + fi + git config pull.rebase true git config branch.autoSetupRebase always git config branch.main.remote origin @@ -257,6 +268,18 @@ pkgctl_repo_configure() { git config user.signingKey "${GPGKEY}" fi + # set default git exclude + mkdir -p .git/info + ln -sf "${_DEVTOOLS_LIBRARY_DIR}/git.conf.d/template/info/exclude" \ + .git/info/exclude + + # set default git hooks + mkdir -p .git/hooks + rm -f .git/hooks/*.sample + for hook in "${_DEVTOOLS_LIBRARY_DIR}"/git.conf.d/template/hooks/*; do + ln -sf "${hook}" ".git/hooks/$(basename "${hook}")" + done + if ! git ls-remote origin &>/dev/null; then warning "configured remote origin may not exist, run:" msg2 "pkgctl repo create ${pkgbase}" -- cgit v1.2.3-54-g00ecf From b264c7f1c771790c53561f5d3f4de589e0000c05 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sun, 10 Sep 2023 00:00:27 +0200 Subject: feat(util): parallelize srcinfo generation Heavily improve the runtime of huge split packages, by creating an own parallelized high level implementation of the makepkg low level building blocks for srcinfo generation. This reduces the runtine to generate the srcinfo file for thunderbird from 24 seconds down to 1 second. --- src/commitpkg.in | 10 ++++--- src/lib/build/build.sh | 5 ++-- src/lib/util/srcinfo.sh | 69 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 src/lib/util/srcinfo.sh (limited to 'src/commitpkg.in') diff --git a/src/commitpkg.in b/src/commitpkg.in index 016ab22..e17b270 100644 --- a/src/commitpkg.in +++ b/src/commitpkg.in @@ -5,9 +5,12 @@ _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} # shellcheck source=src/lib/common.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/util/srcinfo.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/srcinfo.sh source /usr/share/makepkg/util/util.sh -source /usr/share/makepkg/srcinfo.sh + +set -eo pipefail check_pkgbuild_validity() { @@ -185,10 +188,9 @@ done check_pkgbuild_validity # auto generate .SRCINFO -stat_busy 'Generating .SRCINFO' -write_srcinfo_content > .SRCINFO +# shellcheck disable=SC2119 +write_srcinfo_file git add --force .SRCINFO -stat_done if [[ -n $(git status --porcelain --untracked-files=no) ]]; then stat_busy 'Staging files' diff --git a/src/lib/build/build.sh b/src/lib/build/build.sh index e0982e4..8b4240c 100644 --- a/src/lib/build/build.sh +++ b/src/lib/build/build.sh @@ -14,6 +14,8 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/db/update.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/release.sh # shellcheck source=src/lib/util/git.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh +# shellcheck source=src/lib/util/srcinfo.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/srcinfo.sh # shellcheck source=src/lib/util/pacman.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/pacman.sh # shellcheck source=src/lib/valid-repos.sh @@ -26,8 +28,7 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-inspect.sh source /usr/share/makepkg/util/config.sh source /usr/share/makepkg/util/message.sh -set -e -set -o pipefail +set -eo pipefail pkgctl_build_usage() { diff --git a/src/lib/util/srcinfo.sh b/src/lib/util/srcinfo.sh new file mode 100644 index 0000000..b646dc3 --- /dev/null +++ b/src/lib/util/srcinfo.sh @@ -0,0 +1,69 @@ +#!/hint/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_UTIL_SRCINFO_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_UTIL_SRCINFO_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh + +source /usr/share/makepkg/util/util.sh +source /usr/share/makepkg/srcinfo.sh + +set -eo pipefail + + +print_srcinfo() { + local pkgpath=${1:-.} + local outdir pkg pid + local pids=() + + # source the PKGBUILD + # shellcheck source=contrib/makepkg/PKGBUILD.proto + . "${pkgpath}"/PKGBUILD + + # run without parallelization for single packages + if (( ${#pkgname[@]} == 1 )); then + write_srcinfo_content + return 0 + fi + + [[ -z ${WORKDIR:-} ]] && setup_workdir + outdir=$(mktemp --directory --tmpdir="${WORKDIR}" pkgctl-srcinfo.XXXXXXXXXX) + + # fork workload for each split pkgname + for pkg in "${pkgname[@]}"; do + ( + # deactivate errexit to avoid makepkg abort on grep_function + set +e + srcinfo_write_package "$pkg" > "${outdir}/${pkg}" + )& + pids+=($!) + done + + # join workload + for pid in "${pids[@]}"; do + if ! wait "${pid}"; then + return 1 + fi + done + + # collect output + srcinfo_write_global + for pkg in "${pkgname[@]}"; do + srcinfo_separate_section + cat "${outdir}/${pkg}" + done +} + +write_srcinfo_file() { + local pkgpath=${1:-.} + stat_busy 'Generating .SRCINFO' + if ! print_srcinfo "${pkgpath}" > "${pkgpath}"/.SRCINFO; then + error 'Failed to write .SRCINFO file' + return 1 + fi + stat_done +} -- cgit v1.2.3-54-g00ecf