From d94badcd0be4f1f0bdc85a9e17f622373fcc42b2 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Wed, 18 May 2022 02:31:26 +0200 Subject: make: split out source files into src folder --- src/arch-nspawn.in | 130 ++++++++++++++ src/archbuild.in | 98 +++++++++++ src/archco.in | 26 +++ src/archrelease.in | 87 ++++++++++ src/checkpkg.in | 155 +++++++++++++++++ src/commitpkg.in | 241 ++++++++++++++++++++++++++ src/crossrepomove.in | 86 +++++++++ src/diffpkg.in | 228 ++++++++++++++++++++++++ src/export-pkgbuild-keys.in | 75 ++++++++ src/find-libdeps.in | 89 ++++++++++ src/finddeps.in | 41 +++++ src/lddd.in | 49 ++++++ src/makechrootpkg.in | 414 ++++++++++++++++++++++++++++++++++++++++++++ src/makerepropkg.in | 270 +++++++++++++++++++++++++++++ src/mkarchroot.in | 95 ++++++++++ src/offload-build.in | 121 +++++++++++++ src/rebuildpkgs.in | 111 ++++++++++++ src/sogrep.in | 170 ++++++++++++++++++ 18 files changed, 2486 insertions(+) create mode 100644 src/arch-nspawn.in create mode 100644 src/archbuild.in create mode 100644 src/archco.in create mode 100644 src/archrelease.in create mode 100644 src/checkpkg.in create mode 100644 src/commitpkg.in create mode 100644 src/crossrepomove.in create mode 100644 src/diffpkg.in create mode 100644 src/export-pkgbuild-keys.in create mode 100644 src/find-libdeps.in create mode 100644 src/finddeps.in create mode 100644 src/lddd.in create mode 100644 src/makechrootpkg.in create mode 100644 src/makerepropkg.in create mode 100644 src/mkarchroot.in create mode 100644 src/offload-build.in create mode 100644 src/rebuildpkgs.in create mode 100644 src/sogrep.in (limited to 'src') diff --git a/src/arch-nspawn.in b/src/arch-nspawn.in new file mode 100644 index 0000000..275cff7 --- /dev/null +++ b/src/arch-nspawn.in @@ -0,0 +1,130 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +m4_include(lib/common.sh) +m4_include(lib/archroot.sh) + +# umask might have been changed in /etc/profile +# ensure that sane default is set again +umask 0022 + +working_dir='' + +files=() +mount_args=() + +usage() { + echo "Usage: ${0##*/} [options] working-dir [systemd-nspawn arguments]" + echo "A wrapper around systemd-nspawn. Provides support for pacman." + echo + echo ' options:' + echo ' -C Location of a pacman config file' + echo ' -M Location of a makepkg config file' + echo ' -c Set pacman cache' + echo ' -f Copy file from the host to the chroot' + echo ' -s Do not run setarch' + echo ' -h This message' + exit 1 +} + +while getopts 'hC:M:c:f:s' arg; do + case "$arg" in + C) pac_conf="$OPTARG" ;; + M) makepkg_conf="$OPTARG" ;; + c) cache_dirs+=("$OPTARG") ;; + f) files+=("$OPTARG") ;; + s) nosetarch=1 ;; + h|?) usage ;; + *) error "invalid argument '%s'" "$arg"; usage ;; + esac +done +shift $((OPTIND - 1)) + +(( $# < 1 )) && die 'You must specify a directory.' +check_root + +working_dir=$(readlink -f "$1") +shift 1 + +[[ -z $working_dir ]] && die 'Please specify a working directory.' + +if (( ${#cache_dirs[@]} == 0 )); then + mapfile -t cache_dirs < <(pacman-conf --config "${pac_conf:-$working_dir/etc/pacman.conf}" CacheDir) +fi + +# shellcheck disable=2016 +host_mirrors=($(pacman-conf --repo extra Server 2> /dev/null | sed -r 's#(.*/)extra/os/.*#\1$repo/os/$arch#')) + +for host_mirror in "${host_mirrors[@]}"; do + if [[ $host_mirror == *file://* ]]; then + host_mirror=$(echo "$host_mirror" | sed -r 's#file://(/.*)/\$repo/os/\$arch#\1#g') + for m in "$host_mirror"/pool/*/; do + in_array "$m" "${cache_dirs[@]}" || cache_dirs+=("$m") + done + fi +done + +while read -r line; do + mapfile -t lines < <(pacman-conf --config "${pac_conf:-$working_dir/etc/pacman.conf}" \ + --repo $line Server | sed -r 's#(.*/)[^/]+/os/.+#\1#') + for line in "${lines[@]}"; do + if [[ $line = file://* ]]; then + line=${line#file://} + in_array "$line" "${cache_dirs[@]}" || cache_dirs+=("$line") + fi + done +done < <(pacman-conf --config "${pac_conf:-$working_dir/etc/pacman.conf}" --repo-list) + +mount_args+=("--bind=${cache_dirs[0]//:/\\:}") + +for cache_dir in "${cache_dirs[@]:1}"; do + mount_args+=("--bind-ro=${cache_dir//:/\\:}") +done + +# {{{ functions +copy_hostconf () { + unshare --fork --pid gpg --homedir "$working_dir"/etc/pacman.d/gnupg/ --no-permission-warning --quiet --batch --import --import-options import-local-sigs "$(pacman-conf GpgDir)"/pubring.gpg >/dev/null 2>&1 + pacman-key --gpgdir "$working_dir"/etc/pacman.d/gnupg/ --import-trustdb "$(pacman-conf GpgDir)" >/dev/null 2>&1 + + printf 'Server = %s\n' "${host_mirrors[@]}" >"$working_dir/etc/pacman.d/mirrorlist" + + [[ -n $pac_conf ]] && cp "$pac_conf" "$working_dir/etc/pacman.conf" + [[ -n $makepkg_conf ]] && cp "$makepkg_conf" "$working_dir/etc/makepkg.conf" + + local file + for file in "${files[@]}"; do + mkdir -p "$(dirname "$working_dir$file")" + cp -T "$file" "$working_dir$file" + done + + sed -r "s|^#?\\s*CacheDir.+|CacheDir = ${cache_dirs[*]}|g" -i "$working_dir/etc/pacman.conf" +} +# }}} + +umask 0022 + +# Sanity check +if [[ ! -f "$working_dir/.arch-chroot" ]]; then + die "'%s' does not appear to be an Arch chroot." "$working_dir" +elif [[ $(cat "$working_dir/.arch-chroot") != "$CHROOT_VERSION" ]]; then + die "chroot '%s' is not at version %s. Please rebuild." "$working_dir" "$CHROOT_VERSION" +fi + +copy_hostconf + +eval "$(grep -a '^CARCH=' "$working_dir/etc/makepkg.conf")" + +[[ -z $nosetarch ]] || unset CARCH +if [[ -f "@pkgdatadir@/setarch-aliases.d/${CARCH}" ]]; then + read -r set_arch < "@pkgdatadir@/setarch-aliases.d/${CARCH}" +else + set_arch="${CARCH}" +fi + +exec ${CARCH:+setarch "$set_arch"} systemd-nspawn -q \ + -D "$working_dir" \ + -E "PATH=/usr/local/sbin:/usr/local/bin:/usr/bin" \ + --register=no --keep-unit --as-pid2 \ + "${mount_args[@]}" \ + "$@" diff --git a/src/archbuild.in b/src/archbuild.in new file mode 100644 index 0000000..e6cf19a --- /dev/null +++ b/src/archbuild.in @@ -0,0 +1,98 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +m4_include(lib/common.sh) +m4_include(lib/archroot.sh) + +base_packages=(base-devel) +makechrootpkg_args=(-c -n -C) + +cmd="${0##*/}" +if [[ "${cmd%%-*}" == 'multilib' ]]; then + repo="${cmd%-build}" + arch='x86_64' + base_packages+=(multilib-devel) +else + tag="${cmd%-build}" + repo=${tag%-*} + arch=${tag##*-} +fi +if [[ -f "@pkgdatadir@/setarch-aliases.d/${arch}" ]]; then + read -r set_arch < "@pkgdatadir@/setarch-aliases.d/${arch}" +else + set_arch="${arch}" +fi +chroots='/var/lib/archbuild' +clean_first=false + +pacman_config="@pkgdatadir@/pacman-${repo}.conf" +if [[ -f @pkgdatadir@/pacman-${repo}-${arch}.conf ]]; then + pacman_config="@pkgdatadir@/pacman-${repo}-${arch}.conf" +fi +makepkg_config="@pkgdatadir@/makepkg-${arch}.conf" +if [[ -f @pkgdatadir@/makepkg-${repo}-${arch}.conf ]]; then + makepkg_config="@pkgdatadir@/makepkg-${repo}-${arch}.conf" +fi + +usage() { + echo "Usage: $cmd [options] -- [makechrootpkg args]" + echo ' -h This help' + echo ' -c Recreate the chroot before building' + echo ' -r Create chroots in this directory' + echo '' + echo "Default makechrootpkg args: ${makechrootpkg_args[*]}" + echo '' + exit 1 +} + +while getopts 'hcr:' arg; do + case "${arg}" in + c) clean_first=true ;; + r) chroots="$OPTARG" ;; + *) usage ;; + esac +done + +check_root SOURCE_DATE_EPOCH,SRCDEST,SRCPKGDEST,PKGDEST,LOGDEST,MAKEFLAGS,PACKAGER,GNUPGHOME + +# Pass all arguments after -- right to makepkg +makechrootpkg_args+=("${@:$OPTIND}") + +if ${clean_first} || [[ ! -d "${chroots}/${repo}-${arch}" ]]; then + msg "Creating chroot for [%s] (%s)..." "${repo}" "${arch}" + + for copy in "${chroots}/${repo}-${arch}"/*; do + [[ -d $copy ]] || continue + msg2 "Deleting chroot copy '%s'..." "$(basename "${copy}")" + + lock 9 "$copy.lock" "Locking chroot copy '%s'" "$copy" + + subvolume_delete_recursive "${copy}" + rm -rf --one-file-system "${copy}" + done + lock_close 9 + + rm -rf --one-file-system "${chroots}/${repo}-${arch}" + (umask 0022; mkdir -p "${chroots}/${repo}-${arch}") + setarch "${set_arch}" mkarchroot \ + -C "${pacman_config}" \ + -M "${makepkg_config}" \ + "${chroots}/${repo}-${arch}/root" \ + "${base_packages[@]}" || abort +else + lock 9 "${chroots}/${repo}-${arch}/root.lock" "Locking clean chroot" + arch-nspawn \ + -C "${pacman_config}" \ + -M "${makepkg_config}" \ + "${chroots}/${repo}-${arch}/root" \ + pacman -Syuu --noconfirm || abort +fi + +# Always build official packages reproducibly +if [[ ! -v SOURCE_DATE_EPOCH ]]; then + export SOURCE_DATE_EPOCH=$(date +%s) +fi + +msg "Building in chroot for [%s] (%s)..." "${repo}" "${arch}" +exec makechrootpkg -r "${chroots}/${repo}-${arch}" "${makechrootpkg_args[@]}" diff --git a/src/archco.in b/src/archco.in new file mode 100644 index 0000000..a93d819 --- /dev/null +++ b/src/archco.in @@ -0,0 +1,26 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +m4_include(lib/common.sh) + +scriptname=${0##*/} + +if [[ -z $1 ]]; then + printf 'Usage: %s ...\n' "$scriptname" + exit 1 +fi + +case $scriptname in + archco) + SVNURL="svn+ssh://svn-packages@repos.archlinux.org/srv/repos/svn-packages/svn";; + communityco) + SVNURL="svn+ssh://svn-community@repos.archlinux.org/srv/repos/svn-community/svn";; + *) + die "Couldn't find svn url for %s" "$scriptname" + ;; +esac + +for i in "$@"; do + svn co "$SVNURL/$i" +done diff --git a/src/archrelease.in b/src/archrelease.in new file mode 100644 index 0000000..3490ee2 --- /dev/null +++ b/src/archrelease.in @@ -0,0 +1,87 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +m4_include(lib/common.sh) +m4_include(lib/valid-tags.sh) + +# parse command line options +FORCE= +while getopts ':f' flag; do + case $flag in + f) FORCE=1 ;; + :) die "Option requires an argument -- '%s'" "$OPTARG" ;; + \?) die "Invalid option -- '%s'" "$OPTARG" ;; + esac +done +shift $(( OPTIND - 1 )) + +if ! (( $# )); then + echo 'Usage: archrelease [-f] ...' + exit 1 +fi + +# validate repo is really repo-arch +if [[ -z $FORCE ]]; then + for tag in "$@"; do + if ! in_array "$tag" "${_tags[@]}"; then + die "archrelease: Invalid tag: '%s' (use -f to force release)" "$tag" + fi + done +fi + +if [[ ! -f PKGBUILD ]]; then + die 'archrelease: PKGBUILD not found' +fi + +trunk=${PWD##*/} + +# Normally this should be trunk, but it may be something +# such as 'gnome-unstable' +IFS='/' read -r -d '' -a parts <<< "$PWD" +if [[ "${parts[*]:(-2):1}" == "repos" ]]; then + die 'archrelease: Should not be in repos dir (try from trunk/)' +fi +unset parts + +if [[ $(svn status -q) ]]; then + die 'archrelease: You have not committed your changes yet!' +fi + +pushd .. >/dev/null +mapfile -t known_files < <(svn ls -r HEAD "$trunk") +wait $! || die "failed to discover committed files" + +# gracefully handle files containing an "@" character +known_files=("${known_files[@]/%/@}") + +# update repo directory first to avoid a commit failure +svn up repos + +for tag in "$@"; do + stat_busy "Copying %s to %s" "${trunk}" "${tag}" + + if [[ -d repos/$tag ]]; then + mapfile -t trash < <(svn ls --recursive "repos/$tag") + wait $! || die "failed to discover existing files" + if (( ${#trash[@]} )); then + trash=("${trash[@]/#/repos/$tag/}") + svn rm -q "${trash[@]/%/@}" + fi + else + mkdir -p "repos/$tag" + svn add --parents -q "repos/$tag" + fi + + # copy all files at once from trunk to the subdirectory in repos/ + svn copy -q -r HEAD "${known_files[@]/#/$trunk/}" "repos/$tag/" + + stat_done +done + +stat_busy "Releasing package" +printf -v tag_list ", %s" "$@"; tag_list="${tag_list#, }" +svn commit -q -m "archrelease: copy ${trunk} to $tag_list" || abort +stat_done + +popd >/dev/null diff --git a/src/checkpkg.in b/src/checkpkg.in new file mode 100644 index 0000000..059f752 --- /dev/null +++ b/src/checkpkg.in @@ -0,0 +1,155 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +shopt -s extglob + +m4_include(lib/common.sh) + +usage() { + cat <<- _EOF_ + Usage: ${BASH_SOURCE[0]##*/} [OPTIONS] + + Searches for a locally built package corresponding to the PKGBUILD, and + downloads the last version of that package from the Pacman repositories. + It then compares the list of .so files provided by each version of the + package and outputs if there are soname differences for the new package. + A directory is also created using mktemp with files containing a file + list for both packages and a library list for both packages. + + OPTIONS + -r, --rmdir Remove the temporary directory + -w, --warn Print a warning in case of differences + -M, --makepkg-config Set an alternate makepkg configuration file + -h, --help Show this help text +_EOF_ +} + +RMDIR=0 +WARN=0 +MAKEPKG_CONF=/etc/makepkg.conf + +# option checking +while (( $# )); do + case $1 in + -h|--help) + usage + exit 0 + ;; + -r|--rmdir) + RMDIR=1 + shift + ;; + -w|--warn) + WARN=1 + shift + ;; + -M|--makepkg-config) + MAKEPKG_CONF="$2" + shift 2 + ;; + --) + shift + break + ;; + -*,--*) + die "invalid argument: %s" "$1" + ;; + *) + break + ;; + esac +done + +# Source makepkg.conf; fail if it is not found +if [[ -r "${MAKEPKG_CONF}" ]]; then + # shellcheck source=config/makepkg/x86_64.conf + source "${MAKEPKG_CONF}" +else + die "${MAKEPKG_CONF} not found!" +fi + +# Source user-specific makepkg.conf overrides +if [[ -r "${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" ]]; then + # shellcheck source=/dev/null + source "${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" +elif [[ -r "$HOME/.makepkg.conf" ]]; then + # shellcheck source=/dev/null + source "$HOME/.makepkg.conf" +fi + +if [[ ! -f PKGBUILD ]]; then + die 'This must be run in the directory of a built package.' +fi + +# shellcheck source=PKGBUILD.proto +. ./PKGBUILD +if [[ ${arch[0]} == 'any' ]]; then + CARCH='any' +fi + +STARTDIR=$(pwd) +(( RMDIR )) && trap 'rm -rf $TEMPDIR' EXIT INT TERM QUIT +TEMPDIR=$(mktemp -d --tmpdir checkpkg-script.XXXX) + +for _pkgname in "${pkgname[@]}"; do + comparepkg=$_pkgname + pkgurl= + target_pkgver=$(get_full_version "$_pkgname") + if ! pkgfile=$(find_cached_package "$_pkgname" "$target_pkgver" "$CARCH"); then + die 'tarball not found for package: %s' "${_pkgname}-$target_pkgver" + fi + + ln -s "$pkgfile" "$TEMPDIR" + + if (( $# )); then + case $1 in + *://*) + pkgurl=$1 ;; + /*|*/*) + pkgurl=$(readlink -m "$1") ;; + *.pkg.tar*) + pkgurl=$1 ;; + '') + ;; + *) + comparepkg=$1 ;; + esac + shift + fi + [[ -n $pkgurl ]] || pkgurl=$(pacman -Spdd --print-format '%l' --noconfirm "$comparepkg") || + die "Couldn't download previous package for %s." "$comparepkg" + + oldpkg=${pkgurl##*/} + + if [[ ${oldpkg} = "${pkgfile##*/}" ]]; then + die "The built package (%s) is the one in the repo right now!" "$_pkgname" + fi + + if [[ $pkgurl = file://* || ( $pkgurl = /* && -f $pkgurl ) ]]; then + ln -s "${pkgurl#file://}" "$TEMPDIR/$oldpkg" + elif [[ -f "$PKGDEST/$oldpkg" ]]; then + ln -s "$PKGDEST/$oldpkg" "$TEMPDIR/$oldpkg" + elif [[ -f "$STARTDIR/$oldpkg" ]]; then + ln -s "$STARTDIR/$oldpkg" "$TEMPDIR/$oldpkg" + else + curl -fsLC - --retry 3 --retry-delay 3 -o "$TEMPDIR/$oldpkg" "$pkgurl" + fi + + bsdtar tf "$TEMPDIR/$oldpkg" | sort > "$TEMPDIR/filelist-$_pkgname-old" + bsdtar tf "$pkgfile" | sort > "$TEMPDIR/filelist-$_pkgname" + + sdiff -s "$TEMPDIR/filelist-$_pkgname-old" "$TEMPDIR/filelist-$_pkgname" + + find-libprovides "$TEMPDIR/$oldpkg" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname-old" + find-libprovides "$pkgfile" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname" + if ! diff_output="$(sdiff -s "$TEMPDIR/libraries-$_pkgname-old" "$TEMPDIR/libraries-$_pkgname")"; then + message="Sonames differ in $_pkgname!" + (( WARN )) && warning "$message" || msg "$message" + echo "$diff_output" + else + msg "No soname differences for %s." "$_pkgname" + fi +done + +(( RMDIR )) || msg "Files saved to %s" "$TEMPDIR" diff --git a/src/commitpkg.in b/src/commitpkg.in new file mode 100644 index 0000000..31adcd6 --- /dev/null +++ b/src/commitpkg.in @@ -0,0 +1,241 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +m4_include(lib/common.sh) + +# Source makepkg.conf; fail if it is not found +if [[ -r '/etc/makepkg.conf' ]]; then + # shellcheck source=config/makepkg/x86_64.conf + source '/etc/makepkg.conf' +else + die '/etc/makepkg.conf not found!' +fi + +# Source user-specific makepkg.conf overrides +if [[ -r "${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" ]]; then + # shellcheck source=/dev/null + source "${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" +elif [[ -r "$HOME/.makepkg.conf" ]]; then + # shellcheck source=/dev/null + source "$HOME/.makepkg.conf" +fi + +cmd=${0##*/} + +if [[ ! -f PKGBUILD ]]; then + die 'No PKGBUILD file' +fi + +source=() +# shellcheck source=PKGBUILD.proto +. ./PKGBUILD +pkgbase=${pkgbase:-$pkgname} + +case "$cmd" in + commitpkg) + if (( $# == 0 )); then + die 'Usage: commitpkg [-f] [-s server] [-l limit] [-a arch] [commit message]' + fi + repo="$1" + shift + ;; + *pkg) + repo="${cmd%pkg}" + ;; + *) + die 'Usage: commitpkg [-f] [-s server] [-l limit] [-a arch] [commit message]' + ;; +esac + + +if (( ${#validpgpkeys[@]} != 0 )); then + if [[ -d keys ]]; then + for key in "${validpgpkeys[@]}"; do + if [[ ! -f keys/pgp/$key.asc ]]; then + export-pkgbuild-keys || die 'Failed to export valid PGP keys for source files' + fi + done + else + export-pkgbuild-keys || die 'Failed to export valid PGP keys for source files' + fi + + svn add --parents --force keys/pgp/* +fi + +# find files which should be under source control +needsversioning=() +for s in "${source[@]}"; do + [[ $s != *://* ]] && needsversioning+=("$s") +done +for i in 'changelog' 'install'; do + while read -r file; do + # evaluate any bash variables used + eval "file=\"$(sed "s/^\(['\"]\)\(.*\)\1\$/\2/" <<< "$file")\"" + needsversioning+=("$file") + done < <(sed -n "s/^[[:space:]]*$i=//p" PKGBUILD) +done +for key in "${validpgpkeys[@]}"; do + needsversioning+=("keys/pgp/$key.asc") +done + +# assert that they really are controlled by SVN +if (( ${#needsversioning[*]} )); then + # svn status's output is only two columns when the status is unknown + while read -r status filename; do + [[ $status = '?' ]] && unversioned+=("$filename") + done < <(svn status -v "${needsversioning[@]}") + (( ${#unversioned[*]} )) && die "%s is not under version control" "${unversioned[@]}" +fi + +rsyncopts=(-e ssh -p '--chmod=ug=rw,o=r' -c -h -L --progress --partial -y) +archreleaseopts=() +while getopts ':l:a:s:f' flag; do + case $flag in + f) archreleaseopts+=('-f') ;; + s) server=$OPTARG ;; + l) rsyncopts+=("--bwlimit=$OPTARG") ;; + a) commit_arch=$OPTARG ;; + :) die "Option requires an argument -- '%s'" "$OPTARG" ;; + \?) die "Invalid option -- '%s'" "$OPTARG" ;; + esac +done +shift $(( OPTIND - 1 )) + +# check packages for validity +for _arch in "${arch[@]}"; do + if [[ -n $commit_arch && ${_arch} != "$commit_arch" ]]; then + continue + fi + for _pkgname in "${pkgname[@]}"; do + fullver=$(get_full_version "$_pkgname") + + if pkgfile=$(find_cached_package "$_pkgname" "$fullver" "$_arch"); then + check_package_validity "$pkgfile" + fi + done + + fullver=$(get_full_version "$pkgbase") + if pkgfile=$(find_cached_package "$pkgbase-debug" "$fullver" "$_arch"); then + check_package_validity "$pkgfile" + fi +done + +if [[ -z $server ]]; then + server='repos.archlinux.org' +fi + +if [[ -n $(svn status -q) ]]; then + msgtemplate="upgpkg: $pkgbase $(get_full_version)" + if [[ -n $1 ]]; then + stat_busy 'Committing changes to trunk' + svn commit -q -m "${msgtemplate}: ${1}" || die + stat_done + else + msgfile="$(mktemp)" + echo "$msgtemplate" > "$msgfile" + if [[ -n $SVN_EDITOR ]]; then + $SVN_EDITOR "$msgfile" + elif [[ -n $VISUAL ]]; then + $VISUAL "$msgfile" + elif [[ -n $EDITOR ]]; then + $EDITOR "$msgfile" + else + vi "$msgfile" + fi + [[ -s $msgfile ]] || die + stat_busy 'Committing changes to trunk' + svn commit -q -F "$msgfile" || die + unlink "$msgfile" + stat_done + fi +fi + +declare -a uploads +declare -a commit_arches +declare -a skip_arches + +for _arch in "${arch[@]}"; do + if [[ -n $commit_arch && ${_arch} != "$commit_arch" ]]; then + skip_arches+=("$_arch") + continue + fi + + for _pkgname in "${pkgname[@]}"; do + fullver=$(get_full_version "$_pkgname") + if ! pkgfile=$(find_cached_package "$_pkgname" "$fullver" "${_arch}"); then + warning "Skipping %s: failed to locate package file" "$_pkgname-$fullver-$_arch" + skip_arches+=("$_arch") + continue 2 + fi + uploads+=("$pkgfile") + done + + fullver=$(get_full_version "$pkgbase") + if ! pkgfile=$(find_cached_package "$pkgbase-debug" "$fullver" "$_arch"); then + continue + fi + if ! is_debug_package "$pkgfile"; then + continue + fi + uploads+=("$pkgfile") +done + +for pkgfile in "${uploads[@]}"; do + sigfile="${pkgfile}.sig" + if [[ ! -f $sigfile ]]; then + msg "Signing package %s..." "${pkgfile}" + if [[ -n $GPGKEY ]]; then + SIGNWITHKEY=(-u "${GPGKEY}") + fi + gpg --detach-sign --use-agent --no-armor "${SIGNWITHKEY[@]}" "${pkgfile}" || die + fi + if ! gpg --verify "$sigfile" "$pkgfile" >/dev/null 2>&1; then + die "Signature %s is incorrect!" "$sigfile" + fi + uploads+=("$sigfile") +done + +for _arch in "${arch[@]}"; do + if ! in_array "$_arch" "${skip_arches[@]}"; then + commit_arches+=("$_arch") + fi +done + +if [[ ${#commit_arches[*]} -gt 0 ]]; then + archrelease "${archreleaseopts[@]}" "${commit_arches[@]/#/$repo-}" || die +fi + +if [[ ${#uploads[*]} -gt 0 ]]; then + new_uploads=() + + # convert to absolute paths so rsync can work with colons (epoch) + while read -r -d '' upload; do + new_uploads+=("$upload") + done < <(realpath -z "${uploads[@]}") + + uploads=("${new_uploads[@]}") + unset new_uploads + msg 'Uploading all package and signature files' + rsync "${rsyncopts[@]}" "${uploads[@]}" "$server:staging/$repo/" || die +fi + +if [[ "${arch[*]}" == 'any' ]]; then + if [[ -d ../repos/$repo-x86_64 ]]; then + pushd ../repos/ >/dev/null + stat_busy "Removing %s" "$repo-x86_64" + svn rm -q "$repo-x86_64" + svn commit -q -m "Removed $repo-x86_64 for $pkgname" + stat_done + popd >/dev/null + fi +else + if [[ -d ../repos/$repo-any ]]; then + pushd ../repos/ >/dev/null + stat_busy "Removing %s" "$repo-any" + svn rm -q "$repo-any" + svn commit -q -m "Removed $repo-any for $pkgname" + stat_done + popd >/dev/null + fi +fi diff --git a/src/crossrepomove.in b/src/crossrepomove.in new file mode 100644 index 0000000..c028d62 --- /dev/null +++ b/src/crossrepomove.in @@ -0,0 +1,86 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +m4_include(lib/common.sh) + +scriptname=${0##*/} + +if [[ -z $1 ]]; then + printf 'Usage: %s [pkgbase]\n' "$scriptname" + exit 1 +fi + +pkgbase="${1}" + +case $scriptname in + extra2community) + source_name='packages' + target_name='community' + source_repo='extra' + target_repo='community' + ;; + community2extra) + source_name='community' + target_name='packages' + source_repo='community' + target_repo='extra' + ;; + *) + die "Couldn't find configuration for %s" "$scriptname" + ;; +esac + +server='repos.archlinux.org' +source_svn="svn+ssh://svn-${source_name}@${server}/srv/repos/svn-${source_name}/svn" +target_svn="svn+ssh://svn-${target_name}@${server}/srv/repos/svn-${target_name}/svn" +source_dbscripts="/srv/repos/svn-${source_name}/dbscripts" +target_dbscripts="/srv/repos/svn-${target_name}/dbscripts" + +setup_workdir + +pushd "$WORKDIR" >/dev/null + +msg "Downloading sources for %s" "${pkgbase}" +svn -q checkout -N "${target_svn}" target_checkout +mkdir -p "target_checkout/${pkgbase}/repos" +svn -q export "${source_svn}/${pkgbase}/trunk" "target_checkout/${pkgbase}/trunk" || die +# shellcheck source=PKGBUILD.proto +. "target_checkout/${pkgbase}/trunk/PKGBUILD" + +msg "Downloading packages for %s" "${pkgbase}" +for _arch in "${arch[@]}"; do + if [[ "${_arch[*]}" == 'any' ]]; then + repo_arch='x86_64' + else + repo_arch=${_arch} + fi + for _pkgname in "${pkgname[@]}"; do + fullver=$(get_full_version "$_pkgname") + pkgpath="/srv/ftp/$source_repo/os/$repo_arch/$_pkgname-$fullver-${_arch}.pkg.tar.*" + # shellcheck disable=2029 + ssh "$server" "cp $pkgpath staging/$target_repo" || die + done +done + +msg "Adding %s to %s" "${pkgbase}" "${target_repo}" +svn -q add "target_checkout/${pkgbase}" +svn -q commit -m"${scriptname}: Moving ${pkgbase} from ${source_repo} to ${target_repo}" target_checkout +pushd "target_checkout/${pkgbase}/trunk" >/dev/null +archrelease "${arch[@]/#/$target_repo-}" || die +popd >/dev/null + +# shellcheck disable=2029 +ssh "${server}" "${target_dbscripts}/db-update" || die + +msg "Removing %s from %s" "${pkgbase}" "${source_repo}" +for _arch in "${arch[@]}"; do + # shellcheck disable=2029 + ssh "${server}" "${source_dbscripts}/db-remove ${source_repo} ${_arch} ${pkgbase}" +done +svn -q checkout -N "${source_svn}" source_checkout +svn -q up "source_checkout/${pkgbase}" +svn -q rm "source_checkout/${pkgbase}" +svn -q commit -m"${scriptname}: Moving ${pkgbase} from ${source_repo} to ${target_repo}" source_checkout + +popd >/dev/null diff --git a/src/diffpkg.in b/src/diffpkg.in new file mode 100644 index 0000000..963f2c6 --- /dev/null +++ b/src/diffpkg.in @@ -0,0 +1,228 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +shopt -s extglob + +m4_include(lib/common.sh) + +usage() { + cat <<- _EOF_ + Usage: ${BASH_SOURCE[0]##*/} [OPTIONS] [MODES] [FILE|PKGNAME...] + + Searches for a locally built package corresponding to the PKGBUILD, and + downloads the last version of that package from the Pacman repositories. + It then compares the package archives using different modes while using + simple tar content list by default. + + When given one package, use it to diff against the locally built one. + When given two packages, diff both packages against each other. + + In either case, a package name will be converted to a filename from the + cache, and diffpkg will proceed as though this filename was initially + specified. + + OPTIONS + -M, --makepkg-config Set an alternate makepkg configuration file + -v, --verbose Provide more detailed/unfiltered output + -h, --help Show this help text + + MODES + -l, --list Activate content list diff mode (default) + -d, --diffoscope Activate diffoscope diff mode + -p, --pkginfo Activate .PKGINFO diff mode + -b, --buildinfo Activate .BUILDINFO diff mode +_EOF_ +} + +MAKEPKG_CONF=/etc/makepkg.conf +VERBOSE=0 +TARLIST=0 +DIFFOSCOPE=0 +PKGINFO=0 +BUILDINFO=0 + +# option checking +while (( $# )); do + case $1 in + -h|--help) + usage + exit 0 + ;; + -M|--makepkg-config) + MAKEPKG_CONF="$2" + shift 2 + ;; + -l|--list) + TARLIST=1 + shift + ;; + -d|--diffoscope) + DIFFOSCOPE=1 + shift + ;; + -p|--pkginfo) + PKGINFO=1 + shift + ;; + -b|--buildinfo) + BUILDINFO=1 + shift + ;; + -v|--verbose) + VERBOSE=1 + shift + ;; + --) + shift + break + ;; + -*,--*) + die "invalid argument: %s" "$1" + ;; + *) + break + ;; + esac +done + +if ! (( DIFFOSCOPE || TARLIST || PKGINFO || BUILDINFO )); then + TARLIST=1 +fi + +# Source makepkg.conf; fail if it is not found +if [[ -r "${MAKEPKG_CONF}" ]]; then + # shellcheck source=config/makepkg/x86_64.conf + source "${MAKEPKG_CONF}" +else + die "${MAKEPKG_CONF} not found!" +fi + +# Source user-specific makepkg.conf overrides +if [[ -r "${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" ]]; then + # shellcheck source=/dev/null + source "${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" +elif [[ -r "$HOME/.makepkg.conf" ]]; then + # shellcheck source=/dev/null + source "$HOME/.makepkg.conf" +fi + +STARTDIR=$(pwd) +trap 'rm -rf $TMPDIR' EXIT INT TERM QUIT +TMPDIR=$(mktemp -d --tmpdir diffpkg-script.XXXXXXXX) +export TMPDIR + +tar_list() { + bsdtar tf "$*" | if (( VERBOSE )); then + cat + else + sed -E 's|^usr/lib/modules/[0-9][^/]+|usr/lib/modules/[…]|g' + fi | sort +} + +diff_pkgs() { + local oldpkg newpkg + oldpkg=$(readlink -m "$1") + newpkg=$(readlink -m "$2") + + [[ -f $oldpkg ]] || die "No such file: %s" "${oldpkg}" + [[ -f $newpkg ]] || die "No such file: %s" "${newpkg}" + + if (( TARLIST )); then + tar_list "$oldpkg" > "$TMPDIR/filelist-old" + tar_list "$newpkg" > "$TMPDIR/filelist" + + sdiff -s "$TMPDIR/filelist-old" "$TMPDIR/filelist" + fi + + if (( PKGINFO )); then + bsdtar xOqf "$oldpkg" .PKGINFO > "$TMPDIR/pkginfo-old" + bsdtar xOqf "$newpkg" .PKGINFO > "$TMPDIR/pkginfo" + + sdiff -s "$TMPDIR/pkginfo-old" "$TMPDIR/pkginfo" + fi + + if (( BUILDINFO )); then + bsdtar xOqf "$oldpkg" .BUILDINFO > "$TMPDIR/buildinfo-old" + bsdtar xOqf "$newpkg" .BUILDINFO > "$TMPDIR/buildinfo" + + sdiff -s "$TMPDIR/buildinfo-old" "$TMPDIR/buildinfo" + fi + + if (( DIFFOSCOPE )); then + diffoscope "$oldpkg" "$newpkg" + fi +} + +fetch_pkg() { + local pkg pkgdest pkgurl + case $1 in + *://*) + pkgurl=$1 ;; + /*|*/*) + pkgurl=$(readlink -m "$1") ;; + *.pkg.tar*) + pkgurl=$1 ;; + '') + ;; + *) + pkg=$1 ;; + esac + + [[ -n $pkgurl ]] || pkgurl=$(pacman -Spdd --print-format '%l' --noconfirm "$pkg") || + die "Couldn't download previous package for %s." "$pkg" + + pkg=${pkgurl##*/} + pkgdest=$(mktemp -t -d "${pkg}-XXXXXX")/${pkg} + + if [[ $pkgurl = file://* || ( $pkgurl = /* && -f $pkgurl ) ]]; then + ln -sf "${pkgurl#file://}" "$pkgdest" + elif [[ -f "$PKGDEST/$pkg" ]]; then + ln -sf "$PKGDEST/$pkg" "$pkgdest" + elif [[ -f "$STARTDIR/$pkg" ]]; then + ln -sf "$STARTDIR/$pkg" "$pkgdest" + elif [[ $pkgurl = *://* ]]; then + curl -fsLC - --retry 3 --retry-delay 3 -o "$pkgdest" "$pkgurl" || \ + die "Couldn't download %s" "$pkgurl" + else + die "File not found: %s" "$pkgurl" + fi + + echo "$pkgdest" +} + +if (( $# < 2 )); then + if [[ ! -f PKGBUILD ]]; then + die "This must be run in the directory of a built package.\nTry '$(basename "$0") --help' for more information." + fi + + # shellcheck source=PKGBUILD.proto + . ./PKGBUILD + if [[ ${arch[0]} == 'any' ]]; then + CARCH='any' + fi + + for _pkgname in "${pkgname[@]}"; do + comparepkg=$_pkgname + pkgurl= + target_pkgver=$(get_full_version "$_pkgname") + if ! pkgfile=$(find_cached_package "$_pkgname" "$target_pkgver" "$CARCH"); then + die 'tarball not found for package: %s' "${_pkgname}-$target_pkgver" + fi + + ln -s "$pkgfile" "$TMPDIR" + + if (( $# )); then + comparepkg="$1" + fi + + oldpkg=$(fetch_pkg "$comparepkg") || exit 1 + + diff_pkgs "$oldpkg" "$pkgfile" + done +else + file1=$(fetch_pkg "$1") || exit 1 + file2=$(fetch_pkg "$2") || exit 1 + + diff_pkgs "$file1" "$file2" +fi diff --git a/src/export-pkgbuild-keys.in b/src/export-pkgbuild-keys.in new file mode 100644 index 0000000..8697b3d --- /dev/null +++ b/src/export-pkgbuild-keys.in @@ -0,0 +1,75 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +m4_include(lib/common.sh) + +usage() { + cat <<- _EOF_ + Usage: ${BASH_SOURCE[0]##*/} + + Export the PGP keys from a PKGBUILDs validpgpkeys array into the keys/pgp/ + subdirectory. Useful for distributing packager validated source signing + keys alongside PKGBUILDs. + + OPTIONS + -h, --help Show this help text +_EOF_ +} + +# option checking +while (( $# )); do + case $1 in + -h|--help) + usage + exit 0 + ;; + *) + die "invalid argument: %s" "$1" + ;; + esac +done + +if [[ ! -f PKGBUILD ]]; then + die "This must be run a directory containing a PKGBUILD." +fi + +mapfile -t validpgpkeys < <( + # shellcheck source=PKGBUILD.proto + . ./PKGBUILD + if (( ${#validpgpkeys[@]} )); then + printf "%s\n" "${validpgpkeys[@]}" + fi +) + +msg "Exporting ${#validpgpkeys[@]} PGP keys..." +if (( ${#validpgpkeys[@]} == 0 )); then + exit 0 +fi + +trap 'rm -rf $TEMPDIR' EXIT INT TERM QUIT +TEMPDIR=$(mktemp -d --tmpdir export-pkgbuild-keys.XXXXXXXXXX) + +mkdir -p keys/pgp +error=0 + +for key in "${validpgpkeys[@]}"; do + gpg --output "$TEMPDIR/$key.asc" --armor --export --export-options export-minimal "$key" 2>/dev/null + + # gpg does not give a non-zero return value if it fails to export... + if [[ -f $TEMPDIR/$key.asc ]]; then + msg2 "Exported $key" + mv "$TEMPDIR/$key.asc" "keys/pgp/$key.asc" + else + if [[ -f keys/pgp/$key.asc ]]; then + warning "Failed to update key: $key" + else + error "Key unavailable: $key" + error=1 + fi + fi +done + +if (( error )); then + die "Failed to export all \'validpgpkeys\' entries." +fi diff --git a/src/find-libdeps.in b/src/find-libdeps.in new file mode 100644 index 0000000..e1423b8 --- /dev/null +++ b/src/find-libdeps.in @@ -0,0 +1,89 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +m4_include(lib/common.sh) + +set -e +shopt -s extglob + +IGNORE_INTERNAL=0 + +if [[ $1 = "--ignore-internal" ]]; then + IGNORE_INTERNAL=1 + shift +fi + +script_mode=${BASH_SOURCE[0]##*/find-lib} + +case $script_mode in + deps|provides) true;; + *) die "Unknown mode %s" "$script_mode" ;; +esac + +if [[ -z $1 ]]; then + echo "${0##*/} [options] " + echo "Options:" + echo " --ignore-internal ignore internal libraries" + exit 1 +fi + +if [[ -d $1 ]]; then + pushd "$1" >/dev/null +else + setup_workdir + + case ${script_mode} in + deps) bsdtar -C "$WORKDIR" -xf "$1";; + provides) bsdtar -C "$WORKDIR" -xf "$1" --include="*.so*";; + esac + + pushd "$WORKDIR" >/dev/null +fi + +process_sofile() { + # extract the library name: libfoo.so + soname="${sofile%.so?(+(.+([0-9])))}".so + # extract the major version: 1 + soversion="${sofile##*\.so\.}" + if [[ "$soversion" = "$sofile" ]] && ((IGNORE_INTERNAL)); then + return + fi + if ! in_array "${soname}=${soversion}-${soarch}" "${soobjects[@]}"; then + # libfoo.so=1-64 + echo "${soname}=${soversion}-${soarch}" + soobjects+=("${soname}=${soversion}-${soarch}") + fi +} + +case $script_mode in + deps) find_args=(-perm -u+x);; + provides) find_args=(-name '*.so*');; +esac + +find . -type f "${find_args[@]}" | while read -r filename; do + if [[ $script_mode = "provides" ]]; then + # ignore if we don't have a shared object + if ! LC_ALL=C readelf -h "$filename" 2>/dev/null | grep -q '.*Type:.*DYN (Shared object file).*'; then + continue + fi + fi + + # get architecture of the file; if soarch is empty it's not an ELF binary + soarch=$(LC_ALL=C readelf -h "$filename" 2>/dev/null | sed -n 's/.*Class.*ELF\(32\|64\)/\1/p') + [[ -n $soarch ]] || continue + + if [[ $script_mode = "provides" ]]; then + # get the string binaries link to: libfoo.so.1.2 -> libfoo.so.1 + sofile=$(LC_ALL=C readelf -d "$filename" 2>/dev/null | sed -n 's/.*Library soname: \[\(.*\)\].*/\1/p') + [[ -z $sofile ]] && sofile="${filename##*/}" + process_sofile + elif [[ $script_mode = "deps" ]]; then + # process all libraries needed by the binary + for sofile in $(LC_ALL=C readelf -d "$filename" 2>/dev/null | sed -nr 's/.*Shared library: \[(.*)\].*/\1/p'); do + process_sofile + done + fi +done + +popd >/dev/null diff --git a/src/finddeps.in b/src/finddeps.in new file mode 100644 index 0000000..05b3530 --- /dev/null +++ b/src/finddeps.in @@ -0,0 +1,41 @@ +#!/bin/bash +# +# finddeps - find packages that depend on a given depname +# +# SPDX-License-Identifier: GPL-3.0-or-later + +m4_include(lib/common.sh) + +match=$1 + +if [[ -z $match ]]; then + echo 'Usage: finddeps ' + echo '' + echo 'Find packages that depend on a given depname.' + echo 'Run this script from the top-level directory of your ABS tree.' + echo '' + exit 1 +fi + +find . -type d -print0 2>/dev/null| while read -r -d '' d; do + if [[ -f "$d/PKGBUILD" ]]; then + pkgname=() depends=() makedepends=() optdepends=() + # shellcheck source=PKGBUILD.proto + . "$d/PKGBUILD" + for dep in "${depends[@]}"; do + # lose the version comparator, if any + depname=${dep%%[<>=]*} + [[ $depname = "$match" ]] && echo "$d (depends)" + done + for dep in "${makedepends[@]}"; do + # lose the version comparator, if any + depname=${dep%%[<>=]*} + [[ $depname = "$match" ]] && echo "$d (makedepends)" + done + for dep in "${optdepends[@]/:*}"; do + # lose the version comaparator, if any + depname=${dep%%[<>=]*} + [[ $depname = "$match" ]] && echo "$d (optdepends)" + done + fi +done diff --git a/src/lddd.in b/src/lddd.in new file mode 100644 index 0000000..12f8d67 --- /dev/null +++ b/src/lddd.in @@ -0,0 +1,49 @@ +#!/bin/bash +# +# lddd - find broken library links on your machine +# +# SPDX-License-Identifier: GPL-3.0-or-later + +m4_include(lib/common.sh) + +ifs=$IFS +IFS="${IFS}:" + +libdirs="/lib /usr/lib /usr/local/lib $(cat /etc/ld.so.conf.d/*)" +extras= + +TEMPDIR=$(mktemp -d --tmpdir lddd-script.XXXX) + +msg 'Go out and drink some tea, this will take a while :) ...' +# Check ELF binaries in the PATH and specified dir trees. +for tree in $PATH $libdirs $extras; do + msg2 "DIR %s" "$tree" + + # Get list of files in tree. + files=$(find "$tree" -type f ! -name '*.a' ! -name '*.la' ! -name '*.py*' ! -name '*.txt' ! -name '*.h' ! -name '*.ttf' ! \ + -name '*.rb' ! -name '*.ko' ! -name '*.pc' ! -name '*.enc' ! -name '*.cf' ! -name '*.def' ! -name '*.rules' ! -name \ + '*.cmi' ! -name '*.mli' ! -name '*.ml' ! -name '*.cma' ! -name '*.cmx' ! -name '*.cmxa' ! -name '*.pod' ! -name '*.pm' \ + ! -name '*.pl' ! -name '*.al' ! -name '*.tcl' ! -name '*.bs' ! -name '*.o' ! -name '*.png' ! -name '*.gif' ! -name '*.cmo' \ + ! -name '*.cgi' ! -name '*.defs' ! -name '*.conf' ! -name '*_LOCALE' ! -name 'Compose' ! -name '*_OBJS' ! -name '*.msg' ! \ + -name '*.mcopclass' ! -name '*.mcoptype') + IFS=$ifs + for i in $files; do + if (( $(file "$i" | grep -c 'ELF') != 0 )); then + # Is an ELF binary. + if (( $(ldd "$i" 2>/dev/null | grep -c 'not found') != 0 )); then + # Missing lib. + echo "$i:" >> "$TEMPDIR/raw.txt" + ldd "$i" 2>/dev/null | grep 'not found' >> "$TEMPDIR/raw.txt" + fi + fi + done +done +grep '^/' "$TEMPDIR/raw.txt" | sed -e 's/://g' >> "$TEMPDIR/affected-files.txt" +# invoke pacman +while read -r i; do + pacman -Qo "$i" | awk '{print $4,$5}' >> "$TEMPDIR/pacman.txt" +done < "$TEMPDIR/affected-files.txt" +# clean list +sort -u "$TEMPDIR/pacman.txt" >> "$TEMPDIR/possible-rebuilds.txt" + +msg "Files saved to %s" "$TEMPDIR" diff --git a/src/makechrootpkg.in b/src/makechrootpkg.in new file mode 100644 index 0000000..126d1da --- /dev/null +++ b/src/makechrootpkg.in @@ -0,0 +1,414 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +m4_include(lib/common.sh) +m4_include(lib/archroot.sh) + +source /usr/share/makepkg/util/config.sh + +shopt -s nullglob + +default_makepkg_args=(--syncdeps --noconfirm --log --holdver --skipinteg) +makepkg_args=("${default_makepkg_args[@]}") +verifysource_args=() +chrootdir= +passeddir= +makepkg_user= +declare -a install_pkgs +declare -i ret=0 + +keepbuilddir=0 +update_first=0 +clean_first=0 +run_namcap=0 +run_checkpkg=0 +temp_chroot=0 + +bindmounts_ro=() +bindmounts_rw=() + +copy=$USER +[[ -n ${SUDO_USER:-} ]] && copy=$SUDO_USER +[[ -z "$copy" || $copy = root ]] && copy=copy +src_owner=${SUDO_USER:-$USER} + +usage() { + echo "Usage: ${0##*/} [options] -r [--] [makepkg args]" + echo ' Run this script in a PKGBUILD dir to build a package inside a' + echo ' clean chroot. Arguments passed to this script after the' + echo ' end-of-options marker (--) will be passed to makepkg.' + echo '' + echo ' The chroot dir consists of the following directories:' + echo ' /{root, copy} but only "root" is required' + echo ' by default. The working copy will be created as needed' + echo '' + echo 'The chroot "root" directory must be created via the following' + echo 'command:' + echo ' mkarchroot /root base-devel' + echo '' + echo 'This script reads {SRC,SRCPKG,PKG,LOG}DEST, MAKEFLAGS and PACKAGER' + echo 'from makepkg.conf(5), if those variables are not part of the' + echo 'environment.' + echo '' + echo "Default makepkg args: ${default_makepkg_args[*]}" + echo '' + echo 'Flags:' + echo '-h This help' + echo '-c Clean the chroot before building' + echo '-d Bind directory into build chroot as read-write' + echo '-D Bind directory into build chroot as read-only' + echo '-u Update the working copy of the chroot before building' + echo ' This is useful for rebuilds without dirtying the pristine' + echo ' chroot' + echo '-r The chroot dir to use' + echo '-I Install a package into the working copy of the chroot' + echo '-l The directory to use as the working copy of the chroot' + echo ' Useful for maintaining multiple copies' + echo " Default: $copy" + echo '-n Run namcap on the package' + echo '-C Run checkpkg on the package' + echo '-T Build in a temporary directory' + echo '-U Run makepkg as a specified user' + exit 1 +} + +# {{{ functions +# Usage: sync_chroot $chrootdir $copydir [$copy] +sync_chroot() { + local chrootdir=$1 + local copydir=$2 + local copy=${3:-$2} + + if [[ "$chrootdir/root" -ef "$copydir" ]]; then + error 'Cannot sync copy with itself: %s' "$copydir" + return 1 + fi + + # Get a read lock on the root chroot to make + # sure we don't clone a half-updated chroot + slock 8 "$chrootdir/root.lock" \ + "Locking clean chroot [%s]" "$chrootdir/root" + + stat_busy "Synchronizing chroot copy [%s] -> [%s]" "$chrootdir/root" "$copy" + if is_btrfs "$chrootdir" && ! mountpoint -q "$copydir"; then + subvolume_delete_recursive "$copydir" || + die "Unable to delete subvolume %s" "$copydir" + btrfs subvolume snapshot "$chrootdir/root" "$copydir" >/dev/null || + die "Unable to create subvolume %s" "$copydir" + else + mkdir -p "$copydir" + rsync -a --delete -q -W -x "$chrootdir/root/" "$copydir" + fi + stat_done + + # Drop the read lock again + lock_close 8 + + # Update mtime + touch "$copydir" +} + +# Usage: delete_chroot $copydir [$copy] +delete_chroot() { + local copydir=$1 + local copy=${1:-$2} + + stat_busy "Removing chroot copy [%s]" "$copy" + if is_subvolume "$copydir" && ! mountpoint -q "$copydir"; then + subvolume_delete_recursive "$copydir" || + die "Unable to delete subvolume %s" "$copydir" + else + # avoid change of filesystem in case of an umount failure + rm --recursive --force --one-file-system "$copydir" || + die "Unable to delete %s" "$copydir" + fi + + # remove lock file + rm -f "$copydir.lock" + stat_done +} + +install_packages() { + local -a pkgnames + local ret + + pkgnames=("${install_pkgs[@]##*/}") + + cp -- "${install_pkgs[@]}" "$copydir/root/" + arch-nspawn "$copydir" "${bindmounts_ro[@]}" "${bindmounts_rw[@]}" \ + bash -c 'yes y | pacman -U -- "$@"' -bash "${pkgnames[@]/#//root/}" + ret=$? + rm -- "${pkgnames[@]/#/$copydir/root/}" + + return $ret +} + +prepare_chroot() { + (( keepbuilddir )) || rm -rf "$copydir/build" + + local builduser_uid builduser_gid + builduser_uid="$(id -u "$makepkg_user")" + builduser_gid="$(id -g "$makepkg_user")" + local install="install -o $builduser_uid -g $builduser_gid" + local x + + # We can't use useradd without chrooting, otherwise it invokes PAM modules + # which we might not be able to load (i.e. when building i686 packages on + # an x86_64 host). + sed -e '/^builduser:/d' -i "$copydir"/etc/{passwd,shadow,group} + printf >>"$copydir/etc/group" 'builduser:x:%d:\n' "$builduser_gid" + printf >>"$copydir/etc/passwd" 'builduser:x:%d:%d:builduser:/build:/bin/bash\n' "$builduser_uid" "$builduser_gid" + printf >>"$copydir/etc/shadow" 'builduser:!!:%d::::::\n' "$(( $(date -u +%s) / 86400 ))" + + $install -d "$copydir"/{build,startdir,{pkg,srcpkg,src,log}dest} + + sed -e '/^MAKEFLAGS=/d' -e '/^PACKAGER=/d' -i "$copydir/etc/makepkg.conf" + for x in BUILDDIR=/build PKGDEST=/pkgdest SRCPKGDEST=/srcpkgdest SRCDEST=/srcdest LOGDEST=/logdest \ + "MAKEFLAGS='${MAKEFLAGS:-}'" "PACKAGER='${PACKAGER:-}'" + do + grep -q "^$x" "$copydir/etc/makepkg.conf" && continue + echo "$x" >>"$copydir/etc/makepkg.conf" + done + + cat > "$copydir/etc/sudoers.d/builduser-pacman" </dev/null || true + declare -p BUILDTOOL 2>/dev/null + declare -p BUILDTOOLVER 2>/dev/null + printf '_chrootbuild "$@" || exit\n' + + if (( run_namcap )); then + declare -f _chrootnamcap + printf '_chrootnamcap || exit\n' + fi + } >"$copydir/chrootbuild" + chmod +x "$copydir/chrootbuild" +} + +# These functions aren't run in makechrootpkg, +# so no global variables +_chrootbuild() { + # No coredumps + ulimit -c 0 + + # shellcheck source=/dev/null + . /etc/profile + + # Beware, there are some stupid arbitrary rules on how you can + # use "$" in arguments to commands with "sudo -i". ${foo} or + # ${1} is OK, but $foo or $1 isn't. + # https://bugzilla.sudo.ws/show_bug.cgi?id=765 + sudo --preserve-env=SOURCE_DATE_EPOCH \ + --preserve-env=BUILDTOOL \ + --preserve-env=BUILDTOOLVER \ + -iu builduser bash -c 'cd /startdir; makepkg "$@"' -bash "$@" + ret=$? + case $ret in + 0|14) + return 0;; + *) + return $ret;; + esac +} + +_chrootnamcap() { + pacman -S --needed --noconfirm namcap + for pkgfile in /startdir/PKGBUILD /pkgdest/*; do + echo "Checking ${pkgfile##*/}" + sudo -u builduser namcap "$pkgfile" 2>&1 | tee "/logdest/${pkgfile##*/}-namcap.log" + done +} + +download_sources() { + setup_workdir + chown "$makepkg_user:" "$WORKDIR" + + # Ensure sources are downloaded + sudo -u "$makepkg_user" --preserve-env=GNUPGHOME,SSH_AUTH_SOCK \ + env SRCDEST="$SRCDEST" BUILDDIR="$WORKDIR" \ + makepkg --config="$copydir/etc/makepkg.conf" --verifysource -o "${verifysource_args[@]}" || + die "Could not download sources." +} + +move_logfiles() { + local l + for l in "$copydir"/logdest/*; do + [[ $l == */logpipe.* ]] && continue + chown "$src_owner" "$l" + mv "$l" "$LOGDEST" + done +} + +move_products() { + local pkgfile + for pkgfile in "$copydir"/pkgdest/*; do + chown "$src_owner" "$pkgfile" + mv "$pkgfile" "$PKGDEST" + + # Fix broken symlink because of temporary chroot PKGDEST /pkgdest + if [[ "$PWD" != "$PKGDEST" && -L "$PWD/${pkgfile##*/}" ]]; then + ln -sf "$PKGDEST/${pkgfile##*/}" + fi + done + + move_logfiles + + for s in "$copydir"/srcpkgdest/*; do + chown "$src_owner" "$s" + mv "$s" "$SRCPKGDEST" + + # Fix broken symlink because of temporary chroot SRCPKGDEST /srcpkgdest + if [[ "$PWD" != "$SRCPKGDEST" && -L "$PWD/${s##*/}" ]]; then + ln -sf "$SRCPKGDEST/${s##*/}" + fi + done +} +# }}} + +while getopts 'hcur:I:l:nCTD:d:U:' arg; do + case "$arg" in + c) clean_first=1 ;; + D) bindmounts_ro+=("--bind-ro=$OPTARG") ;; + d) bindmounts_rw+=("--bind=$OPTARG") ;; + u) update_first=1 ;; + r) passeddir="$OPTARG" ;; + I) install_pkgs+=("$OPTARG") ;; + l) copy="$OPTARG" ;; + n) run_namcap=1; makepkg_args+=(--install) ;; + C) run_checkpkg=1 ;; + T) temp_chroot=1; copy+="-$$" ;; + U) makepkg_user="$OPTARG" ;; + h|*) usage ;; + esac +done + +[[ ! -f PKGBUILD && -z "${install_pkgs[*]}" ]] && die 'This must be run in a directory containing a PKGBUILD.' +[[ -n $makepkg_user && -z $(id -u "$makepkg_user") ]] && die 'Invalid makepkg user.' +makepkg_user=${makepkg_user:-${SUDO_USER:-$USER}} + +check_root SOURCE_DATE_EPOCH,BUILDTOOL,BUILDTOOLVER,GNUPGHOME,SRCDEST,SRCPKGDEST,PKGDEST,LOGDEST,MAKEFLAGS,PACKAGER + +# Canonicalize chrootdir, getting rid of trailing / +chrootdir=$(readlink -e "$passeddir") +[[ ! -d $chrootdir ]] && die "No chroot dir defined, or invalid path '%s'" "$passeddir" +[[ ! -d $chrootdir/root ]] && die "Missing chroot dir root directory. Try using: mkarchroot %s/root base-devel" "$chrootdir" + +if [[ ${copy:0:1} = / ]]; then + copydir=$copy +else + copydir="$chrootdir/$copy" +fi + +# Pass all arguments after -- right to makepkg +makepkg_args+=("${@:$OPTIND}") + +# See if -R or -e was passed to makepkg +for arg in "${@:$OPTIND}"; do + case ${arg%%=*} in + --skip*|--holdver) verifysource_args+=("$arg") ;; + --repackage|--noextract) keepbuilddir=1 ;; + --*) ;; + -*R*|-*e*) keepbuilddir=1 ;; + esac +done + +umask 0022 + +ORIG_HOME=$HOME +IFS=: read -r _ _ _ _ _ HOME _ < <(getent passwd "${SUDO_USER:-$USER}") +load_makepkg_config +HOME=$ORIG_HOME + +# Use PKGBUILD directory if these don't exist +[[ -d $PKGDEST ]] || PKGDEST=$PWD +[[ -d $SRCDEST ]] || SRCDEST=$PWD +[[ -d $SRCPKGDEST ]] || SRCPKGDEST=$PWD +[[ -d $LOGDEST ]] || LOGDEST=$PWD + +# Lock the chroot we want to use. We'll keep this lock until we exit. +lock 9 "$copydir.lock" "Locking chroot copy [%s]" "$copy" + +if [[ ! -d $copydir ]] || (( clean_first )); then + sync_chroot "$chrootdir" "$copydir" "$copy" +fi + +(( update_first )) && arch-nspawn "$copydir" \ + "${bindmounts_ro[@]}" "${bindmounts_rw[@]}" \ + pacman -Syuu --noconfirm + +if [[ -n ${install_pkgs[*]:-} ]]; then + install_packages + ret=$? + # If there is no PKGBUILD we are done + [[ -f PKGBUILD ]] || exit $ret +fi + +if [[ "$(id -u "$makepkg_user")" == 0 ]]; then + error "Running makepkg as root is not allowed." + exit 1 +fi + +download_sources + +prepare_chroot + +if arch-nspawn "$copydir" \ + --bind="${PWD//:/\\:}:/startdir" \ + --bind="${SRCDEST//:/\\:}:/srcdest" \ + "${bindmounts_ro[@]}" "${bindmounts_rw[@]}" \ + /chrootbuild "${makepkg_args[@]}" +then + mapfile -t pkgnames < <(sudo -u "$makepkg_user" bash -c 'source PKGBUILD; printf "%s\n" "${pkgname[@]}"') + move_products +else + (( ret += 1 )) + move_logfiles +fi + +(( temp_chroot )) && delete_chroot "$copydir" "$copy" + +if (( ret != 0 )); then + if (( temp_chroot )); then + die "Build failed" + else + die "Build failed, check %s/build" "$copydir" + fi +else + if (( run_checkpkg )); then + msg "Running checkpkg" + + mapfile -t remotepkgs < <(pacman --config "$copydir"/etc/pacman.conf \ + --dbpath "$copydir"/var/lib/pacman \ + -Sddp "${pkgnames[@]}") + + if ! wait $!; then + warning "Skipped checkpkg due to missing repo packages" + exit 0 + fi + + # download package files if any non-local location exists + for remotepkg in "${remotepkgs[@]}"; do + if [[ $remotepkg != file://* ]]; then + msg2 "Downloading current versions" + arch-nspawn "$copydir" pacman --noconfirm -Swdd "${pkgnames[@]}" + mapfile -t remotepkgs < <(pacman --config "$copydir"/etc/pacman.conf \ + --dbpath "$copydir"/var/lib/pacman \ + -Sddp "${pkgnames[@]}") + break + fi + done + + msg2 "Checking packages" + sudo -u "$makepkg_user" checkpkg --rmdir --warn --makepkg-config "$copydir/etc/makepkg.conf" "${remotepkgs[@]/#file:\/\//}" + fi + true +fi diff --git a/src/makerepropkg.in b/src/makerepropkg.in new file mode 100644 index 0000000..b271f25 --- /dev/null +++ b/src/makerepropkg.in @@ -0,0 +1,270 @@ +#!/bin/bash +# +# makerepropkg - rebuild a package to see if it is reproducible +# +# Copyright (c) 2019 by Eli Schwartz +# +# SPDX-License-Identifier: GPL-3.0-or-later + +m4_include(lib/common.sh) +m4_include(lib/archroot.sh) + +source /usr/share/makepkg/util/config.sh +source /usr/share/makepkg/util/message.sh + +declare -A buildinfo +declare -a buildenv buildopts installed installpkgs + +archiveurl='https://archive.archlinux.org/packages' +buildroot=/var/lib/archbuild/reproducible +diffoscope=0 + +chroot=$USER +[[ -n ${SUDO_USER:-} ]] && chroot=$SUDO_USER +[[ -z "$chroot" || $chroot = root ]] && chroot=copy + +parse_buildinfo() { + local line var val + + while read -r line; do + var="${line%% = *}" + val="${line#* = }" + case ${var} in + buildenv) + buildenv+=("${val}") + ;; + options) + buildopts+=("${val}") + ;; + installed) + installed+=("${val}") + ;; + *) + buildinfo["${var}"]="${val}" + ;; + esac + done +} + +get_pkgfile() { + local cdir=${cache_dirs[0]} + local pkgfilebase=${1} + local mode=${2} + local pkgname=${pkgfilebase%-*-*-*} + local pkgfile ext + + # try without downloading + if [[ ${mode} != localonly ]] && get_pkgfile "${pkgfilebase}" localonly; then + return 0 + fi + + for ext in .zst .xz ''; do + pkgfile=${pkgfilebase}.pkg.tar${ext} + + for c in "${cache_dirs[@]}"; do + if [[ -f ${c}/${pkgfile} ]]; then + cdir=${c} + break + fi + done + + for f in "${pkgfile}" "${pkgfile}.sig"; do + if [[ ! -f "${cdir}/${f}" ]]; then + if [[ ${mode} = localonly ]]; then + continue 2 + fi + msg2 "retrieving '%s'..." "${f}" >&2 + curl -Llf -# -o "${cdir}/${f}" "${archiveurl}/${pkgname:0:1}/${pkgname}/${f}" || continue 2 + fi + done + printf '%s\n' "file://${cdir}/${pkgfile}" + return 0 + done + + return 1 +} + +get_makepkg_conf() { + local fname=${1} + local makepkg_conf="${2}" + if ! buildtool_file=$(get_pkgfile "${fname}"); then + error "failed to retrieve ${fname}" + return 1 + fi + msg2 "using makepkg.conf from ${fname}" + bsdtar xOqf "${buildtool_file/file:\/\//}" usr/share/devtools/makepkg-x86_64.conf > "${makepkg_conf}" + return 0 +} + +usage() { + cat << __EOF__ +usage: ${BASH_SOURCE[0]##*/} [options] + +Run this script in a PKGBUILD dir to build a package inside a +clean chroot while attempting to reproduce it. The package file +will be used to derive metadata needed for reproducing the +package, including the .PKGINFO as well as the buildinfo. + +For more details see https://reproducible-builds.org/ + +OPTIONS + -d Run diffoscope if the package is unreproducible + -c Set pacman cache + -M Location of a makepkg config file + -l The directory name to use as the chroot namespace + Useful for maintaining multiple copies + Default: $chroot + -h Show this usage message +__EOF__ +} + +while getopts 'dM:c:l:h' arg; do + case "$arg" in + d) diffoscope=1 ;; + M) archroot_args+=(-M "$OPTARG") ;; + c) cache_dirs+=("$OPTARG") ;; + l) chroot="$OPTARG" ;; + h) usage; exit 0 ;; + *|?) usage; exit 1 ;; + esac +done +shift $((OPTIND - 1)) + +check_root + +[[ -f PKGBUILD ]] || { error "No PKGBUILD in current directory."; exit 1; } + +# without arguments, get list of packages from PKGBUILD +if [[ -z $1 ]]; then + mapfile -t pkgnames < <(source PKGBUILD; pacman -Sddp --print-format '%r/%n' "${pkgname[@]}") + wait $! || { + error "No package file specified and failed to retrieve package names from './PKGBUILD'." + plain "Try '${BASH_SOURCE[0]##*/} -h' for more information." >&2 + exit 1 + } + msg "Reproducing all pkgnames listed in ./PKGBUILD" + set -- "${pkgnames[@]}" +fi + +# check each package to see if it's a file, and if not, try to download it +# using pacman -Sw, and get the filename from there +splitpkgs=() +for p in "$@"; do + if [[ -f ${p} ]]; then + splitpkgs+=("${p}") + else + pkgfile_remote=$(pacman -Sddp "${p}" 2>/dev/null) || { error "package name '%s' not in repos" "${p}"; exit 1; } + pkgfile=${pkgfile_remote#file://} + if [[ ! -f ${pkgfile} ]]; then + msg "Downloading package '%s' into pacman's cache" "${pkgfile}" + sudo pacman -Swdd --noconfirm --logfile /dev/null "${p}" || exit 1 + pkgfile_remote=$(pacman -Sddp "${p}" 2>/dev/null) + pkgfile="${pkgfile_remote#file://}" + fi + splitpkgs+=("${pkgfile}") + fi +done + +for f in "${splitpkgs[@]}"; do + if ! bsdtar -tqf "${f}" .BUILDINFO >/dev/null 2>&1; then + error "file is not a valid pacman package: '%s'" "${f}" + exit 1 + fi +done + +if (( ${#cache_dirs[@]} == 0 )); then + mapfile -t cache_dirs < <(pacman-conf CacheDir) +fi + +ORIG_HOME=${HOME} +IFS=: read -r _ _ _ _ _ HOME _ < <(getent passwd "${SUDO_USER:-$USER}") +load_makepkg_config +HOME=${ORIG_HOME} +[[ -d ${SRCDEST} ]] || SRCDEST=${PWD} + +parse_buildinfo < <(bsdtar -xOqf "${splitpkgs[0]}" .BUILDINFO) +export SOURCE_DATE_EPOCH="${buildinfo[builddate]}" +PACKAGER="${buildinfo[packager]}" +BUILDDIR="${buildinfo[builddir]}" +BUILDTOOL="${buildinfo[buildtool]}" +BUILDTOOLVER="${buildinfo[buildtoolver]}" +PKGEXT=${splitpkgs[0]#${splitpkgs[0]%.pkg.tar*}} + +# nuke and restore reproducible testenv +namespace="$buildroot/$chroot" +lock 9 "${namespace}.lock" "Locking chroot namespace '%s'" "${namespace}" +for copy in "${namespace}"/*/; do + [[ -d ${copy} ]] || continue + subvolume_delete_recursive "${copy}" +done +rm -rf --one-file-system "${namespace}" +(umask 0022; mkdir -p "${namespace}") + +for fname in "${installed[@]}"; do + if ! allpkgfiles+=("$(get_pkgfile "${fname}")"); then + error "failed to retrieve ${fname}" + exit 1 + fi +done + +trap 'rm -rf $TEMPDIR' EXIT INT TERM QUIT +TEMPDIR=$(mktemp -d --tmpdir makerepropkg.XXXXXXXXXX) + +makepkg_conf="${TEMPDIR}/makepkg.conf" +# anything before buildtool support is pinned to the last none buildtool aware release +if [[ -z "${BUILDTOOL}" ]]; then + get_makepkg_conf "devtools-20210202-3-any" "${makepkg_conf}" || exit 1 +# prefere to assume devtools up until matching makepkg version so repository packages remain reproducible +elif [[ "${BUILDTOOL}" = makepkg ]] && (( $(vercmp "${BUILDTOOLVER}" 6.0.1) <= 0 )); then + get_makepkg_conf "devtools-20210202-3-any" "${makepkg_conf}" || exit 1 +# all devtools builds +elif [[ "${BUILDTOOL}" = devtools ]] && get_makepkg_conf "${BUILDTOOL}-${BUILDTOOLVER}" "${makepkg_conf}"; then + true +# fallback to current makepkg.conf +else + warning "Unknown buildtool (${BUILDTOOL}-${BUILDTOOLVER}), using fallback" + makepkg_conf=@pkgdatadir@/makepkg-x86_64.conf +fi +printf '%s\n' "${allpkgfiles[@]}" | mkarchroot -M "${makepkg_conf}" -U "${archroot_args[@]}" "${namespace}/root" - || exit 1 + +# use makechrootpkg to prep the build directory +makechrootpkg -r "${namespace}" -l build -- --packagelist || exit 1 + +# set detected makepkg.conf options +{ + for var in PACKAGER BUILDDIR BUILDTOOL BUILDTOOLVER PKGEXT; do + printf '%s=%s\n' "${var}" "${!var@Q}" + done + printf 'OPTIONS=(%s)\n' "${buildopts[*]@Q}" + printf 'BUILDENV=(%s)\n' "${buildenv[*]@Q}" +} >> "${namespace}/build"/etc/makepkg.conf +install -d -o "${SUDO_UID:-$UID}" -g "$(id -g "${SUDO_UID:-$UID}")" "${namespace}/build/${BUILDDIR}" + +# kick off the build +arch-nspawn "${namespace}/build" \ + --bind="${PWD}:/startdir" \ + --bind="${SRCDEST}:/srcdest" \ + /chrootbuild -C --noconfirm --log --holdver --skipinteg +ret=$? + +if (( ${ret} == 0 )); then + msg2 "built succeeded! built packages can be found in ${namespace}/build/pkgdest" + msg "comparing artifacts..." + + for pkgfile in "${splitpkgs[@]}"; do + comparefiles=("${pkgfile}" "${namespace}/build/pkgdest/${pkgfile##*/}") + if cmp -s "${comparefiles[@]}"; then + msg2 "Package '%s' successfully reproduced!" "${pkgfile}" + else + ret=1 + warning "Package '%s' is not reproducible. :(" "${pkgfile}" + sha256sum "${comparefiles[@]}" + if (( diffoscope )); then + diffoscope "${comparefiles[@]}" + fi + fi + done +fi + +# return failure from chrootbuild, or the reproducibility status +exit ${ret} diff --git a/src/mkarchroot.in b/src/mkarchroot.in new file mode 100644 index 0000000..d199bed --- /dev/null +++ b/src/mkarchroot.in @@ -0,0 +1,95 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +m4_include(lib/common.sh) +m4_include(lib/archroot.sh) + +# umask might have been changed in /etc/profile +# ensure that sane default is set again +umask 0022 + +working_dir='' +umode='' + +files=() +nspawn_args=() + +usage() { + echo "Usage: ${0##*/} [options] working-dir package-list..." + echo ' options:' + echo ' -U Use pacman -U to install packages' + echo ' -C Location of a pacman config file' + echo ' -M Location of a makepkg config file' + echo ' -c Set pacman cache' + echo ' -f Copy file from the host to the chroot' + echo ' -s Do not run setarch' + echo ' -h This message' + exit 1 +} + +while getopts 'hUC:M:c:f:s' arg; do + case "$arg" in + U) umode=U ;; + C) pac_conf="$OPTARG" ;; + M) makepkg_conf="$OPTARG" ;; + c) cache_dirs+=("$OPTARG") ;; + f) files+=("$OPTARG") ;; + s) nosetarch=1 ;; + h|?) usage ;; + *) error "invalid argument '%s'" "$arg"; usage ;; + esac + if [[ $arg != U ]]; then + nspawn_args+=("-$arg") + [[ -v OPTARG ]] && nspawn_args+=("$OPTARG") + fi +done +shift $((OPTIND - 1)) + +(( $# < 2 )) && die 'You must specify a directory and one or more packages.' + +check_root + +working_dir="$(readlink -f "$1")" +shift 1 + +[[ -z $working_dir ]] && die 'Please specify a working directory.' + + +if (( ${#cache_dirs[@]} == 0 )); then + mapfile -t cache_dirs < <(pacman-conf CacheDir) +fi + +umask 0022 + +[[ -e $working_dir ]] && die "Working directory '%s' already exists" "$working_dir" + +mkdir -p "$working_dir" + +lock 9 "${working_dir}.lock" "Locking chroot" + +if is_btrfs "$working_dir"; then + rmdir "$working_dir" + if ! btrfs subvolume create "$working_dir"; then + die "Couldn't create subvolume for '%s'" "$working_dir" + fi + chmod 0755 "$working_dir" +fi + +for file in "${files[@]}"; do + mkdir -p "$(dirname "$working_dir$file")" + cp "$file" "$working_dir$file" +done + +unshare --mount pacstrap -${umode}Mcd ${pac_conf:+-C "$pac_conf"} "$working_dir" \ + "${cache_dirs[@]/#/--cachedir=}" "$@" || die 'Failed to install all packages' + +printf '%s.UTF-8 UTF-8\n' C en_US de_DE > "$working_dir/etc/locale.gen" +echo 'LANG=C.UTF-8' > "$working_dir/etc/locale.conf" +echo "$CHROOT_VERSION" > "$working_dir/.arch-chroot" + +systemd-machine-id-setup --root="$working_dir" + +exec arch-nspawn \ + "${nspawn_args[@]}" \ + "$working_dir" locale-gen diff --git a/src/offload-build.in b/src/offload-build.in new file mode 100644 index 0000000..9e9d71e --- /dev/null +++ b/src/offload-build.in @@ -0,0 +1,121 @@ +#!/bin/bash +# +# offload-build - build a PKGBUILD on a remote server using makechrootpkg. +# +# Copyright (c) 2019 by Eli Schwartz +# +# SPDX-License-Identifier: GPL-3.0-or-later + +source /usr/share/makepkg/util/config.sh + + +# global defaults suitable for use by Arch staff +repo=extra +arch=x86_64 +server=build.archlinux.org + +die() { printf "error: $1\n" "${@:2}"; exit 1; } + +usage() { + cat <<- _EOF_ + Usage: ${BASH_SOURCE[0]##*/} [--repo REPO] [--arch ARCHITECTURE] [--server SERVER] -- [ARCHBUILD_ARGS] + + Build a PKGBUILD on a remote server using makechrootpkg. Requires a remote user + that can run archbuild without password auth. Options passed after a -- are + passed on to archbuild, and eventually to makechrootpkg. + + OPTIONS + -r, --repo Build against a specific repository (current: $repo) + -a, --arch Build against a specific architecture (current: $arch) + -s, --server Offload to a specific build server (current: $server) + -h, --help Show this help text +_EOF_ +} + +# option checking +while (( $# )); do + case $1 in + -h|--help) + usage + exit 0 + ;; + -r|--repo) + repo=$2 + shift 2 + ;; + -a|--arch) + arch=$2 + shift 2 + ;; + -s|--server) + server=$2 + shift 2 + ;; + --) + shift + break + ;; + *) + die "invalid argument: %s" "$1" + ;; + esac +done + +# multilib must be handled specially +archbuild_arch="${arch}" +if [[ $repo = multilib* ]]; then + archbuild_arch= +fi + +archbuild_cmd=("${repo}${archbuild_arch:+-$archbuild_arch}-build" "$@") + +trap 'rm -rf $TEMPDIR' EXIT INT TERM QUIT + +# Load makepkg.conf variables to be available +load_makepkg_config + +# Use a source-only tarball as an intermediate to transfer files. This +# guarantees the checksums are okay, and guarantees that all needed files are +# transferred, including local sources, install scripts, and changelogs. +export TEMPDIR=$(mktemp -d --tmpdir offload-build.XXXXXXXXXX) +export SRCPKGDEST=${TEMPDIR} +makepkg --source || die "unable to make source package" + +# Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else +# but an empty src dir is created in PWD. Remove once fixed in makepkg. +rmdir --ignore-fail-on-non-empty src 2>/dev/null || true + +mapfile -t files < <( + # This is sort of bash golfing but it allows running a mildly complex + # command over ssh with a single connection. + # shellcheck disable=SC2145 + cat "$SRCPKGDEST"/*"$SRCEXT" | + ssh $server ' + temp="${XDG_CACHE_HOME:-$HOME/.cache}/offload-build" && + mkdir -p "$temp" && + temp=$(mktemp -d -p "$temp") && + cd "$temp" && + { + bsdtar --strip-components 1 -xvf - && + script -qefc "'"${archbuild_cmd[@]@Q}"'" /dev/null && + printf "%s\n" "" "-> build complete" && + printf "\t%s\n" "$temp"/* + } >&2 && + makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" && + makepkg_config="/usr/share/devtools/makepkg-'"${arch}"'.conf" && + if [[ -f /usr/share/devtools/makepkg-'"${repo}"'-'"${arch}"'.conf ]]; then + makepkg_config="/usr/share/devtools/makepkg-'"${repo}"'-'"${arch}"'.conf" + fi && + makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist && + printf "%s\n" "${temp}/PKGBUILD" +') + + +if (( ${#files[@]} )); then + printf '%s\n' '' '-> copying files...' + scp "${files[@]/#/$server:}" "${TEMPDIR}/" + mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/" + mv "${TEMPDIR}/PKGBUILD" "${PWD}/" +else + exit 1 +fi diff --git a/src/rebuildpkgs.in b/src/rebuildpkgs.in new file mode 100644 index 0000000..164bf08 --- /dev/null +++ b/src/rebuildpkgs.in @@ -0,0 +1,111 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This script rebuilds a list of packages in order +# and reports anything that fails +# +# Due to sudo usage, it is recommended to allow makechrootpkg +# to be run with NOPASSWD in your sudoers file +# +# FIXME +# Currently uses $(pwd)/rebuilds as the directory for rebuilding... +# TODO make this work for community too + +m4_include(lib/common.sh) + +if (( $# < 1 )); then + printf 'Usage: %s \n' "$(basename "${BASH_SOURCE[0]}")" + printf ' example: %s ~/chroot readline bash foo bar baz\n' "$(basename "${BASH_SOURCE[0]}")" + exit 1 +fi + +# Source makepkg.conf; fail if it is not found +if [[ -r '/etc/makepkg.conf' ]]; then + # shellcheck source=config/makepkg/x86_64.conf + source '/etc/makepkg.conf' +else + die '/etc/makepkg.conf not found!' +fi + +# Source user-specific makepkg.conf overrides +if [[ -r "${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" ]]; then + # shellcheck source=/dev/null + source "${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" +elif [[ -r "$HOME/.makepkg.conf" ]]; then + # shellcheck source=/dev/null + source "$HOME/.makepkg.conf" +fi + +bump_pkgrel() { + # Get the current pkgrel from SVN and update the working copy with it + # This prevents us from incrementing out of control :) + pbuild='.svn/text-base/PKGBUILD.svn-base' + oldrel=$(grep 'pkgrel=' $pbuild | cut -d= -f2) + + #remove decimals + rel=${oldrel%%.*} + + newrel=$((rel + 1)) + + sed -i "s/pkgrel=$oldrel/pkgrel=$newrel/" PKGBUILD +} + +pkg_from_pkgbuild() { + # we want the sourcing to be done in a subshell so we don't pollute our current namespace + export CARCH PKGEXT + # shellcheck source=PKGBUILD.proto + (source PKGBUILD; echo "$pkgname-$pkgver-$pkgrel-$CARCH$PKGEXT") +} + +chrootdir="$1"; shift +pkgs=("$@") + +SVNPATH='svn+ssh://repos.archlinux.org/srv/repos/svn-packages/svn' + +msg "Work will be done in %s" "$(pwd)/rebuilds" + +REBUILD_ROOT="$(pwd)/rebuilds" +mkdir -p "$REBUILD_ROOT" +cd "$REBUILD_ROOT" + +/usr/bin/svn co -N $SVNPATH + +FAILED="" +for pkg in "${pkgs[@]}"; do + cd "$REBUILD_ROOT/svn-packages" + + msg2 "Building '%s'" "$pkg" + /usr/bin/svn update "$pkg" + if [[ ! -d "$pkg/trunk" ]]; then + FAILED="$FAILED $pkg" + warning "%s does not exist in SVN" "$pkg" + continue + fi + cd "$pkg/trunk/" + + bump_pkgrel + + if ! sudo makechrootpkg -u -d -r "$chrootdir" -- --noconfirm; then + FAILED="$FAILED $pkg" + error "%s Failed!" "$pkg" + else + pkgfile=$(pkg_from_pkgbuild) + if [[ -e $pkgfile ]]; then + msg2 "%s Complete" "$pkg" + else + FAILED="$FAILED $pkg" + error "%s Failed, no package built!" "$pkg" + fi + fi +done + +cd "$REBUILD_ROOT" +if [[ -n $FAILED ]]; then + msg 'Packages failed:' + for pkg in $FAILED; do + msg2 "%s" "$pkg" + done +fi + +msg 'SVN pkgbumps in svn-packages/ - commit when ready' diff --git a/src/sogrep.in b/src/sogrep.in new file mode 100644 index 0000000..d1ca1a1 --- /dev/null +++ b/src/sogrep.in @@ -0,0 +1,170 @@ +#!/bin/bash +# +# sogrep - find shared library links in an Arch Linux repository. +# +# Copyright (c) 2019 by Eli Schwartz +# +# SPDX-License-Identifier: GPL-3.0-or-later + +m4_include(lib/common.sh) + +# globals +: ${SOLINKS_MIRROR:="https://mirror.pkgbuild.com"} +: ${SOCACHE_DIR:="${XDG_CACHE_HOME:-${HOME}/.cache}/sogrep"} + +m4_include(lib/valid-repos.sh) +arches=('x86_64') + +# options +REFRESH=0 +VERBOSE=0 + +source /usr/share/makepkg/util/parseopts.sh +source /usr/share/makepkg/util/util.sh + +recache() { + local repo arch verbosity=-s + + (( VERBOSE )) && verbosity=--progress-bar + + for repo in "${_repos[@]}"; do + for arch in "${arches[@]}"; do + # delete extracted tarballs from previous sogrep versions + rm -rf "${SOCACHE_DIR}/${arch}/${repo}" + + # fetch repo links database if newer than our cached copy + local dbpath=${SOCACHE_DIR}/${arch}/${repo}.links.tar.gz + mkdir -p "${dbpath%/*}" + (( VERBOSE )) && echo "Fetching ${repo}.links.tar.gz..." + if ! curl -fLR "${verbosity}" -o "${dbpath}" -z "${dbpath}" \ + "${SOLINKS_MIRROR}/${repo}/os/${arch}/${repo}.links.tar.gz"; then + echo "error: failed to download links database for repo ${repo}" + exit 1 + fi + done + done +} + +is_outdated_cache() { + local repo arch + + # links databases are generated at about the same time every day; we should + # attempt to check for new database files if any of them are over a day old + + for repo in "${_repos[@]}"; do + for arch in "${arches[@]}"; do + local dbpath=${SOCACHE_DIR}/${arch}/${repo}.links.tar.gz + if [[ ! -f ${dbpath} ]] || [[ $(find "${dbpath}" -mtime +0) ]]; then + return 0 + fi + done + done + + return 1 +} + +search() { + local repo=$1 arch lib=$2 srepos=("${_repos[@]}") + + if [[ $repo != all ]]; then + if ! in_array "${repo}" "${_repos[@]}"; then + echo "${BASH_SOURCE[0]##*/}: unrecognized repo '$repo'" + echo "Try '${BASH_SOURCE[0]##*/} --help' for more information." + exit 1 + fi + srepos=("${repo}") + fi + + setup_workdir + + for arch in "${arches[@]}"; do + for repo in "${srepos[@]}"; do + local prefix= + (( VERBOSE && ${#srepos[@]} > 1 )) && prefix=${repo}/ + local db=${SOCACHE_DIR}/${arch}/${repo}.links.tar.gz + if [[ -f ${db} ]]; then + local extracted=${WORKDIR}/${arch}/${repo} + mkdir -p "${extracted}" + bsdtar -C "${extracted}" -xf "${db}" + while read -rd '' pkg; do + read -r match + pkg=${pkg#${extracted}/} + pkg="${prefix}${pkg%-*-*/links}" + + if (( VERBOSE )); then + printf '%-35s %s\n' "${pkg}" "${match}" + else + printf '%s\n' "${pkg}" + fi + done < <(grep -rZ "${lib}" "${extracted}") | sort -u + fi + done + done | resort +} + +usage() { + cat <<- _EOF_ + Usage: ${BASH_SOURCE[0]##*/} [OPTIONS] REPO LIBNAME + + Check the soname links database for Arch Linux repositories containing + packages linked to a given shared library. If the repository specified + is "all", then all repositories will be searched, otherwise only the + named repository will be searched. + + If the links database does not exist, it will be downloaded first. + + OPTIONS + -v, --verbose Show matched links in addition to pkgname + -r, --refresh Refresh the links databases + -h, --help Show this help text +_EOF_ +} + +# utility function to resort with multiple repos + no-verbose +resort() { sort -u; } + +if (( $# == 0 )); then + echo "error: No arguments passed." + echo "Try '${BASH_SOURCE[0]##*/} --help' for more information." + exit 1 +fi +OPT_SHORT='vrh' +OPT_LONG=('verbose' 'refresh' 'help') +if ! parseopts "$OPT_SHORT" "${OPT_LONG[@]}" -- "$@"; then + exit 1 +fi +set -- "${OPTRET[@]}" + +while :; do + case $1 in + -v|--verbose) + resort() { cat; } + VERBOSE=1 + ;; + -r|--refresh) + REFRESH=1 + ;; + -h|--help) + usage + exit 0 + ;; + --) + shift; break + ;; + esac + shift +done + +if ! (( ( REFRESH && $# == 0 ) || $# == 2 )); then + echo "error: Incorrect number of arguments passed." + echo "Try '${BASH_SOURCE[0]##*/} --help' for more information." + exit 1 +fi + +# trigger a refresh if requested explicitly or the cached dbs might be outdated +if (( REFRESH )) || [[ ! -d ${SOCACHE_DIR} ]] || is_outdated_cache; then + recache + (( $# == 2 )) || exit 0 +fi + +search "$@" -- cgit v1.2.3-54-g00ecf