| index : builder | |
| Archlinux32 build system | gitolite user |
| summaryrefslogtreecommitdiff |
| -rwxr-xr-x | lib/common-functions | 307 |
diff --git a/lib/common-functions b/lib/common-functions index 8c263ab..f13b1f2 100755 --- a/lib/common-functions +++ b/lib/common-functions @@ -9,109 +9,6 @@ if [ -z "${base_dir}" ]; then . '../lib/load-configuration' fi -# find_pkgbuilds package repository git_repository git_revision mod_git_revision -# find the PKGBUILD and modification of $package from $repository -# sets $PKGBUILD and $PKGBUILD_mod - -find_pkgbuilds() { - - local package="$1" - local repository="$2" - local git_repository="$3" - local git_revision="$4" - local mod_git_revision="$5" - - local repo_path - eval 'repo_path="${repo_paths__'"${git_repository}"'}"' - - if [ ! "$(git -C "${repo_path}" cat-file -t "${git_revision}" 2> /dev/null)" = "commit" ]; then - >&2 printf 'Repository %s does not contain commit %s, but it should.\n' \ - "${git_repository}" \ - "${git_revision}" - return 2 - fi - - if [ ! "$(git -C "${repo_paths__archlinux32}" cat-file -t "${mod_git_revision}" 2> /dev/null)" = "commit" ]; then - >&2 printf 'Repository archlinux32 does not contain commit %s, but it should.\n' \ - "${mod_git_revision}" - return 2 - fi - - PKGBUILD=$( - git -C "${repo_path}" archive "${git_revision}" -- "${package}/repos/${repository}-*/PKGBUILD" 2> /dev/null | \ - tar -t 2> /dev/null | \ - grep -- '/PKGBUILD$' | \ - grep -v -- '-i686/PKGBUILD$' | \ - grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \ - sort | \ - tail -n1 - ) - - PKGBUILD_mod=$( - git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" -- "${repository}/${package}/PKGBUILD" 2> /dev/null | \ - tar -t "${repository}/${package}/PKGBUILD" 2> /dev/null - ) || true - - if [ -z "${PKGBUILD}" ] && \ - [ -z "${PKGBUILD_mod}" ]; then - >&2 printf 'Neither PKGBUILD nor modification of PKGBUILD found for package "%s" from %s (%s), revisions %s and %s.\n' \ - "${package}" \ - "${repository}" \ - "${git_repository}" \ - "${git_revision}" \ - "${mod_git_revision}" - return 1 - fi - -} - -# find_repository_with_commit commit -# find the repository which has $commit - -find_repository_with_commit() { - - local repository - - for repository in ${repo_names}; do - # shellcheck disable=SC2016 - if [ "$(eval git -C "$(printf '"${repo_paths__%s}"' "${repository}")" cat-file -t '"$1"' 2> /dev/null)" = "commit" ]; then - echo "${repository}" - return 0 - fi - done - >&2 printf 'find_repository_with_commit: Cannot find repository with commit "%s"\n' "$1" - return 2 - -} - -# find_git_repository_to_package_repository repository -# find the git repository which tracks the package repository $repository - -find_git_repository_to_package_repository() { - - local repository - - repository=$( - # shellcheck disable=SC2016 - { - printf 'SELECT `git_repositories`.`name`' - printf ' FROM `git_repositories`' - mysql_join_git_repositories_upstream_repositories - printf ' WHERE `upstream_repositories`.`name`=from_base64("%s");\n' \ - "$(printf '%s' "$1" | base64 -w0)" - } | \ - mysql_run_query - ) - if [ -z "${repository}" ]; then - >&2 echo "can't find git repository with package repository '$1'" - exit 1 - else - echo "${repository}" - return 0 - fi - -} - # ls_master_mirror $path # list content of $path on the master mirror (via rsync) @@ -122,7 +19,8 @@ ls_master_mirror() { ${master_mirror_rsync_command} \ "${master_mirror_rsync_directory}/${path}/" | \ grep -v '\s\.$' | \ - awk '{print $5}' + awk '{print $5}' | \ + grep -v build-support-manual } @@ -277,8 +175,8 @@ remove_old_package_versions() { # wait between minimum and minimum+diff seconds (diff defaults to 30) wait_some_time() { - local minimum=$1 - local diff=$2 + local minimum="$1" + local diff="$2" local random if [ -z "${diff}" ]; then @@ -304,29 +202,27 @@ str_to_regex() { ' } -# make_source_info $package $repository $git_revision $mod_git_revision $output -# create .SRCINFO from PKGBUILD within git repositories, output to $output - -make_source_info() { +make_source_info_using_cache() { local package="$1" local repository="$2" local git_revision="$3" local mod_git_revision="$4" local output="$5" - local git_repo - local PKGBUILD - local PKGBUILD_mod + curl -LSs "https://buildmaster.archlinux32.org/pkginfo/${package}=${repository}=${git_revision}=${mod_git_revision}" \ + >"${output}" +} - if ! git_repo=$(find_repository_with_commit "${git_revision}") || \ - [ -z "${git_repo}" ]; then - return 1 - fi +# make_source_info $package $repository $git_revision $mod_git_revision $output +# create .SRCINFO from PKGBUILD within git repositories, output to $output +make_source_info() { - if ! find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}"; then - return 1 - fi + local package="$1" + local repository="$2" + local git_revision="$3" + local mod_git_revision="$4" + local output="$5" ( # the new shell is intentional @@ -336,14 +232,10 @@ make_source_info() { tmp_dir=$(mktemp -d "${work_dir}/tmp.make_source_info.XXXXXX") trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" '0' + extract_source_directory "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" '0' { cd "${tmp_dir}" - # some additional info - printf 'upstream_git_repository = %s\n' "${git_repo}" - printf 'PKGBUILD = %s\n' "${PKGBUILD}" - printf 'PKGBUILD_mod = %s\n' "${PKGBUILD_mod}" makepkg --printsrcinfo cd .. } > \ @@ -358,6 +250,7 @@ make_source_info() { p ' "${tmp_dir}/SRCINFO" )" + # shellcheck disable=SC2031 sed -i ' /^pkgname = /! b /= gtk-doc$/ b @@ -447,7 +340,7 @@ mangle_pkgbuild() { /^arch=[^#]*any/!{ /^arch=(/s/(/(i486 i686 pentium4 / } - s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9.]\+\)['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel}"'"/ + s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9]\+\)\(\.[0-9.]*\)\?['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel:-\\3}"'"/ ' "${PKGBUILD}" # shellcheck disable=SC2016 @@ -520,42 +413,105 @@ find_package_repository_to_package() { "${git_repository}" \ "${git_commit}" >&2 printf '%s\n' "${repo}" - return 1 + >&2 printf 'trying to filter based on git\n' + + if ! repo=$( + # shellcheck disable=SC2016 + { + printf 'SELECT `upstream_repositories`.`name`' + printf ' FROM `upstream_repositories`' + mysql_join_upstream_repositories_git_repositories + printf ' WHERE `git_repositories`.`name`=from_base64("%s")' \ + "$( + printf '%s' "${git_repository}" \ + | base64 -w0 + )" + } | \ + mysql_run_query 'unimportant' | \ + grep -xF "${repo}" + ); then + >&2 printf 'find_package_repository_to_package %s %s %s: no repo found after filtering\n' \ + "${package}" \ + "${git_repository}" \ + "${git_commit}" + return 1 + fi + + if [ "$( + echo "${repo}" | \ + wc -l + )" -ne 1 ]; then + >&2 printf 'find_package_repository_to_package %s %s %s: still multiple repos found after filter:\n' \ + "${package}" \ + "${git_repository}" \ + "${git_commit}" + >&2 printf '%s\n' "${repo}" + return 1 + fi fi echo "${repo}" } -# extract_source_directory $git_repo $rev $mod_rev $output $sub_pkgrel +# extract_source_directory $pkgbase $repository $revision $mod_revision $output $sub_pkgrel # extract files found in the svn/git source directories -# $PKGBUILD and $PKGBUILD_mod are expected to be set correctly extract_source_directory() { - local git_repo="$1" - # shellcheck disable=SC2034 - local rev="$2" - local mod_rev="$3" - local output="$4" - local sub_pkgrel="$5" - - if [ -n "${PKGBUILD}" ]; then - eval 'git -C "${repo_paths__'"${git_repo}"'}" archive "${rev}" -- "${PKGBUILD%/*}"' | \ - tar -x --strip-components=3 -C "${output}" - printf '\n' >> \ - "${output}/PKGBUILD" + local pkgbase="$1" + local repository="$2" + local revision="$3" + local mod_revision="$4" + local output="$5" + local sub_pkgrel="$6" + local res + local tmptarfile + local pkgtarfile + + if [ -n "${revision}" ] \ + && [ "${revision}" != '0000000000000000000000000000000000000000' ] \ + && [ "${repository}" != 'build-support' ]; then + local pkgbase_translated + pkgbase_translated=$(gitlab_project_name_to_path "${pkgbase}") + tmptarfile=$(mktemp "${tmp_dir}"'/tmp.upstream-package.XXXXXXXXXX') + url="https://buildmaster.archlinux32.org/upstream-packages/${pkgbase_translated}-${revision}.tar.gz" + for i in {1..4}; do + curl -o "$tmptarfile" -fLSs "${url}" + res=$? + if [ "$res" -ne 0 ]; then + >&2 printf 'Download upstream-package "%s" from upstream via buildmaster cache failed. Curl exit code was %s.' \ + "${url}" \ + "${res}" + return 1 + fi + + # If zero bytes, try again + if [ -s "${tmptarfile}" ]; then + break + fi + + sleep 5 + done + if [ -f "${tmptarfile}" ]; then + tar -xz -f "${tmptarfile}" --strip-components=1 -C "${output}" -- "${pkgbase_translated}-${revision}" + printf '\n' >> \ + "${output}/PKGBUILD" + else + >&2 echo 'No upstream tarfile, assuming the package is a local one and has a full PKGBUILD not a diff-PKGBUILD.' + fi fi - if [ -n "${PKGBUILD_mod}" ]; then - git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod%/*}" | \ - tar -x --overwrite --exclude 'PKGBUILD' --strip-components=2 -C "${output}" 2> /dev/null || \ - true - git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod}" | \ - tar -Ox "${PKGBUILD_mod}" >> \ - "${output}/PKGBUILD" - printf '\n' >> \ - "${output}/PKGBUILD" + if [ -n "${mod_revision}" ] \ + && [ "${mod_revision}" != '0000000000000000000000000000000000000000' ]; then + git -C "${repo_paths__archlinux32}" archive "${mod_revision}" -- "${repository}/${pkgbase}" \ + |tar -x --overwrite --exclude 'PKGBUILD' --strip-components=2 -C "${output}" 2> /dev/null \ + ||true + git -C "${repo_paths__archlinux32}" archive "${mod_revision}" -- "${repository}/${pkgbase}/PKGBUILD" \ + |tar -Ox "${repository}/${pkgbase}/PKGBUILD" \ + >>"${output}/PKGBUILD" + printf '\n' \ + >>"${output}/PKGBUILD" fi # we do not want to update pkgver, so we just undefine it @@ -570,7 +526,6 @@ extract_source_directory() { # we don't want write permissions on the PKGBUILD - otherwise pkgver() # will change the version! (**HACK**) chmod oga-w "${output}/PKGBUILD" - } # download_sources_by_hash @@ -950,31 +905,6 @@ failsafe_rsync() { return 1 } -# apply_trunk_patch source_dir diff_source_dir -# apply a patch between diff_source_dir/PKGBUILD and -# diff_source_dir/../../trunk/PKGBUILD onto source_dir/PKGBUILD -apply_trunk_patch() { - local source_dir="$1" - local diff_source_dir="$2" - if [ -z "${diff_source_dir}" ]; then - # no diff_source_dir => no action - return - fi - chmod +w "${source_dir}/PKGBUILD" - diff -u3 "${diff_source_dir}/PKGBUILD" "${diff_source_dir}/../../trunk/PKGBUILD" \ - | sed ' - 1,2 s#^\(\(+++\|---\)\s\+\)\S\+/\(PKGBUILD\s.*\)$#\1\3# - ' \ - | patch -p0 "${source_dir}/PKGBUILD" - chmod -w "${source_dir}/PKGBUILD" - find "${diff_source_dir}/../../trunk/" \ - -mindepth 1 \ - -maxdepth 1 \ - -not -name 'PKGBUILD' \ - -type f \ - -exec cp '{}' "${source_dir}" \; -} - # update_blocked_packages_count # update the count how many packages are blocked by a build assignment update_blocked_packages_count() { @@ -1129,4 +1059,29 @@ expand_blacklist_architectures() { sort -k1,1 \ | join -1 1 -2 1 -o 1.2,2.2 "$1" - \ | sort -u + rm "$1" +} + +# taken verbatim from upstream archlinux: +# https://gitlab.archlinux.org/archlinux/devtools/-/raw/master/src/lib/api/gitlab.sh + +# Convert arbitrary project names to GitLab valid path names. +# +# GitLab has several limitations on project and group names and also maintains +# a list of reserved keywords as documented on their docs. +# https://docs.gitlab.com/ee/user/reserved_names.html +# +# 1. replace single '+' between word boundaries with '-' +# 2. replace any other '+' with literal 'plus' +# 3. replace any special chars other than '_', '-' and '.' with '-' +# 4. replace consecutive '_-' chars with a single '-' +# 5. replace 'tree' with 'unix-tree' due to GitLab reserved keyword +gitlab_project_name_to_path() { + local name=$1 + printf "%s" "${name}" \ + | sed -E 's/([a-zA-Z0-9]+)\+([a-zA-Z]+)/\1-\2/g' \ + | sed -E 's/\+/plus/g' \ + | sed -E 's/[^a-zA-Z0-9_\-\.]/-/g' \ + | sed -E 's/[_\-]{2,}/-/g' \ + | sed -E 's/^tree$/unix-tree/g' } |