Send patches - preferably formatted by git format-patch - to patches at archlinux32 dot org.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorErich Eckner <git@eckner.net>2017-06-19 13:47:32 +0200
committerErich Eckner <git@eckner.net>2017-06-19 13:47:32 +0200
commite4677e04f39bfca06ca7b1fb92f89b7684f20e87 (patch)
tree8d6724ef0ee19e840d8e62df5c71ec70b1e9cd3e
parent5e7e1ac0a4932156cfd399447968959f6081d3fe (diff)
switch from sshfs to rsync - to avoid needing to remount
-rwxr-xr-xbin/build-master-status27
-rwxr-xr-xbin/common-functions28
-rwxr-xr-xbin/db-update196
-rwxr-xr-xbin/return-assignment23
-rwxr-xr-xconf/default.conf4
5 files changed, 198 insertions, 80 deletions
diff --git a/bin/build-master-status b/bin/build-master-status
index 84e5dc6..ab052be 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -5,8 +5,13 @@
. "${0%/*}/../conf/default.conf"
stable="$(
- find "${master_mirror_directory}/i686/" \( -name '*testing' -o -name '*staging' \) -prune -o -name '*.pkg.tar.xz' -print 2> /dev/null | \
- wc -l
+ ${master_mirror_command} "${master_mirror_directory}/i686/" | \
+ awk '{print $5}' | \
+ grep -v 'testing$\|staging$\|^\.$' | \
+ while read -r dir; do
+ ${master_mirror_command} "${master_mirror_directory}/i686/${dir}/"
+ done | \
+ grep -c '\.pkg\.tar\.xz$'
)"
tasks="$(
wc -l < \
@@ -22,12 +27,22 @@ pending_packages="$(
wc -l
)"
staging="$(
- find "${master_mirror_directory}/i686/"*staging -name '*.pkg.tar.xz' 2> /dev/null | \
- wc -l
+ ${master_mirror_command} "${master_mirror_directory}/i686/" | \
+ awk '{print $5}' | \
+ grep 'staging$' | \
+ while read -r dir; do
+ ${master_mirror_command} "${master_mirror_directory}/i686/${dir}/"
+ done | \
+ grep -c '\.pkg\.tar\.xz$'
)"
testing="$(
- find "${master_mirror_directory}/i686/"*testing -name '*.pkg.tar.xz' 2> /dev/null | \
- wc -l
+ ${master_mirror_command} "${master_mirror_directory}/i686/" | \
+ awk '{print $5}' | \
+ grep 'testing$' | \
+ while read -r dir; do
+ ${master_mirror_command} "${master_mirror_directory}/i686/${dir}/"
+ done | \
+ grep -c '\.pkg\.tar\.xz$'
)"
broken="$(
ls "${work_dir}/package-states/" | \
diff --git a/bin/common-functions b/bin/common-functions
index 911a580..cc59e94 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -106,7 +106,7 @@ find_repository_with_commit() {
return 0
fi
done
- >&2 echo "can't find repository with commit '$1'"
+ >&2 printf 'find_repository_with_commit: Cannot find repository with commit "%s"\n' "$1"
exit 1
}
@@ -299,18 +299,30 @@ official_or_community() {
esac
}
-# remove_old_package_versions $package_file
-# removes all other versions of $package_file in the current directory
+# remove_old_package_versions $directory $package_file
+# removes all other versions of $package_file in $directory on the master mirror
remove_old_package_versions() {
- local package="$1"
+ local directory="$1"
+ local package="$2"
+
local pkgname="${package%-*-*-*.pkg.tar.xz}"
- ls | \
- grep "^$(str_to_regex "${pkgname}")\(-[^-]\+\)\{3\}\.pkg\.tar\.xz\(\.sig\)\?\$" | \
- grep -v "^$(str_to_regex "${package}")\(\.sig\)\?\$" | \
- xargs -rn1 rm
+ ${master_mirror_command} \
+ --recursive \
+ --delete \
+ $( \
+ ${master_mirror_command} \
+ "${master_mirror_directory}/${directory}/" | \
+ awk '{print $5}' | \
+ grep "^$(str_to_regex "${pkgname}")\(-[^-]\+\)\{3\}\.pkg\.tar\.xz\(\.sig\)\?\$" | \
+ grep -v "^$(str_to_regex "${package}")\(\.sig\)\?\$" | \
+ sed 's|^|--include=|'
+ ) \
+ '--exclude=*' \
+ ./ \
+ "${master_mirror_directory}/${directory}/"
}
diff --git a/bin/db-update b/bin/db-update
index 5284c6a..1163633 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -30,6 +30,131 @@ usage() {
[ -z "$1" ] && exit 1 || exit $1
}
+# move_package $package $from_repository $to_repository
+# the existence of a directory $tmp_dir is assumed
+
+move_package() {
+
+ if [ -z "${tmp_dir}" ] || [ ! -d "${tmp_dir}" ]; then
+ >&2 echo 'move_package: No tmp_dir provided.'
+ exit 2
+ fi
+
+ local package="$1"
+ local from_repo="$2"
+ local to_repo="$3"
+ local from_ending
+ local to_ending
+ local part
+
+ if echo "${from_repo}" | \
+ grep -q 'staging$' && \
+ echo "${to_repo}" | \
+ grep -q 'testing$'; then
+ from_ending='done'
+ to_ending='testing'
+ elif echo "${from_repo}" | \
+ grep -q 'testing$' && \
+ ! echo "${to_repo}" | \
+ grep -q 'testing$\|staging$'; then
+ from_ending='testing'
+ to_ending=''
+ else
+ >&2 printf 'move_package: Cannot move package from "%s" to "%s".\n' "${from_repo}" "${to_repo}"
+ exit 2
+ fi
+
+ if [ ! -f "${work_dir}/package-states/${package}.${from_ending}" ]; then
+ >&2 printf 'move_package: Cannot find package state file "%s"\n' "${package}.${from_ending}"
+ exit 2
+ fi
+
+ rm -rf --one-file-system "${tmp_dir}/"*
+
+ cp \
+ "${work_dir}/package-states/${package}.${from_ending}" \
+ "${tmp_dir}/parts"
+ sed \
+ 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' \
+ "${tmp_dir}/parts" > \
+ "${tmp_dir}/parts_names"
+ sed \
+ 'p;s|$|.sig|' \
+ "${tmp_dir}/parts" > \
+ "${tmp_dir}/parts_and_signatures"
+
+ mkdir "${tmp_dir}/from"
+ mkdir "${tmp_dir}/to"
+
+ ${master_mirror_command} \
+ --files-from="${tmp_dir}/parts_and_signatures" \
+ "${master_mirror_directory}/i686/${from_repo}/" \
+ "${tmp_dir}/from/"
+
+ ${master_mirror_command} \
+ "${master_mirror_directory}/i686/${from_repo}/${from_repo}.db."* \
+ "${tmp_dir}/from"
+
+ ${master_mirror_command} \
+ "${master_mirror_directory}/i686/${to_repo}/${to_repo}.db."* \
+ "${tmp_dir}/to"
+
+ repo-remove \
+ "${tmp_dir}/from/${from_repo}.db.tar.gz" \
+ $(cat "${tmp_dir}/parts_names")
+
+ while read -r part; do
+ mv \
+ "${tmp_dir}/from/${part}" \
+ "${tmp_dir}/to/"
+ done < \
+ "${tmp_dir}/parts_and_signatures"
+
+ repo-add \
+ "${tmp_dir}/to/${to_repo}.db.tar.gz" \
+ $(
+ sed \
+ "s|^|${tmp_dir}/to/|" \
+ "${tmp_dir}/parts"
+ )
+
+ ${master_mirror_command} \
+ --files-from="${tmp_dir}/parts_and_signatures" \
+ "${tmp_dir}/to/" \
+ "${master_mirror_directory}/i686/${to_repo}/"
+
+ ${master_mirror_command} \
+ "${tmp_dir}/to/${to_repo}.db."* \
+ "${master_mirror_directory}/i686/${to_repo}/"
+
+ ${master_mirror_command} \
+ "${tmp_dir}/from/${from_repo}.db."* \
+ "${master_mirror_directory}/i686/${from_repo}/"
+
+ while read -r part; do
+ echo remove_old_package_versions "i686/${to_repo}" "${part}"
+ remove_old_package_versions "i686/${to_repo}" "${part}"
+ # the next line will remove _all_ versions of the package $part from $from_repo
+ echo remove_old_package_versions "i686/${from_repo}" "${part%.pkg.tar.xz}0.pkg.tar.xz"
+ remove_old_package_versions "i686/${from_repo}" "${part%.pkg.tar.xz}0.pkg.tar.xz"
+ done < \
+ "${tmp_dir}/parts"
+
+ if [ -z "${to_ending}" ]; then
+ rm \
+ "${work_dir}/package-states/${package}.${from_ending}"
+ else
+ mv \
+ "${work_dir}/package-states/${package}.${from_ending}" \
+ "${work_dir}/package-states/${package}.${to_ending}"
+ fi
+
+ updated_package_database=true
+
+ rm -rf --one-file-system "${tmp_dir}/"*
+
+}
+
eval set -- "$(
getopt -o bh \
--long block \
@@ -135,8 +260,9 @@ keep_packages="$(
"${tmp_dir}/done_packages"
)"
printf '%s\n' "${keep_packages}" | \
- grep -vxF '' || true > \
- "${tmp_dir}/keep_packages"
+ grep -vxF '' > \
+ "${tmp_dir}/keep_packages" || \
+ true
# find all dependencies of the unstageable packages
mv \
@@ -200,20 +326,11 @@ if [ -z "${delete_packages}" ]; then
rm -f "${build_list_lock_file}"
flock -u 9
- rm -rf --one-file-system "${tmp_dir}"
-
clean_up_lock_file() {
+ rm -rf --one-file-system "${tmp_dir}"
rm -f "${package_database_lock_file}"
}
-else
-
- rm -rf --one-file-system "${tmp_dir}"
-
- clean_up_lock_file() {
- rm -f "${package_database_lock_file}" "${build_list_lock_file}"
- }
-
fi
# calculate unstageable packages from keep_packages and done_packages
@@ -234,55 +351,28 @@ done_packages="$(
unset keep_packages
for package in ${done_packages}; do
+
if [ -z "${package}" ]; then
continue
fi
- parts=$(cat "${work_dir}/package-states/${package}.done")
- parts_names=$(
- printf '%s\n' ${parts} | \
- sed 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||'
- )
+
is_community="$(official_or_community "${package}")"
- cd "${master_mirror_directory}/i686/${is_community}staging"
- mkdir -p "../${is_community}testing"
- repo-remove "${is_community}staging.db.tar.gz" ${parts_names}
- for part in ${parts}; do
- mv "${part}" "${part}.sig" "../${is_community}testing/"
- updated_package_database=true
- done
- cd "${master_mirror_directory}/i686/${is_community}testing"
- repo-add "${is_community}testing.db.tar.gz" ${parts}
- for part in ${parts}; do
- remove_old_package_versions "${part}"
- done
-
- mv \
- "${work_dir}/package-states/${package}.done" \
- "${work_dir}/package-states/${package}.testing"
+
+ move_package "${package}" "${is_community}staging" "${is_community}testing"
+
done
+exit
+
# move packages in packages_to_stabilize from *testing/ to the stable repos
for package in ${packages_to_stabilize}; do
- parts=$(cat "${work_dir}/package-states/${package}.testing")
- parts_names=$(
- printf '%s\n' ${parts} | \
- sed 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||'
- )
- source="$(official_or_community ${package})testing"
- destination="${package##*.}"
- cd "${master_mirror_directory}/i686/${source}"
- repo-remove "${source}.db.tar.gz" ${parts_names}
- for part in ${parts}; do
- mv "${part}" "${part}.sig" "../${destination}/"
- updated_package_database=true
- done
- cd "${master_mirror_directory}/i686/${destination}"
- repo-add "${destination}.db.tar.gz" ${parts}
- for part in ${parts}; do
- remove_old_package_versions "${part}"
- done
-
- rm "${work_dir}/package-states/${package}.testing"
+
+ if [ -z "${package}" ]; then
+ continue
+ fi
+
+ move_package "${package}" "$(official_or_community ${package})testing" "${package##*.}"
+
done
if [ -n "${delete_packages}" ]; then
diff --git a/bin/return-assignment b/bin/return-assignment
index 78d888a..5252518 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -144,22 +144,21 @@ fi
# move packages
destination="$(official_or_community "$1.$2.$3.$4")staging"
-mkdir -p "${master_mirror_directory}/i686/${destination}/"
-mv \
+${master_mirror_command} \
+ "${master_mirror_directory}/i686/${destination}/${destination}.db.*" \
+ .
+repo-add "${destination}.db.tar.gz" ${packages}
+# repo-add -v -s -k "${repo_key}" "${destination}.db.tar.gz" ${packages}
+
+${master_mirror_command} \
+ "${destination}.db."*
*".pkg.tar.xz" \
*".pkg.tar.xz.sig" \
"${master_mirror_directory}/i686/${destination}/"
-(
- cd "${master_mirror_directory}/i686/${destination}"
- repo-add "${destination}.db.tar.gz" ${packages}
- # repo-add -v -s -k "${repo_key}" "${destination}.db.tar.gz" ${packages}
-
- # remove old versions of same packages
- for package in ${packages}; do
- remove_old_package_versions "${package}"
- done
-)
+for package in ${packages}; do
+ remove_old_package_versions "i686/${destination}" "${package}"
+done
# remove old state files (these should be only "done" markers, but
# actually we don't care what it is)
diff --git a/conf/default.conf b/conf/default.conf
index 0e9dc6e..e4e3fc4 100755
--- a/conf/default.conf
+++ b/conf/default.conf
@@ -30,7 +30,9 @@ max_parallel_build_per_client=2
repo_key='0xdeadbeef'
package_key='0x15eebadc0de'
-master_mirror_directory='/path/to/master/mirror'
+
+master_mirror_command='rsync --password-file=/home/slave/rsync.password'
+master_mirror_directory='rsync://buildmaster@mirror.archlinux32.org/packages32'
# possibly pull in custom modifications