Tools for building 32-bit archlinux packages from archlinux.org's official, 64-bit tested PKGBUILDs et al.

nit-picker 11KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336
  1. #!/bin/sh
  2. # shellcheck disable=SC2119
  3. # shellcheck source=../lib/load-configuration
  4. . "${0%/*}/../lib/load-configuration"
  5. # this script shall host all the tests that are too slow to be run on
  6. # the buildmaster (thus, it runs asynchronously and must therefor be
  7. # resilient against transient states of the buildmaster):
  8. # - check sanity of git
  9. # TODO:
  10. # - check for differences of dependencies between mysql and git
  11. # - check for differences of dependencies between mysql and packages
  12. # - check for installability of packages
  13. # - It would be also nice to have the possibility to fix broken
  14. # dependencies in the database automatically.
  15. if [ $# -ge 1 ] && [ "x$1" = 'x-n' ]; then
  16. >&2 echo 'not joining irc'
  17. irc=false
  18. shift
  19. else
  20. irc=true
  21. # shellcheck disable=SC2016
  22. if [ $# -ne 0 ]; then
  23. >&2 echo 'usage: nit-picker [-n [$single_test]]'
  24. >&2 echo ' -n: do not join irc'
  25. >&2 echo ' $single_test: only execute the given test'
  26. exit 1
  27. fi
  28. fi
  29. clean_up() {
  30. rm -rf --one-file-system "${tmp_dir}"
  31. }
  32. tmp_dir=$(mktemp -d 'tmp.nit-picker.XXXXXXXXXX' --tmpdir)
  33. trap 'clean_up' EXIT
  34. if ${irc}; then
  35. if pgrep -x ii; then
  36. >&2 echo 'ii is already running - this will not work'
  37. exit 1
  38. fi
  39. rm -rf --one-file-system "${irc_dir}"
  40. ii -s irc.freenode.net -n nit-picker -f nit-picker >/dev/null 2>&1 &
  41. ii_pid=$!
  42. clean_up() {
  43. rm -rf --one-file-system "${tmp_dir}"
  44. kill "${ii_pid}"
  45. }
  46. # wait for nickserv complaint
  47. while ! grep -qF 'This nickname is registered. Please choose a different nickname' "${irc_dir}/nickserv/out"; do
  48. sleep 1
  49. done
  50. # register
  51. printf 'identify %s\n' "${irc_password}" | \
  52. sponge "${irc_dir}/nickserv/in"
  53. # wait for registering to succeed
  54. while ! grep -qF 'You are now identified for' "${irc_dir}/nickserv/out"; do
  55. sleep 1
  56. done
  57. # join channel
  58. echo '/j #archlinux32' | \
  59. sponge "${irc_dir}/in"
  60. while [ ! -f "${irc_dir}/#archlinux32/out" ]; do
  61. sleep 1
  62. done
  63. else
  64. irc_say() {
  65. sed 's/^/irc: /'
  66. }
  67. fi
  68. mysql_load_min_and_max_versions
  69. while pgrep -x ii >/dev/null \
  70. || ! ${irc}; do
  71. if [ $# -eq 0 ]; then
  72. # shellcheck disable=SC2016
  73. {
  74. printf 'SELECT DISTINCT'
  75. printf ' "commit",'
  76. printf '`git_repositories`.`name`,'
  77. printf '`git_repositories`.`head`,'
  78. printf '`package_sources`.`git_revision`'
  79. printf ' FROM `package_sources`'
  80. mysql_join_package_sources_upstream_repositories
  81. mysql_join_upstream_repositories_git_repositories
  82. printf ';\n'
  83. printf 'SELECT DISTINCT'
  84. printf ' "commit",'
  85. printf '"archlinux32",'
  86. # shellcheck disable=SC2154
  87. printf '"%s",' \
  88. "${repo_heads__archlinux32}"
  89. printf '`package_sources`.`mod_git_revision`'
  90. printf ' FROM `package_sources`'
  91. printf ';\n'
  92. printf 'SELECT DISTINCT'
  93. printf ' "binary-signature",'
  94. mysql_package_name_query
  95. printf ' FROM `binary_packages`'
  96. mysql_join_binary_packages_architectures
  97. mysql_join_binary_packages_binary_packages_in_repositories
  98. mysql_join_binary_packages_in_repositories_repositories
  99. printf ' WHERE `repositories`.`is_on_master_mirror`'
  100. printf ';\n'
  101. printf 'SELECT DISTINCT'
  102. printf ' "binary-dependencies",'
  103. mysql_package_name_query
  104. printf ' FROM `binary_packages`'
  105. mysql_join_binary_packages_architectures
  106. mysql_join_binary_packages_binary_packages_in_repositories
  107. mysql_join_binary_packages_in_repositories_repositories
  108. printf ' WHERE `repositories`.`is_on_master_mirror`'
  109. printf ';\n'
  110. } | \
  111. mysql_run_query | \
  112. tr '\t' ' ' | \
  113. shuf
  114. else
  115. printf '%s\n' "$*"
  116. fi | \
  117. while read -r action parameters; do
  118. if ${irc} && ! pgrep -x ii >/dev/null; then
  119. break
  120. fi
  121. case "${action}" in
  122. 'commit') # check whether a given commit is present in the git repo
  123. git_repo="${parameters%% *}"
  124. git_rev="${parameters#${git_repo} }"
  125. git_head="${git_rev%% *}"
  126. git_rev="${git_rev#${git_head} }"
  127. # shellcheck disable=SC2016
  128. eval "$(
  129. printf 'git_dir="${repo_paths__%s}"\n' \
  130. "${git_repo}"
  131. )"
  132. # shellcheck disable=SC2154
  133. if ! git -C "${git_dir}" cat-file -t "${git_rev}" 2> /dev/null | \
  134. grep -qxF 'commit'; then
  135. git -C "${git_dir}" fetch --all -p >/dev/null 2>&1
  136. if ! git -C "${git_dir}" cat-file -t "${git_rev}" 2> /dev/null | \
  137. grep -qxF 'commit'; then
  138. printf 'commit %s is missing from repository %s\n' \
  139. "${git_rev}" \
  140. "${git_repo}" \
  141. | irc_say
  142. fi
  143. fi
  144. # shellcheck disable=SC2154
  145. if ! git -C "${git_dir}" cat-file -t "${git_head}" 2> /dev/null | \
  146. grep -qxF 'commit'; then
  147. git -C "${git_dir}" fetch --all -p >/dev/null 2>&1
  148. if ! git -C "${git_dir}" cat-file -t "${git_head}" 2> /dev/null | \
  149. grep -qxF 'commit'; then
  150. printf 'commit %s is missing from repository %s\n' \
  151. "${git_head}" \
  152. "${git_repo}" \
  153. | irc_say
  154. fi
  155. fi
  156. # shellcheck disable=SC2154
  157. if ! git -C "${git_dir}" merge-base --is-ancestor "${git_rev}" "${git_head}" 2> /dev/null; then
  158. current_git_head=$(
  159. # shellcheck disable=SC2016
  160. {
  161. printf 'SELECT DISTINCT'
  162. printf ' `git_repositories`.`head`'
  163. printf ' FROM `git_repositories`'
  164. printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
  165. "$(
  166. printf '%s' "${git_repo}" \
  167. | base64 -w0
  168. )"
  169. } \
  170. | mysql_run_query
  171. )
  172. if ! git -C "${git_dir}" merge-base --is-ancestor "${git_rev}" "${current_git_head}" 2> /dev/null; then
  173. printf 'commit %s is not an ancestor of HEAD %s in repository %s\n' \
  174. "${git_rev}" \
  175. "${current_git_head}" \
  176. "${git_repo}" \
  177. | irc_say
  178. fi
  179. fi
  180. ;;
  181. 'binary-dependencies')
  182. ${master_mirror_rsync_command} \
  183. "${master_mirror_rsync_directory}/pool/${parameters}" \
  184. "${tmp_dir}/"
  185. extract_dependencies_from_package \
  186. "${tmp_dir}/${parameters}" \
  187. > "${tmp_dir}/pkg-deps"
  188. # shellcheck disable=SC2016
  189. {
  190. printf 'SELECT'
  191. printf ' `dependency_types`.`name`,'
  192. printf '`install_targets`.`name`,'
  193. printf '`dependencies`.`version_relation`,'
  194. printf '`versions`.`epoch`,'
  195. printf '`versions`.`version`'
  196. printf ' FROM `binary_packages`'
  197. mysql_join_binary_packages_dependencies
  198. mysql_join_binary_packages_architectures
  199. mysql_join_dependencies_dependency_types
  200. mysql_join_dependencies_versions
  201. mysql_join_dependencies_install_targets
  202. printf ' WHERE '
  203. mysql_package_name_query
  204. printf '="%s"' \
  205. "${parameters}"
  206. printf ' AND `dependency_types`.`name` IN ("run","make","check")'
  207. printf ';\n'
  208. } \
  209. | mysql_run_query \
  210. | tr '\t' ' ' \
  211. | sort -u \
  212. > "${tmp_dir}/db-deps"
  213. if ! diff -q "${tmp_dir}/db-deps" "${tmp_dir}/pkg-deps"; then
  214. build_date=$(
  215. bsdtar -Oxf "${tmp_dir}/${parameters}" '.PKGINFO' \
  216. | sed '
  217. s/^builddate = //
  218. t
  219. d
  220. '
  221. )
  222. build_date=$(
  223. date -I -d@"${build_date}"
  224. )
  225. printf 'dependencies of %s (built on %s) differ between the package and our database\n' \
  226. "${parameters}" \
  227. "${build_date}" \
  228. | irc_say 'deep42thought'
  229. if ! ${irc}; then
  230. diff -u --color "${tmp_dir}/db-deps" "${tmp_dir}/pkg-deps" || true
  231. fi
  232. if [ $# -eq 0 ]; then
  233. sleep 60
  234. fi
  235. fi
  236. rm \
  237. "${tmp_dir}/${parameters}" \
  238. "${tmp_dir}/db-deps" \
  239. "${tmp_dir}/pkg-deps"
  240. ;;
  241. 'binary-signature')
  242. ${master_mirror_rsync_command} \
  243. "${master_mirror_rsync_directory}/pool/${parameters}" \
  244. "${master_mirror_rsync_directory}/pool/${parameters}.sig" \
  245. "${tmp_dir}/"
  246. unset error_message
  247. if ! gpg_output=$(
  248. gpg --batch --status-fd 1 -q --homedir /etc/pacman.d/gnupg \
  249. --verify "${tmp_dir}/${parameters}.sig" "${tmp_dir}/${parameters}" \
  250. 2>/dev/null
  251. ); then
  252. error_message="package ${parameters} has an invalid signature."
  253. fi
  254. if [ -z "${error_message}" ]; then
  255. gpg_key=$(
  256. printf '%s\n' "${gpg_output}" \
  257. | sed '
  258. s/^\[GNUPG:] KEY_CONSIDERED \([0-9A-F]\{40\}\) 0$/\1/
  259. t
  260. d
  261. ' \
  262. | sort -u
  263. )
  264. if [ -z "${gpg_key}" ]; then
  265. error_message="cannot find pgp_key of package ${parameters}."
  266. fi
  267. fi
  268. if [ -z "${error_message}" ]; then
  269. for expiration in $(
  270. gpg --batch --homedir /etc/pacman.d/gnupg --with-colons --list-keys "0x${gpg_key}" \
  271. 2>/dev/null \
  272. | grep '^\(sub\|pub\):' \
  273. | cut -d: -f7
  274. ); do
  275. expiration_days=$(((expiration - $(date +%s))/24/60/60))
  276. if [ ${expiration_days} -lt 100 ]; then
  277. error_message=$(
  278. printf 'signing key %s (from %s) for package %s expires on %s (in %s < 100 days).\n' \
  279. "${gpg_key}" \
  280. "$(
  281. gpg --batch --homedir /etc/pacman.d/gnupg --with-colons --list-keys "0x${gpg_key}" \
  282. 2>/dev/null \
  283. | grep '^\(uid\):' \
  284. | cut -d: -f10
  285. )" \
  286. "${parameters}" \
  287. "$(date -I -d@"${expiration}")" \
  288. "${expiration_days}"
  289. )
  290. break
  291. fi
  292. done
  293. fi
  294. if [ -n "${error_message}" ]; then
  295. printf '%s\n' "${error_message}" \
  296. | irc_say
  297. if [ $# -eq 0 ]; then
  298. sleep 60
  299. fi
  300. fi
  301. rm \
  302. "${tmp_dir}/${parameters}" \
  303. "${tmp_dir}/${parameters}.sig"
  304. ;;
  305. *)
  306. >&2 printf 'action "%s" is not yet implemented ...\n' "${action}"
  307. ;;
  308. esac
  309. done
  310. if [ $# -ge 1 ]; then
  311. break
  312. fi
  313. sleep 120
  314. done