Tools for building 32-bit archlinux packages from archlinux.org's official, 64-bit tested PKGBUILDs et al.

nit-picker 11KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345
  1. #!/bin/sh
  2. # shellcheck disable=SC2119
  3. # shellcheck source=../lib/load-configuration
  4. . "${0%/*}/../lib/load-configuration"
  5. # this script shall host all the tests that are too slow to be run on
  6. # the buildmaster (thus, it runs asynchronously and must therefor be
  7. # resilient against transient states of the buildmaster):
  8. # - check sanity of git
  9. # TODO:
  10. # - check for differences of dependencies between mysql and git
  11. # - check for differences of dependencies between mysql and packages
  12. # - check for installability of packages
  13. # - It would be also nice to have the possibility to fix broken
  14. # dependencies in the database automatically.
  15. if [ $# -ge 1 ] && [ "x$1" = 'x-n' ]; then
  16. >&2 echo 'not joining irc'
  17. irc=false
  18. shift
  19. else
  20. irc=true
  21. # shellcheck disable=SC2016
  22. if [ $# -ne 0 ]; then
  23. >&2 echo 'usage: nit-picker [-n [$single_test]]'
  24. >&2 echo ' -n: do not join irc'
  25. >&2 echo ' $single_test: only execute the given test'
  26. exit 1
  27. fi
  28. fi
  29. clean_up() {
  30. rm -rf --one-file-system "${tmp_dir}"
  31. }
  32. tmp_dir=$(mktemp -d 'tmp.nit-picker.XXXXXXXXXX' --tmpdir)
  33. trap 'clean_up' EXIT
  34. if ${irc}; then
  35. if pgrep -x ii; then
  36. >&2 echo 'ii is already running - this will not work'
  37. exit 1
  38. fi
  39. rm -rf --one-file-system "${irc_dir}"
  40. ii -s irc.freenode.net -n nit-picker -f nit-picker >/dev/null 2>&1 &
  41. ii_pid=$!
  42. clean_up() {
  43. rm -rf --one-file-system "${tmp_dir}"
  44. kill "${ii_pid}"
  45. }
  46. # wait for nickserv complaint
  47. while ! grep -qF 'This nickname is registered. Please choose a different nickname' "${irc_dir}/nickserv/out"; do
  48. sleep 1
  49. done
  50. # register
  51. printf 'identify %s\n' "${irc_password}" | \
  52. sponge "${irc_dir}/nickserv/in"
  53. # wait for registering to succeed
  54. while ! grep -qF 'You are now identified for' "${irc_dir}/nickserv/out"; do
  55. sleep 1
  56. done
  57. # join channel
  58. echo '/j #archlinux32' | \
  59. sponge "${irc_dir}/in"
  60. while [ ! -f "${irc_dir}/#archlinux32/out" ]; do
  61. sleep 1
  62. done
  63. else
  64. irc_say() {
  65. sed 's/^/irc: /'
  66. }
  67. fi
  68. mysql_load_min_and_max_versions
  69. while pgrep -x ii >/dev/null \
  70. || ! ${irc}; do
  71. if [ $# -eq 0 ]; then
  72. # shellcheck disable=SC2016
  73. {
  74. printf 'SELECT DISTINCT'
  75. printf ' "commit",'
  76. printf '`git_repositories`.`name`,'
  77. printf '`git_repositories`.`head`,'
  78. printf '`package_sources`.`git_revision`'
  79. printf ' FROM `package_sources`'
  80. mysql_join_package_sources_upstream_repositories
  81. mysql_join_upstream_repositories_git_repositories
  82. printf ';\n'
  83. printf 'SELECT DISTINCT'
  84. printf ' "commit",'
  85. printf '"archlinux32",'
  86. # shellcheck disable=SC2154
  87. printf '"%s",' \
  88. "${repo_heads__archlinux32}"
  89. printf '`package_sources`.`mod_git_revision`'
  90. printf ' FROM `package_sources`'
  91. printf ';\n'
  92. printf 'SELECT DISTINCT'
  93. printf ' "binary-signature",'
  94. mysql_package_name_query
  95. printf ' FROM `binary_packages`'
  96. mysql_join_binary_packages_architectures
  97. mysql_join_binary_packages_binary_packages_in_repositories
  98. mysql_join_binary_packages_in_repositories_repositories
  99. printf ' WHERE `repositories`.`is_on_master_mirror`'
  100. printf ';\n'
  101. printf 'SELECT DISTINCT'
  102. printf ' "binary-dependencies",'
  103. mysql_package_name_query
  104. printf ' FROM `binary_packages`'
  105. mysql_join_binary_packages_architectures
  106. mysql_join_binary_packages_binary_packages_in_repositories
  107. mysql_join_binary_packages_in_repositories_repositories
  108. printf ' WHERE `repositories`.`is_on_master_mirror`'
  109. printf ';\n'
  110. } | \
  111. mysql_run_query | \
  112. tr '\t' ' ' | \
  113. shuf
  114. else
  115. printf '%s\n' "$*"
  116. fi | \
  117. while read -r action parameters; do
  118. if ${irc} && ! pgrep -x ii >/dev/null; then
  119. break
  120. fi
  121. case "${action}" in
  122. 'commit') # check whether a given commit is present in the git repo
  123. git_repo="${parameters%% *}"
  124. git_rev="${parameters#${git_repo} }"
  125. git_head="${git_rev%% *}"
  126. git_rev="${git_rev#${git_head} }"
  127. # shellcheck disable=SC2016
  128. eval "$(
  129. printf 'git_dir="${repo_paths__%s}"\n' \
  130. "${git_repo}"
  131. )"
  132. # shellcheck disable=SC2154
  133. if ! git -C "${git_dir}" cat-file -t "${git_rev}" 2> /dev/null | \
  134. grep -qxF 'commit'; then
  135. git -C "${git_dir}" fetch --all -p >/dev/null 2>&1
  136. if ! git -C "${git_dir}" cat-file -t "${git_rev}" 2> /dev/null | \
  137. grep -qxF 'commit'; then
  138. printf 'commit %s is missing from repository %s\n' \
  139. "${git_rev}" \
  140. "${git_repo}" \
  141. | irc_say
  142. fi
  143. fi
  144. # shellcheck disable=SC2154
  145. if ! git -C "${git_dir}" cat-file -t "${git_head}" 2> /dev/null | \
  146. grep -qxF 'commit'; then
  147. git -C "${git_dir}" fetch --all -p >/dev/null 2>&1
  148. if ! git -C "${git_dir}" cat-file -t "${git_head}" 2> /dev/null | \
  149. grep -qxF 'commit'; then
  150. printf 'commit %s is missing from repository %s\n' \
  151. "${git_head}" \
  152. "${git_repo}" \
  153. | irc_say
  154. fi
  155. fi
  156. # shellcheck disable=SC2154
  157. if ! git -C "${git_dir}" merge-base --is-ancestor "${git_rev}" "${git_head}" 2> /dev/null; then
  158. current_git_head=$(
  159. # shellcheck disable=SC2016
  160. {
  161. printf 'SELECT DISTINCT'
  162. printf ' `git_repositories`.`head`'
  163. printf ' FROM `git_repositories`'
  164. printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
  165. "$(
  166. printf '%s' "${git_repo}" \
  167. | base64 -w0
  168. )"
  169. } \
  170. | mysql_run_query
  171. )
  172. if ! git -C "${git_dir}" merge-base --is-ancestor "${git_rev}" "${current_git_head}" 2> /dev/null; then
  173. git -C "${git_dir}" fetch --all -p >/dev/null 2>&1
  174. if ! git -C "${git_dir}" merge-base --is-ancestor "${git_rev}" "${current_git_head}" 2> /dev/null; then
  175. printf 'commit %s is not an ancestor of HEAD %s in repository %s\n' \
  176. "${git_rev}" \
  177. "${current_git_head}" \
  178. "${git_repo}" \
  179. | irc_say
  180. fi
  181. fi
  182. fi
  183. ;;
  184. 'binary-dependencies')
  185. if ! ${master_mirror_rsync_command} \
  186. "${master_mirror_rsync_directory}/pool/${parameters}" \
  187. "${tmp_dir}/"; then
  188. rm -f "${tmp_dir}/${parameters}"
  189. continue
  190. fi
  191. extract_dependencies_from_package \
  192. "${tmp_dir}/${parameters}" \
  193. > "${tmp_dir}/pkg-deps"
  194. # shellcheck disable=SC2016
  195. {
  196. printf 'SELECT'
  197. printf ' `dependency_types`.`name`,'
  198. printf '`install_targets`.`name`,'
  199. printf '`dependencies`.`version_relation`,'
  200. printf '`versions`.`epoch`,'
  201. printf '`versions`.`version`'
  202. printf ' FROM `binary_packages`'
  203. mysql_join_binary_packages_dependencies
  204. mysql_join_binary_packages_architectures
  205. mysql_join_dependencies_dependency_types
  206. mysql_join_dependencies_versions
  207. mysql_join_dependencies_install_targets
  208. printf ' WHERE '
  209. mysql_package_name_query
  210. printf '="%s"' \
  211. "${parameters}"
  212. printf ' AND `dependency_types`.`name` IN ("run","make","check")'
  213. printf ';\n'
  214. } \
  215. | mysql_run_query \
  216. | tr '\t' ' ' \
  217. | sort -u \
  218. > "${tmp_dir}/db-deps"
  219. if ! diff -q "${tmp_dir}/db-deps" "${tmp_dir}/pkg-deps"; then
  220. build_date=$(
  221. bsdtar -Oxf "${tmp_dir}/${parameters}" '.PKGINFO' \
  222. | sed '
  223. s/^builddate = //
  224. t
  225. d
  226. '
  227. )
  228. build_date=$(
  229. date -I -d@"${build_date}"
  230. )
  231. printf 'dependencies of %s (built on %s) differ between the package and our database\n' \
  232. "${parameters}" \
  233. "${build_date}" \
  234. | irc_say 'deep42thought'
  235. if ! ${irc}; then
  236. diff -u --color "${tmp_dir}/db-deps" "${tmp_dir}/pkg-deps" || true
  237. fi
  238. if [ $# -eq 0 ]; then
  239. sleep 60
  240. fi
  241. fi
  242. rm \
  243. "${tmp_dir}/${parameters}" \
  244. "${tmp_dir}/db-deps" \
  245. "${tmp_dir}/pkg-deps"
  246. ;;
  247. 'binary-signature')
  248. if ! ${master_mirror_rsync_command} \
  249. "${master_mirror_rsync_directory}/pool/${parameters}" \
  250. "${master_mirror_rsync_directory}/pool/${parameters}.sig" \
  251. "${tmp_dir}/"; then
  252. rm -f "${tmp_dir}/${parameters}" "${tmp_dir}/${parameters}.sig"
  253. continue
  254. fi
  255. unset error_message
  256. if ! gpg_output=$(
  257. gpg --batch --status-fd 1 -q --homedir /etc/pacman.d/gnupg \
  258. --verify "${tmp_dir}/${parameters}.sig" "${tmp_dir}/${parameters}" \
  259. 2>/dev/null
  260. ); then
  261. error_message="package ${parameters} has an invalid signature."
  262. fi
  263. if [ -z "${error_message}" ]; then
  264. gpg_key=$(
  265. printf '%s\n' "${gpg_output}" \
  266. | sed '
  267. s/^\[GNUPG:] KEY_CONSIDERED \([0-9A-F]\{40\}\) 0$/\1/
  268. t
  269. d
  270. ' \
  271. | sort -u
  272. )
  273. if [ -z "${gpg_key}" ]; then
  274. error_message="cannot find pgp_key of package ${parameters}."
  275. fi
  276. fi
  277. if [ -z "${error_message}" ]; then
  278. for expiration in $(
  279. gpg --batch --homedir /etc/pacman.d/gnupg --with-colons --list-keys "0x${gpg_key}" \
  280. 2>/dev/null \
  281. | grep '^\(sub\|pub\):' \
  282. | cut -d: -f7
  283. ); do
  284. expiration_days=$(((expiration - $(date +%s))/24/60/60))
  285. if [ ${expiration_days} -lt 100 ]; then
  286. error_message=$(
  287. printf 'signing key %s (from %s) for package %s expires on %s (in %s < 100 days).\n' \
  288. "${gpg_key}" \
  289. "$(
  290. gpg --batch --homedir /etc/pacman.d/gnupg --with-colons --list-keys "0x${gpg_key}" \
  291. 2>/dev/null \
  292. | grep '^\(uid\):' \
  293. | cut -d: -f10
  294. )" \
  295. "${parameters}" \
  296. "$(date -I -d@"${expiration}")" \
  297. "${expiration_days}"
  298. )
  299. break
  300. fi
  301. done
  302. fi
  303. if [ -n "${error_message}" ]; then
  304. printf '%s\n' "${error_message}" \
  305. | irc_say
  306. if [ $# -eq 0 ]; then
  307. sleep 60
  308. fi
  309. fi
  310. rm \
  311. "${tmp_dir}/${parameters}" \
  312. "${tmp_dir}/${parameters}.sig"
  313. ;;
  314. *)
  315. >&2 printf 'action "%s" is not yet implemented ...\n' "${action}"
  316. ;;
  317. esac
  318. done
  319. if [ $# -ge 1 ]; then
  320. break
  321. fi
  322. sleep 120
  323. done