Tools for building 32-bit archlinux packages from archlinux.org's official, 64-bit tested PKGBUILDs et al.

nit-picker 14KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415
  1. #!/bin/sh
  2. # shellcheck disable=SC2119
  3. # shellcheck source=../lib/load-configuration
  4. . "${0%/*}/../lib/load-configuration"
  5. # this script shall host all the tests that are too slow to be run on
  6. # the buildmaster (thus, it runs asynchronously and must therefor be
  7. # resilient against transient states of the buildmaster):
  8. # - check sanity of git
  9. # TODO:
  10. # - check for differences of dependencies between mysql and git
  11. # - check for differences of dependencies between mysql and packages
  12. # - check for installability of packages
  13. # - It would be also nice to have the possibility to fix broken
  14. # dependencies in the database automatically.
  15. if [ $# -ge 1 ] && [ "x$1" = 'x-n' ]; then
  16. >&2 echo 'not joining irc'
  17. irc=false
  18. shift
  19. else
  20. irc=true
  21. # shellcheck disable=SC2016
  22. if [ $# -ne 0 ]; then
  23. >&2 echo 'usage: nit-picker [-n [$single_test]]'
  24. >&2 echo ' -n: do not join irc'
  25. >&2 echo ' $single_test: only execute the given test'
  26. exit 1
  27. fi
  28. fi
  29. clean_up() {
  30. rm -rf --one-file-system "${tmp_dir}"
  31. }
  32. tmp_dir=$(mktemp -d 'tmp.nit-picker.XXXXXXXXXX' --tmpdir)
  33. trap 'clean_up' EXIT
  34. if ${irc}; then
  35. if pgrep -x ii; then
  36. >&2 echo 'ii is already running - this will not work'
  37. exit 1
  38. fi
  39. rm -rf --one-file-system "${irc_dir}"
  40. ii -s irc.freenode.net -n nit-picker -f nit-picker >/dev/null 2>&1 &
  41. ii_pid=$!
  42. clean_up() {
  43. rm -rf --one-file-system "${tmp_dir}"
  44. kill "${ii_pid}"
  45. }
  46. # wait for nickserv complaint
  47. while ! grep -qF 'This nickname is registered. Please choose a different nickname' "${irc_dir}/nickserv/out"; do
  48. sleep 1
  49. done
  50. # register
  51. printf 'identify %s\n' "${irc_password}" | \
  52. sponge "${irc_dir}/nickserv/in"
  53. # wait for registering to succeed
  54. while ! grep -qF 'You are now identified for' "${irc_dir}/nickserv/out"; do
  55. sleep 1
  56. done
  57. # join channel
  58. echo '/j #archlinux32' | \
  59. sponge "${irc_dir}/in"
  60. while [ ! -f "${irc_dir}/#archlinux32/out" ]; do
  61. sleep 1
  62. done
  63. else
  64. irc_say() {
  65. sed 's/^/irc: /'
  66. }
  67. fi
  68. mysql_load_min_and_max_versions
  69. build_duration_check=0
  70. while pgrep -x ii >/dev/null \
  71. || ! ${irc}; do
  72. if [ $# -eq 0 ]; then
  73. if [ "$(date +%s)" -gt "$((build_duration_check + 60*60*24))" ]; then
  74. check_build_duration=true
  75. build_duration_check=$(date +%s)
  76. else
  77. check_build_duration=false
  78. fi
  79. # shellcheck disable=SC2016
  80. {
  81. printf 'SELECT DISTINCT'
  82. printf ' "commit",'
  83. printf '`git_repositories`.`name`,'
  84. printf '`git_repositories`.`head`,'
  85. printf '`package_sources`.`git_revision`'
  86. printf ' FROM `package_sources`'
  87. mysql_join_package_sources_upstream_repositories
  88. mysql_join_upstream_repositories_git_repositories
  89. printf ';\n'
  90. printf 'SELECT DISTINCT'
  91. printf ' "commit",'
  92. printf '"archlinux32",'
  93. # shellcheck disable=SC2154
  94. printf '"%s",' \
  95. "${repo_heads__archlinux32}"
  96. printf '`package_sources`.`mod_git_revision`'
  97. printf ' FROM `package_sources`'
  98. printf ';\n'
  99. printf 'SELECT DISTINCT'
  100. printf ' "binary-signature",'
  101. mysql_package_name_query
  102. printf ' FROM `binary_packages`'
  103. mysql_join_binary_packages_architectures
  104. mysql_join_binary_packages_binary_packages_in_repositories
  105. mysql_join_binary_packages_in_repositories_repositories
  106. printf ' WHERE `repositories`.`is_on_master_mirror`'
  107. printf ';\n'
  108. if "${check_build_duration}"; then
  109. printf 'SELECT'
  110. printf ' "build-duration",'
  111. printf '`build_slaves`.`name`'
  112. printf ' FROM `build_slaves`'
  113. printf ';\n'
  114. fi
  115. printf 'SELECT DISTINCT'
  116. printf ' "binary-dependencies",'
  117. mysql_package_name_query
  118. printf ' FROM `binary_packages`'
  119. mysql_join_binary_packages_architectures
  120. mysql_join_binary_packages_binary_packages_in_repositories
  121. mysql_join_binary_packages_in_repositories_repositories
  122. printf ' WHERE `repositories`.`is_on_master_mirror`'
  123. printf ';\n'
  124. } | \
  125. mysql_run_query | \
  126. tr '\t' ' ' | \
  127. shuf
  128. else
  129. printf '%s\n' "$*"
  130. fi | \
  131. while read -r action parameters; do
  132. if ${irc} && ! pgrep -x ii >/dev/null; then
  133. break
  134. fi
  135. case "${action}" in
  136. 'commit') # check whether a given commit is present in the git repo
  137. git_repo="${parameters%% *}"
  138. git_rev="${parameters#${git_repo} }"
  139. git_head="${git_rev%% *}"
  140. git_rev="${git_rev#${git_head} }"
  141. # shellcheck disable=SC2016
  142. eval "$(
  143. printf 'git_dir="${repo_paths__%s}"\n' \
  144. "${git_repo}"
  145. )"
  146. # shellcheck disable=SC2154
  147. if ! git -C "${git_dir}" cat-file -t "${git_rev}" 2> /dev/null | \
  148. grep -qxF 'commit'; then
  149. git -C "${git_dir}" fetch --all -p >/dev/null 2>&1
  150. if ! git -C "${git_dir}" cat-file -t "${git_rev}" 2> /dev/null | \
  151. grep -qxF 'commit'; then
  152. printf 'commit %s is missing from repository %s\n' \
  153. "${git_rev}" \
  154. "${git_repo}" \
  155. | irc_say
  156. fi
  157. fi
  158. # shellcheck disable=SC2154
  159. if ! git -C "${git_dir}" cat-file -t "${git_head}" 2> /dev/null | \
  160. grep -qxF 'commit'; then
  161. git -C "${git_dir}" fetch --all -p >/dev/null 2>&1
  162. if ! git -C "${git_dir}" cat-file -t "${git_head}" 2> /dev/null | \
  163. grep -qxF 'commit'; then
  164. printf 'commit %s is missing from repository %s\n' \
  165. "${git_head}" \
  166. "${git_repo}" \
  167. | irc_say
  168. fi
  169. fi
  170. # shellcheck disable=SC2154
  171. if ! git -C "${git_dir}" merge-base --is-ancestor "${git_rev}" "${git_head}" 2> /dev/null; then
  172. current_git_head=$(
  173. # shellcheck disable=SC2016
  174. {
  175. printf 'SELECT DISTINCT'
  176. printf ' `git_repositories`.`head`'
  177. printf ' FROM `git_repositories`'
  178. printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
  179. "$(
  180. printf '%s' "${git_repo}" \
  181. | base64 -w0
  182. )"
  183. } \
  184. | mysql_run_query
  185. )
  186. if ! git -C "${git_dir}" merge-base --is-ancestor "${git_rev}" "${current_git_head}" 2> /dev/null; then
  187. git -C "${git_dir}" fetch --all -p >/dev/null 2>&1
  188. if ! git -C "${git_dir}" merge-base --is-ancestor "${git_rev}" "${current_git_head}" 2> /dev/null; then
  189. printf 'commit %s is not an ancestor of HEAD %s in repository %s\n' \
  190. "${git_rev}" \
  191. "${current_git_head}" \
  192. "${git_repo}" \
  193. | irc_say
  194. fi
  195. fi
  196. fi
  197. ;;
  198. 'binary-dependencies')
  199. if ! ${master_mirror_rsync_command} \
  200. "${master_mirror_rsync_directory}/pool/${parameters}" \
  201. "${tmp_dir}/"; then
  202. rm -f "${tmp_dir}/${parameters}"
  203. continue
  204. fi
  205. extract_dependencies_from_package \
  206. "${tmp_dir}/${parameters}" \
  207. > "${tmp_dir}/pkg-deps"
  208. # shellcheck disable=SC2016
  209. {
  210. printf 'SELECT'
  211. printf ' `dependency_types`.`name`,'
  212. printf '`install_targets`.`name`,'
  213. printf '`dependencies`.`version_relation`,'
  214. printf '`versions`.`epoch`,'
  215. printf '`versions`.`version`'
  216. printf ' FROM `binary_packages`'
  217. mysql_join_binary_packages_dependencies
  218. mysql_join_binary_packages_architectures
  219. mysql_join_dependencies_dependency_types
  220. mysql_join_dependencies_versions
  221. mysql_join_dependencies_install_targets
  222. printf ' WHERE '
  223. mysql_package_name_query
  224. printf '="%s"' \
  225. "${parameters}"
  226. printf ' AND `dependency_types`.`name` IN ("run","make","check")'
  227. printf ';\n'
  228. } \
  229. | mysql_run_query \
  230. | tr '\t' ' ' \
  231. | sort -u \
  232. > "${tmp_dir}/db-deps"
  233. if ! diff -q "${tmp_dir}/db-deps" "${tmp_dir}/pkg-deps"; then
  234. build_date=$(
  235. bsdtar -Oxf "${tmp_dir}/${parameters}" '.PKGINFO' \
  236. | sed '
  237. s/^builddate = //
  238. t
  239. d
  240. '
  241. )
  242. build_date=$(
  243. date -I -d@"${build_date}"
  244. )
  245. printf 'dependencies of %s (built on %s) differ between the package and our database\n' \
  246. "${parameters}" \
  247. "${build_date}" \
  248. | irc_say 'deep42thought'
  249. if ! ${irc}; then
  250. diff -u --color "${tmp_dir}/db-deps" "${tmp_dir}/pkg-deps" || true
  251. fi
  252. if [ $# -eq 0 ]; then
  253. sleep 60
  254. fi
  255. fi
  256. rm \
  257. "${tmp_dir}/${parameters}" \
  258. "${tmp_dir}/db-deps" \
  259. "${tmp_dir}/pkg-deps"
  260. ;;
  261. 'binary-signature')
  262. if ! ${master_mirror_rsync_command} \
  263. "${master_mirror_rsync_directory}/pool/${parameters}" \
  264. "${master_mirror_rsync_directory}/pool/${parameters}.sig" \
  265. "${tmp_dir}/"; then
  266. rm -f "${tmp_dir}/${parameters}" "${tmp_dir}/${parameters}.sig"
  267. continue
  268. fi
  269. unset error_message
  270. if ! gpg_output=$(
  271. gpg --batch --status-fd 1 -q --homedir /etc/pacman.d/gnupg \
  272. --verify "${tmp_dir}/${parameters}.sig" "${tmp_dir}/${parameters}" \
  273. 2>/dev/null
  274. ); then
  275. error_message="package ${parameters} has an invalid signature."
  276. fi
  277. if [ -z "${error_message}" ]; then
  278. gpg_key=$(
  279. printf '%s\n' "${gpg_output}" \
  280. | sed '
  281. s/^\[GNUPG:] KEY_CONSIDERED \([0-9A-F]\{40\}\) 0$/\1/
  282. t
  283. d
  284. ' \
  285. | sort -u
  286. )
  287. if [ -z "${gpg_key}" ]; then
  288. error_message="cannot find pgp_key of package ${parameters}."
  289. fi
  290. fi
  291. if [ -z "${error_message}" ]; then
  292. for expiration in $(
  293. gpg --batch --homedir /etc/pacman.d/gnupg --with-colons --list-keys "0x${gpg_key}" \
  294. 2>/dev/null \
  295. | grep '^\(sub\|pub\):' \
  296. | cut -d: -f7
  297. ); do
  298. expiration_days=$(((expiration - $(date +%s))/24/60/60))
  299. if [ ${expiration_days} -lt 100 ]; then
  300. error_message=$(
  301. printf 'signing key %s (from %s) for package %s expires on %s (in %s < 100 days).\n' \
  302. "${gpg_key}" \
  303. "$(
  304. gpg --batch --homedir /etc/pacman.d/gnupg --with-colons --list-keys "0x${gpg_key}" \
  305. 2>/dev/null \
  306. | grep '^\(uid\):' \
  307. | cut -d: -f10
  308. )" \
  309. "${parameters}" \
  310. "$(date -I -d@"${expiration}")" \
  311. "${expiration_days}"
  312. )
  313. break
  314. fi
  315. done
  316. fi
  317. if [ -n "${error_message}" ]; then
  318. printf '%s\n' "${error_message}" \
  319. | irc_say
  320. if [ $# -eq 0 ]; then
  321. sleep 60
  322. fi
  323. fi
  324. rm \
  325. "${tmp_dir}/${parameters}" \
  326. "${tmp_dir}/${parameters}.sig"
  327. ;;
  328. 'build-duration')
  329. infos=$(
  330. # shellcheck disable=SC2016
  331. {
  332. printf 'SELECT'
  333. printf '`persons`.`name`,'
  334. printf '`build_slaves`.`name`,'
  335. printf '`architectures`.`name`,'
  336. printf '`package_sources`.`pkgbase`,'
  337. printf 'UNIX_TIMESTAMP(NOW())-UNIX_TIMESTAMP('
  338. printf 'MAX('
  339. printf 'IF('
  340. printf '`ssh_log`.`action`="get-assignment",'
  341. printf '`ssh_log`.`date`,'
  342. printf 'NULL'
  343. printf ')'
  344. printf ')'
  345. printf ')'
  346. printf ' FROM `build_slaves`'
  347. mysql_join_build_slaves_build_assignments
  348. mysql_join_build_assignments_architectures
  349. mysql_join_build_assignments_package_sources
  350. mysql_join_build_slaves_ssh_keys
  351. mysql_join_ssh_keys_persons
  352. mysql_join_build_slaves_ssh_log
  353. printf ' WHERE `ssh_log`.`date`>=ADDDATE(NOW(),"-7 00:00:00")'
  354. printf ' AND `build_slaves`.`name`=from_base64("%s")' \
  355. "$(
  356. printf '%s' "${parameters}" \
  357. | base64 -w0
  358. )"
  359. printf ' GROUP BY `build_slaves`.`id`'
  360. printf ';\n'
  361. } \
  362. | mysql_run_query \
  363. | tr '\t' ' '
  364. )
  365. if [ -z "${infos}" ]; then
  366. continue
  367. fi
  368. if [ "${infos##* }" -lt $((60*60*24)) ]; then
  369. continue
  370. fi
  371. printf '%s %s\n' \
  372. "${infos% *}" \
  373. "$(
  374. date -d"@${infos##* }" +'%-j %H:%M:%S'
  375. )" \
  376. | awk '{print $1 ": your slave " $2 " builds " $3 "/" $4 " for more than a day, now (" ($5-1) " day(s) " $6 ")"}' \
  377. | irc_say
  378. if [ $# -eq 0 ]; then
  379. sleep 60
  380. fi
  381. ;;
  382. *)
  383. >&2 printf 'action "%s" is not yet implemented ...\n' "${action}"
  384. ;;
  385. esac
  386. done
  387. if [ $# -ge 1 ]; then
  388. break
  389. fi
  390. sleep 120
  391. done