From ae993074a1d70e169a419320fe203790b319c43b Mon Sep 17 00:00:00 2001 From: Andreas Baumann Date: Thu, 3 Feb 2022 20:31:59 +0100 Subject: update-website: - all ISO data fetching and checking is now in a separate script 'check-iso' - reorganized code to do only one architecture, can be called multiple times for every architecture separately - passing hard-coded locations as options - commented and added a single option to every operation which can be done on the website --- update-website | 321 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 156 insertions(+), 165 deletions(-) diff --git a/update-website b/update-website index bdcbfdd..33dbe61 100755 --- a/update-website +++ b/update-website @@ -1,10 +1,22 @@ #!/bin/bash -set -e +# parameters and default values +ARCH='i686' +DESC='i686 only' +OUTPUT_DIR="${HOME}/archisos" +DATE=$(date +%Y.%m.%d) +ISO="archlinux32-${DATE}-${ARCH}.iso" mirror_dir='/mnt/archlinux32' website_dir='/usr/src/archlinux32/archweb32' +# fail on first error +set -e + +base_dir=$( + readlink -e "${0%/*}" +) + usage() { >&2 echo '' >&2 echo 'update-website: update download, torrent, magnet links on' @@ -24,8 +36,14 @@ usage() { >&2 echo ' Set website-dir to $dir.' >&2 echo ' --mirror-dir $dir:' >&2 echo ' Set mirror-dir to $dir.' - >&2 echo ' --update-iso:' - >&2 echo ' Update information about iso, too.' + >&2 echo ' --update-mirrors:' + >&2 echo ' regenerate mirror list and country flags.' + >&2 echo ' --update-iso update information about ISO, too.' + >&2 echo " --iso name of ISO to be checked, default is '${ISO}'." + >&2 echo " --output-dir where to write the isos, default is '${OUTPUT_DIR}." + >&2 echo " --arch architecture of the ISO to update, default '$ARCH'." + >&2 echo " --desc description of the ISO to update, default '$DESC'." + [ -z "$1" ] && exit 1 || exit "$1" } @@ -38,11 +56,16 @@ eval set -- "$( --long torrent-seed-dual: \ --long mirror-dir: \ --long website-dir: \ + --long update-mirrors \ --long update-iso \ + --long iso: \ + --long arch: \ + --long desc: \ -n "$(basename "$0")" -- "$@" || \ echo usage )" +# cleanup hook tmp_dir="$(mktemp -d)" cleanup() { if mountpoint -q "${tmp_dir}"; then @@ -52,6 +75,10 @@ cleanup() { } trap cleanup EXIT +arch="${ARCH}" +desc="${DESC}" +iso="${ISO}" + while [ $# -gt 0 ]; do case "$1" in '--magnet-i686') @@ -112,6 +139,13 @@ while [ $# -gt 0 ]; do website_dir="$1" echo "$website_dir" > "${tmp_dir}/website_dir.set" ;; + '--update-mirrors') + if [ -s "${tmp_dir}/update_mirrors.set" ]; then + printf 'Option %s given multiple times.\n' "$1" + usage + fi + echo "true" > "${tmp_dir}/update_mirrors.set" + ;; '--update-iso') if [ -s "${tmp_dir}/update_iso.set" ]; then printf 'Option %s given multiple times.\n' "$1" @@ -119,6 +153,18 @@ while [ $# -gt 0 ]; do fi echo "true" > "${tmp_dir}/update_iso.set" ;; + '--iso') + shift + iso="$1" + ;; + '--arch') + shift + arch="$1" + ;; + '--desc') + shift + desc="$1" + ;; '--help'|'-h') usage 0 ;; @@ -139,61 +185,64 @@ if [ $# -gt 0 ]; then exit 2 fi -( - printf '
\n' - curl -Ss 'https://archlinux32.org/mirrorlist/?country=all&hash='"$( - head -c 24 /dev/urandom | \ - base64 -w0 - )" | \ - sed ' - 0,/^$/d - /^$/d - /^## /{ - s|^## |\n \n\n
| - s|$|
\n\n \n\n
| + s|$|
\n\n \n\n' + printf '
\n' + ) | \ + sed -i ' + :emptyLines + /^\n*$/{ + N + bemptyLines } - /^#Server = /{ - s|^#Server = \(.*//\)\([^/]*\)\(/.*\)\$arch/\$repo/\?$|'"$( - printf \ - ' %s\\n' \ - '
  • \2
    • ' - )"'| + /^\(\s*\n\)*\s*
    $/{ + :a + s|^.*
    || + tb + $!N + ba + :b + r /dev/stdin } - ' | \ - sed ' - 0,/<\/ul>/d - ' - printf ' \n\n' - printf ' \n' -) | \ - sed -i ' - :emptyLines - /^\n*$/{ - N - bemptyLines - } - /^\(\s*\n\)*\s*
    $/{ - :a - s|^.*
    || - tb - $!N - ba - :b - r /dev/stdin - } - ' "${website_dir}/download/index.html" + ' "${website_dir}/download/index.html" +fi latest_iso_date="$( ls "${mirror_dir}/archisos" | \ @@ -203,134 +252,76 @@ latest_iso_date="$( tail -n1 )" -declare -A desc -desc['i686']='i686 only' -# desc['dual']='dual bootable' - -for arch in "${!desc[@]}"; do - if [ ! -s "${tmp_dir}/magnet.${arch}" ]; then - continue - fi +# update magnet links +if [ -s "${tmp_dir}/magnet.${arch}" ]; then if ! tr '&' '\n' < \ "${tmp_dir}/magnet.${arch}" | \ - grep -qxF "dn=archlinux32-${latest_iso_date}-${arch}.iso"; then - >&2 printf 'Magnet link for %s is not for %s, skipping.\n' "${arch}" "archlinux32-${latest_iso_date}-${arch}.iso" - continue + grep -qxF "dn=${iso}"; then + sed -i ' + /
  • Magnet link for \S\+ ('"${desc["${arch}"]}"')/ s/>Magnet link for \S\+ />Magnet link for '"${latest_iso_date}"' / + ' "${website_dir}/download/index.html" + else + >&2 printf 'Magnet link for %s is not for %s, skipping.\n' "${arch}" "${iso}" fi - sed -i ' - /
  • Magnet link for \S\+ ('"${desc["${arch}"]}"')/ s/>Magnet link for \S\+ />Magnet link for '"${latest_iso_date}"' / - ' "${website_dir}/download/index.html" -done +fi -for arch in "${!desc[@]}"; do - if [ ! -s "${tmp_dir}/torrent-seed.${arch}" ]; then - continue +# update torrent links +if [ -s "${tmp_dir}/torrent-seed.${arch}" ]; then + if grep -qF "archlinux32-${latest_iso_date}-${arch}" "${tmp_dir}/torrent-seed.${arch}"; then + sed -i ' + /
  • Torrent for \S\+ ('"${desc["${arch}"]}"')/ s/>Torrent for \S\+ />Torrent for '"${latest_iso_date}"' / + ' "${website_dir}/download/index.html" + else + >&2 printf 'Torrent seed for %s is not for %s, skipping.\n' "${arch}" "${iso}" fi - if ! grep -qF "archlinux32-${latest_iso_date}-${arch}" "${tmp_dir}/torrent-seed.${arch}"; then - >&2 printf 'Torrent seed for %s is not for %s, skipping.\n' "${arch}" "archlinux32-${latest_iso_date}-${arch}.iso" - continue - fi - sed -i ' - /
  • Torrent for \S\+ ('"${desc["${arch}"]}"')/ s/>Torrent for \S\+ />Torrent for '"${latest_iso_date}"' / - ' "${website_dir}/download/index.html" -done - -if [ ! -s "${tmp_dir}/update_iso.set" ]; then - exit fi -unset kernels -unset sizes -declare -A sizes -declare -A md5 -declare -A sha512 - -for arch in "${!desc[@]}"; do - iso="${mirror_dir}/archisos/archlinux32-${latest_iso_date}-${arch}.iso" - if [ ! -f "${iso}" ]; then - >&2 echo "Can't find iso to arch ${arch}." +# update ISO links and information +if [ -s "${tmp_dir}/update_iso.set" ]; then + if [ ! -f "${mirror_dir}/archisos/${iso}" ]; then + >&2 echo "Can't find iso to arch '${mirror_dir}/archisos/${iso}'." exit 1 fi - sudo mount -o loop,ro "${iso}" "${tmp_dir}" - kernels="$( - printf '%s\n' ${kernels} $( - find "${tmp_dir}/arch" \ - -maxdepth 1 \ - -name 'pkglist.*.txt' \ - -not -name 'pkglist.x86_64.txt' \ - -execdir cat {} \; \ - | sed ' - s/^linux\s\+\([^-]\+-[^-]\+\)$/\1/ - t - d - ' - ) | \ - sort -u - )" - sudo umount "${tmp_dir}" - sizes["${arch}"]="$( - printf 'scale=1; %s/1024/1024\n' "$(stat -c'%s' "${iso}")" | \ - bc - )" - md5["${arch}"]="$( - grep "\s${iso##*/}\$" "${mirror_dir}/archisos/md5sums" | \ - awk '{print $1}' - )" - sha512["${arch}"]="$( - grep "\s${iso##*/}\$" "${mirror_dir}/archisos/sha512sums" | \ - awk '{print $1}' - )" - sign_keys="$( - printf '%s\n' ${sign_keys} $( - gpg --status-fd=1 --verify "${iso}.sig" "${iso}" 2> /dev/null | \ - grep '^\[GNUPG:] VALIDSIG [0-9A-F]\+ ' | \ - cut -d' ' -f3 - ) | \ - sort -u - )" -done - -if [ $(echo "${kernels}" | grep -c '\S') -ne 1 ]; then - >&2 echo 'Not exactly one kernel on the iso:' - >&2 echo "${kernels}" - exit 1 -fi - -if [ $(echo "${sign_keys}" | grep -c '\S') -ne 1 ]; then - >&2 echo 'Not exactly one key used for signing the iso:' - >&2 echo "'${sign_keys}'" - exit 1 -fi - -if [ "${sign_keys}" != '33CA3597B0D161AAE4173F65C17F1214114574A4' ]; then - >&2 printf 'Isos are signed with key %s instead of %s.\n' \ - "${sign_keys}" \ - '33CA3597B0D161AAE4173F65C17F1214114574A4' -# TODO: not fatal, we don't have the sign key for signing ISOS so we use our package sign key for now -# TODO: should not be hard coded, is a parameter for the script! -# exit 1 + kernel="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --kernel-version)" + size="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --size)" + md5="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --md5sum)" + sha512="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --sha512sum)" + sign_key="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --sign-key)" fi +# adapt release info on the top values=( 'Current Release' "${latest_iso_date}" - 'Included Kernel' "${kernels}" - 'PGP Key' ''"${sign_keys}"'' + 'Included Kernel' "${kernel}" + 'PGP Key' ''"${sign_key}"'' ) -for arch in 'i686' 'dual'; do - values+=( - "ISO Size (${desc["${arch}"]})" "${sizes["${arch}"]} MB" - "MD5 (${desc["${arch}"]})" "${md5["${arch}"]}" - "SHA512 (${desc["${arch}"]})" "${sha512["${arch}"]}" - ) -done +sed -i "$( + printf '/%s:/s|.*
    • | %s|\n' "${values[@]}" +) +" \ + "${website_dir}/download/index.html" +# adapt values for each ISO +values=( + "ISO Size (${desc})" "${size} MB" + "MD5 (${desc})" "${md5}" + "SHA512 (${desc})" "${sha512}" +) sed -i "$( printf '/%s:/s|.*| %s|\n' "${values[@]}" +) +" \ + "${website_dir}/download/index.html" + +# adapt dates in signatures +sed -i "$( printf '/archlinux32-[0-9.]\+-%s\.iso\.sig/s@archlinux32-[0-9.]\+-%s\.iso\.sig@archlinux32-%s-%s.iso.sig@\n' \ - 'i686' 'i686' "${latest_iso_date}" 'i686' \ - 'dual' 'dual' "${latest_iso_date}" 'dual' + "${arch}" "${arch}" "${latest_iso_date}" "${arch}" ) " \ "${website_dir}/download/index.html" + +exit 0 + -- cgit v1.2.3-54-g00ecf