much more cleanup

3 files changed, 151 insertions, 96 deletions

diff --git a/build-all b/build-all
index bfa82aa..b0a19d6 100755
--- a/build-all
+++ b/build-all
@@ -21,14 +21,16 @@ ARCH="i686"
 DESC="i686 only"
 #DESC="dual bootable"
 DATE=$(date +%Y.%m.%d)
+#DATE="2022.02.01"
 ISO="archlinux32-${DATE}-${ARCH}.iso"
 SIGNATURE="${ISO}.sig"
 TORRENT="${ISO}.torrent"
-TORRENT_SERVER="localhost"
-#TORRENT_SERVER="archlinux32.org"
+TORRENT_SERVER="archlinux32.org"
+TORRENT_USER="hefur"
 WEBSITE_DIR="/srv/http/archlinux32.org"
 MIRROR_DIR="/srv/http/mirror/mirror.archlinux32.org"
 ARCHISOS_DIR="${MIRROR_DIR}/archisos/"
+ARCHIVE_DIR="/srv/http/mirror/archive.archlinux32.org/iso"
 MIRROR_USER='mirror'
 MIRROR_GROUP='mirror'
 #Andreas Baumann (sign) <mail@andreasbaumann.cc>
@@ -50,6 +52,16 @@ popd() {
 
 export pushd popd
 
+usage() {
+  >&2 echo ""
+  >&2 echo "build-all: do all release engineering for Archlinux32 ISOs and downloads"
+  >&2 echo ""
+  >&2 echo "possible options:"
+  >&2 echo "  -h|--help:       show this help and exit."
+  >&2 echo "  -f|--force:      overwrite already existing artifacts (use with care!)."  
+  [ -z "$1" ] && exit 1 || exit "$1"
+}
+
 # fail on first error
 set -e
 
@@ -63,114 +75,162 @@ cleanup() {
 }
 trap cleanup EXIT
 
+eval set -- "$(
+  getopt -o hf \
+  --long force \
+  --long help \
+  -n "$(basename "$0")" -- "$@" || \
+  echo usage
+)"
+
+force=0
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    '--force'|'-f')
+      force=1
+    ;;
+    '--help'|'-h')
+      usage 0
+    ;;
+    '--')
+      shift
+      break
+    ;;
+    *)
+      >&2 printf 'Whoops, option "%s" is not yet implemented!\n' "$1"
+      exit 42
+    ;;
+  esac
+  shift
+done
+
+if [ $# -gt 0 ]; then
+  >&2 echo 'Too many arguments.'
+  exit 2
+fi
+
 # build the ISO
-echo "Building ISO.."
-"${base_dir}/build-iso" --config="${CONFIG}" --arch "${ARCH}" \
-  --output-dir="${ARCHISOS_DIR}" --iso "${ISO}"
+pushd "${ARCHISOS_DIR}"
+if [ $force = 1 ]; then
+  rm -f "${ISO}"
+fi
+if [ ! -f "${ISO}" ]; then
+  echo "Building ISO.."
+  "${base_dir}/build-iso" --config="${CONFIG}" --arch "${ARCH}" \
+    --output-dir="${ARCHISOS_DIR}" --iso "${ISO}"
+else
+  echo "Refusing to overwrite existing ISO '${ISO}'."
+fi
+popd
+
+if [ ! -f "${ARCHISOS_DIR}/${ISO}" ]; then
+  >&2 echo "The ISO file '${ISO}' does not exist, it makes no sense to continue."
+  exit 1
+fi
 
 # sign ISO
-echo "Signing ISO.."
 pushd "${ARCHISOS_DIR}"
-rm -rf "${ISO}.sig"
-gpg --local-user "${SIGN_KEY}" --batch --no-tty --detach-sign "${ISO}" </dev/null
+if [ $force = 1 ]; then
+  rm -f "${ISO}.sig"
+fi
+if [ ! -f "${ISO}.sig" ]; then
+  echo "Signing ISO.."
+  gpg --local-user "${SIGN_KEY}" --batch --no-tty --detach-sign "${ISO}" </dev/null
+else
+  echo "ISO '${ISO}' is already signed."
+fi
 popd
 
-# checksum the ISO and the signature
+# checksum the ISO
 echo "Updating checksum files.."
 pushd "${ARCHISOS_DIR}"
-[ ! -f sha512sums ] || sed -i "/${ISO}$/d" sha512sums
-sha512sum "${ISO}" >> sha512sums
-[ ! -f md5sums ] || sed -i "/${ISO}$/d" md5sums
-md5sum "${ISO}" >> md5sums
-sort -k2,2 sha512sums --output sha512sums
-sort -k2,2 md5sums --output md5sums
+if [ $(grep "${ISO}$" sha512sums | wc -l) = 0 ]; then
+  [ ! -f sha512sums ] || sed -i "/${ISO}$/d" sha512sums
+  sha512sum "${ISO}" >> sha512sums
+  sort -k2,2 sha512sums --output sha512sums
+else
+  echo "SHA512 checksum already done."
+fi
+if [ $(grep "${ISO}$" md5sums | wc -l) = 0 ]; then
+  [ ! -f md5sums ] || sed -i "/${ISO}$/d" md5sums
+  md5sum "${ISO}" >> md5sums
+  sort -k2,2 md5sums --output md5sums
+else
+  echo "MD5 checksum already done."
+fi
 popd
 
 # check sanity of ISO
 echo "Checking ISO.."
-"${base_dir}/check-iso" --output-dir "${ARCHISOS_DIR}" --check --expected-sign-key "${SIGN_KEY}"
+"${base_dir}/check-iso" \
+  --output-dir "${ARCHISOS_DIR}" --iso="${ISO}" --check --expected-sign-key "${SIGN_KEY}"
 if [ $? = 1 ]; then
-  >&2 echo "The ISO file '${iso}' is not sane. Not continuing!"
-  #~ rm -f "${iso}"
+  >&2 echo "The ISO file '${iso}' is not sane. Not continuing! You should maybe delete this ISO file.."
   exit 1
 fi
 
-# date used for torrents and commit message
-date=$(echo "${ISO}" \
-  | sed 's/^.*-\([^-]\+\)-[^-]\+$/\1/' \
-  | sort -u
-)
-
-# generate torrents (currently broken)
+# generate torrents
 echo "Generating torrents.."
-#~ bash -x "${base_dir}/al32-mktorrent.sh" -d "${date}" -t "hefur@${TORRENT_SERVER}:" "${ARCH}"
+pushd "${ARCHISOS_DIR}"
+if [ $force = 1 ]; then
+  rm -f "${ISO}.torrent"
+fi
+if [ ! -f "${ISO}.torrent" ]; then
+  "${base_dir}/build-torrent" \
+    --arch="${ARCH}" \
+    --date="${DATE}" \
+    --mirror-dir="${MIRROR_DIR}" \
+    --torrent-server="${TORRENT_SERVER}" \
+    --torrent-user="${TORRENT_USER}"
+else
+  echo "The torrent file '${ISO}.torrent' already exists, not generating a new one."
+fi
 
-# update web page
+# get the magnet link and add it to the RSS feed file
+echo "Generating magnet link and publish it in the RSS feed file.."
+magnet_link="$("${base_dir}"/build-torrent --arch="${ARCH}" --date="${DATE}" --mirror-dir="${MIRROR_DIR}" --magnet-link)"
+if [ $force = 1 ]; then
+  rm -f "feed_${ARCH}.rss"
+fi
+if [ ! -f "feed_${ARCH}.rss" ]; then
+  feed_url="https://pool.mirror.archlinux32.org/archisos/feed_${ARCH}.rss"
+  "${base_dir}/magnet2feed.py" \
+    "${magnet_link}" "${ARCH}" "${DATE}" "${feed_url}"
+else
+  echo "The RSS feed file 'feed_${ARCH}.rss' already exists, not generating a new one."
+fi
+
+# update web page for isos and download mirrors
 echo "Updating website.."
-git -C "${WEBSITE_DIR}" reset --hard
-git -C "${WEBSITE_DIR}" pull --ff-only
+#~ git -C "${WEBSITE_DIR}" reset --hard
+#~ git -C "${WEBSITE_DIR}" pull --ff-only
 "${base_dir}/update-website" \
   --website-dir="${WEBSITE_DIR}" \
   --mirror-dir="${MIRROR_DIR}" \
   --iso="${ISO}" \
   --update-iso \
-  --update-mirrors
-
-#~ git -C "${WEBSITE_DIR}" commit 'download/index.html' -m 'download/index.html: new isos ('"${date}"')'
+  --update-mirrors \
+  --torrent-seed="https://pool.mirror.archlinux32.org/archisos/archlinux32-${DATE}-${ARCH}.iso.torrent" \
+  --magnet-link="${magnet_link}"
+#~ git -C "${WEBSITE_DIR}" commit 'download/index.html' -m 'download/index.html: new isos ('"${DATE}"')'
 #~ git -C "${WEBSITE_DIR}" push
 
 # when did we last updates the 
 date +%s > "${ARCHISOS_DIR}/lastupdate"
 
+# cleanup mirror and move old ISOs to the archive
+echo "Cleaning up old ISOs and pushing them into the archive"
+#~ "${base_dir}/cleanup-isos" \
+  #~ --mirror-dir="/srv/http/mirror/mirror.archlinux32.org" \
+  #~ --archive-dir="/srv/http/mirror/archive.archlinux32.org/iso"
+
 # set permissions correctly of all generated or changed fles
 echo "Fixing/setting permissions.."
-#chown "${MIRROR_USER}:${MIRROR_GROUP}" \
-#  "${ARCHISOS_DIR}/${ISO}" "${ARCHISOS_DIR}/${SIGNATURE}" "${ARCHISOS_DIR}/${TORRENT}" \
-#  "${ARCHISOS_DIR}/sha512sums" "${ARCHISOS_DIR}/md5sums" "${ARCHISOS_DIR}/lastupdate"
+chown "${MIRROR_USER}:${MIRROR_GROUP}" \
+  "${ARCHISOS_DIR}/${ISO}" "${ARCHISOS_DIR}/${SIGNATURE}" "${ARCHISOS_DIR}/${TORRENT}" \
+  "${ARCHISOS_DIR}/sha512sums" "${ARCHISOS_DIR}/md5sums" "${ARCHISOS_DIR}/lastupdate"
 
 echo "Finished."
 
 exit 0
-
-#--
-# TODO from here:
-
-archive='/mnt/archlinux32archive'
-destination='/mnt/archlinux32/archisos'
-
-    mv $(
-      printf '%s.torrent\n' ${isos}
-    ) feed_dual.rss feed_i686.rss "${destination}/"
-
-# torrent generation
-
-# ATM no torrents
-# ATM no dual
-#      --torrent-seed-dual "https://pool.mirror.archlinux32.org/archisos/archlinux32-${date}-dual.iso.torrent" \
-#       --torrent-seed-i686 "https://pool.mirror.archlinux32.org/archisos/archlinux32-${date}-i686.iso.torrent" \
-#
-
-# this seems to be cleanup of old isos (which must be moved to the archive
-# should be in a separate script too like cleanup-isos, can then be called in build-all
-
-    find "${destination}" \( -name 'archlinux32-*' -o -name 'archlinux-*' \) -not -name 'archlinux32-'"${date}"'-*' \
-    | while read -r to_delete; do
-      if diff -q "${to_delete}" "${archive}/iso/${to_delete#${destination}/}" >/dev/null; then
-        rm "${to_delete}"
-        printf '%s\n' "${to_delete}" \
-        | sed '
-          s@^.*/@@
-          s/\./\\./g
-          s@.*@/  \0$/d@
-        '
-      fi
-    done \
-    >> "${tmp_dir}/delete-regex"
-    sed -i -f "${tmp_dir}/delete-regex" "${destination}/sha512sums"
-    sed -i -f "${tmp_dir}/delete-regex" "${destination}/md5sums"
-
-    echo '... done.'
-    ;;
-
-
-
diff --git a/build-torrent b/build-torrent
index b3a842d..0c11e06 100755
--- a/build-torrent
+++ b/build-torrent
@@ -141,6 +141,7 @@ mktorrent --announce=http://${torrent_server}:6969/announce \
   --output="${archiso_dir}/${torrent_file}" \
   "${archiso_dir}/${iso}"
 
-scp "${archiso_dir}/${torrent_file}" "${torrent_user}@${torrent_server}:${hefur_dir}"
+# I have to go over port 9586, copying manually ATM
+#scp -P "${torrent_ssh_port}" "${archiso_dir}/${torrent_file}" "${torrent_user}@${torrent_server}:${hefur_dir}"
 
 exit 0
diff --git a/update-website b/update-website
index c653820..2d735a1 100755
--- a/update-website
+++ b/update-website
@@ -220,24 +220,18 @@ date=$(echo "${iso}" \
 
 # update magnet links
 if [ -s "${tmp_dir}/magnet.link.set" ]; then
-  if ! tr '&' '\n' < \
-    "${tmp_dir}/magnet.link.set" | \
-    grep -qxF "dn=${iso}"; then
-    sed -i '
-      /<li>(<strike>)?<a href="magnet:?.*-'"${arch}"'\.iso/ s/href="[^"]\+"/href="'"$(sed 's|&|\\&|g' "${tmp_dir}/magnet.link.set")"'"/
-      /title="Magnet link">Magnet link for \S\+ ('"${desc}"')/ s/>Magnet link for \S\+ />Magnet link for '"${date}"' /
-    ' "${website_dir}/download/index.html"
-  else     
-    >&2 printf 'Magnet link for %s is not for %s, skipping.\n' "${arch}" "${iso}"
-  fi
+  sed -i '
+    /<li><a href="magnet:?.*-'"${arch}"'\.iso/ s/href="[^"]\+"/href="'"$(sed 's|&|\\&|g' "${tmp_dir}/magnet.link.set")"'"/
+    /title="Magnet link">Magnet link for \S\+ ('"${desc}"')/ s/>Magnet link for \S\+ />Magnet link for '"${date}"' /
+  ' "${website_dir}/download/index.html"
 fi
 
 # update torrent links
 if [ -s "${tmp_dir}/torrent-seed.set" ]; then
   if grep -qF "archlinux32-${date}-${arch}" "${tmp_dir}/torrent-seed.set"; then
     sed -i '
-    /<li>(<strike>)?<a href="http.*-'"${arch}"'\.iso.torrent/ s/href="[^"]\+"/href="'"$(sed -e 's_/_\\/_g' "${tmp_dir}/torrent-seed.set")"'"/
-      /title="Download torrent">Torrent for \S\+ ('"${desc["${arch}"]}"')/ s/>Torrent for \S\+ />Torrent for '"${date}"' /
+    /<li><a href="http.*-'"${arch}"'\.iso.torrent/ s|href="[^"]\+"|href="'"$(cat "${tmp_dir}/torrent-seed.set")"'"|
+      /title="Download torrent">Torrent for \S\+ ('"${desc}"')/ s/>Torrent for \S\+ />Torrent for '"${date}"' /
     ' "${website_dir}/download/index.html"
   else
     >&2 printf 'Torrent seed for %s is not for %s, skipping.\n' "${arch}" "${iso}"
@@ -250,11 +244,11 @@ if [ -s "${tmp_dir}/update_iso.set" ]; then
     >&2 echo "Can't find iso to arch '${mirror_dir}/archisos/${iso}'."
     exit 1
   fi
-  kernel="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --kernel-version)"
-  size="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --size)"
-  md5="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --md5sum)"
-  sha512="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --sha512sum)"
-  sign_key="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --sign-key)"
+  kernel="$("${base_dir}"/check-iso --iso="${iso}" --output-dir "${mirror_dir}/archisos" --kernel-version)"
+  size="$("${base_dir}"/check-iso --iso="${iso}" --output-dir "${mirror_dir}/archisos" --size)"
+  md5="$("${base_dir}"/check-iso --iso="${iso}" --output-dir "${mirror_dir}/archisos" --md5sum)"
+  sha512="$("${base_dir}"/check-iso --iso="${iso}" --output-dir "${mirror_dir}/archisos" --sha512sum)"
+  sign_key="$("${base_dir}"/check-iso --iso="${iso}" --output-dir "${mirror_dir}/archisos" --sign-key)"
 fi
 
 # adapt release info on the top