get-package-updates should work again

1 files changed, 165 insertions, 214 deletions

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 9835bea..f83aa4c 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -182,6 +182,25 @@ delete_package() {
     mysql_run_query
 }
 
+# check_if_package_should_be_deleted $pkgbase $repository
+check_if_package_should_be_deleted() {
+  if git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- "${repository}/${pkgbase}" >/dev/null 2>&1; then
+    return 1
+  fi
+  if git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" \
+    | tar -t \
+    | tr '/' ' ' \
+    | cut -d' ' -f1,2 \
+    | sort -k2,2 \
+    | join -1 2 -2 1 - <(echo "${pkgbase}") \
+    | cut -d' ' -f1 \
+    | sed 's@-[^-]\+$@@' \
+    | grep -qxF "${repository}"; then
+    return 1
+  fi
+  return 0
+}
+
 something_new=false
 
 for repo in ${repo_names}; do
@@ -277,229 +296,161 @@ echo 'Check modified packages from the last update, and put them to the build li
 # If a package is deleted, remove from the rebuild list, and add it to the deletion list.
 # If a new package is added, then ensure that it's not on the deletion list.
 
-# shellcheck disable=SC2016
 {
-  printf 'SELECT DISTINCT'
-  printf ' `package_sources`.`pkgbase`,'
-  printf '`upstream_repositories`.`name`,'
-  printf 'IF(`build_assignments`.`architecture`=%s,"any","x86_64")' \
-    "${architecture_ids__any}"
-  printf ' FROM `binary_packages`'
-  mysql_join_binary_packages_binary_packages_in_repositories
-  mysql_join_binary_packages_build_assignments
-  mysql_join_build_assignments_package_sources
-  mysql_join_package_sources_upstream_repositories
-  printf ' WHERE `binary_packages_in_repositories`.`repository`=%s;\n' \
-    "${repository_ids__any_build_list}"
-} | \
-  mysql_run_query | \
-  sed '
-    s@^\(\S*\)\t\(\S*\)\t\(\S*\)$@s,^\\(.\\t\1/repos/\2-\3/\\)[^/]\\+$,\\1PKGBUILD,\ns,^\\(.\\t\2/\1/\\)[^/]\\+$,\\1PKGBUILD,@
-  ' > \
-  "${tmp_dir}/trigger-rebuild-on-any-file-sed-expression"
-
-{
-  # trigger rebuild of packages removed from blacklist
-  # shellcheck disable=SC2016
-  {
-    # TODO: this needs to be properly fixed
-    printf 'SELECT `package_sources`.`pkgbase`,`upstream_repositories`.`name`'
-    printf ' FROM `package_sources`'
-    mysql_join_package_sources_upstream_repositories
-    printf ' WHERE `package_sources`.`pkgbase` IN ('
-    # shellcheck disable=SC2154
-    git -C "${repo_paths__archlinux32}" diff --name-status "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
-      sed '
-        s@^D\tblacklist/[^/]\+/[^/]\+/@@
-        t
-        d
-      ' | \
-      base64_encode_each | \
-      sed '
-        s/^/from_base64("/
-        s/$/")/
-      ' | \
-      tr '\n' ','
-    printf '"");\n'
-  } | \
-    echo mysql_run_query | \
-    while read -r pkgbase repository; do
-      break
-      eval 'git_rev="${new_repo_revisions__'"${git_repository}"'}"'
-      eval 'git_dir="${repo_paths__'"${git_repository}"'}"'
-      # TODO: maybe, we want to exclude our own packages here, too? But why
-      # would they be black listed in the first place?
-      # shellcheck disable=SC2154
-      if git -C "${git_dir}" archive "${git_rev}" -- "${pkgbase}/repos" \
-      | tar -tf - \
-      | grep -q -- '-\(any\|x86_64\)/PKGBUILD$'; then
-        printf 'A %s %s %s\n' \
-          "${pkgbase}" \
-          "${git_rev}" \
-          "${repository}"
-      fi
-    done
-  # actual updates/removes
-  for repo in ${repo_names}; do
-    eval repo_path='"${repo_paths__'"${repo}"'}"'
-    eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"'
-    eval new_repo_revision='"${new_repo_revisions__'"${repo}"'}"'
-    # if old revision unknown, mimic "git diff"-output
-    # shellcheck disable=SC2154
-    if [ "${old_repo_revision}" = "NONE" ]; then
-      git -C "${repo_path}" archive --format=tar HEAD | \
-        tar -t | \
-        sed 's|^|A\t|'
-    else
-      git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" "${new_repo_revision}"
-    fi | \
-      # rename any file to "PKGBUILD" for packages on the build-list
-      sed -f "${tmp_dir}/trigger-rebuild-on-any-file-sed-expression" | \
-      # only track changes in PKGBUILDs
-      grep '/PKGBUILD$' | \
-      if [ "${repo}" = "archlinux32" ]; then
-      # modify the directory structure from the modifiaction-repository
-      # to the one of an original source repository
-        # shellcheck disable=SC2016
-        sed 's|^\(.\t\)\([^/]\+\)/\([^/]\+\)/\(.\+\)$|\2 \1\3/repos/\2-x86_64/\4|' | \
-          while read -r pkg_repo rest; do
-            repo=$(find_git_repository_to_package_repository "${pkg_repo}")
-            eval 'printf '"'"'%s %s\n'"'" \
-              "$(printf '"${new_repo_revisions__%s}"' "${repo}")" \
-              "'${rest}'"
-          done
-      else
-        sed "s|^|${new_repo_revision} |"
-      fi | \
-      grep '^\S\+ .\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \
-      # ignore i686
-      grep -- '-\(any\|x86_64\)/PKGBUILD$' | \
-      # ignore staging and testing
-      grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
-      sed 's|^\(\S\+\) \(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\2 \3 \1 \4|'
-  done | \
-    sort -u | \
-    sed '
-      s|^D\s|0 \0|
-      t
-      s|^[AM]\s|1 \0|
-      t
-      s|^|2 |
-    ' | \
-    sort -k1,1 | \
-    sed 's|^[012] ||'
-} | \
-  while read -r mode package git_revision repository; do
-    if [ "${mode}" = 'D' ]; then
-      # deleted PKGBUILD
-      # shellcheck disable=SC2154
-      git_revision="${new_repo_revisions__archlinux32}"
-      found_package=false
-      # we need to test archlinux32 last, because otherwise
-      # find_package_repository_to_package might look in the wrong git
-      # repository of a package w/o upstream
-      for repository in ${repo_names} archlinux32; do
-        eval 'repo_path="${repo_paths__'"${repository}"'}"'
-        if [ "${repository}" = "archlinux32" ]; then
-          if git -C "${repo_path}" archive "${new_repo_revisions__archlinux32}" 2> /dev/null | \
-            tar -t 2> /dev/null | \
-            grep -q '^[^/]\+/'"$(str_to_regex "${package}")"'/PKGBUILD$'; then
-            found_package=true
-          fi
-        else
-          # shellcheck disable=SC2154
-          if eval 'git -C "${repo_path}" archive "${new_repo_revisions__'"${repository}"'}" -- "${package}/repos"' 2> /dev/null | \
-            tar -t --wildcards "${package}/repos/*/PKGBUILD" 2> /dev/null | \
-            cut -d/ -f3 | \
-            grep -v 'staging\|testing\|-unstable' | \
-            grep -vq -- '-i686$'; then
-            eval 'git_revision="${new_repo_revisions__'"${repository}"'}"'
-            found_package=true
-            break
-          fi
-        fi
-      done
-      if ${found_package}; then
-        mode='M'
-        repository=$(
-          find_package_repository_to_package "${package}" "${repository}" "${git_revision}"
-        )
-      else
-        delete_package 'any' "${package}"
+  directories=$(
+    git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" \
+    | tar -t \
+    | cut -d/ -f1 \
+    | grep -vF -- '-testing-' \
+    | grep -vF -- '-staging-' \
+    | sort -u
+  )
+  diff <(
+    {
+      printf 'SELECT `package_sources`.`pkgbase`,'
+      printf '`package_sources`.`git_revision`,'
+      printf '`upstream_repositories`.`name`'
+      printf ' FROM `package_sources`'
+      mysql_join_package_sources_upstream_repositories
+    } \
+    | mysql_run_query \
+    | tr '\t' ' ' \
+    | sort -u
+  ) <(
+    for directory in ${directories}; do
+      git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" -- "${directory}" \
+      | tar -Ox \
+      | cut -d' ' -f1,4 \
+      | sed '
+        s@$@ '"${directory%-*}"'@
+      '
+    done \
+    | sort -u
+  ) \
+  | grep '^[<>]' \
+  | awk '{print $1 " " $3 " " $2 " " $4}' \
+  | sort -k3,3 -k1r,1 \
+  | uniq -uf2 \
+  | while read -r mode git_revision pkgbase repository; do
+    if [ "${mode}" = '<' ]; then
+      if check_if_package_should_be_deleted "${pkgbase}" "${repository}"; then
+        delete_package 'any' "${pkgbase}"
         continue
+      else
+        mode='>'
       fi
+    if git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- "${repository}/${pkgbase}" >/dev/null 2>&1; then
+      mod_git_revision="${new_repo_revisions__archlinux32}"
+    else
+      mod_git_revision='0000000000000000000000000000000000000000'
     fi
-    if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then
-      # shellcheck disable=SC2016
-      {
-        # delete old binary packages which are not yet built or on the
-        # deletion list
-        mysql_query_delete_packages \
-          '`package_sources`.`pkgbase`=from_base64("'"$(
-            printf '%s' "${package}" | \
-              base64 -w0
-          )"'")' \
-          '`repositories`.`stability` IN ('"${repository_stability_ids__unbuilt}"','"${repository_stability_ids__forbidden}"')'
-        # remove is-to-be-deleted marker from old binary packages
-        printf 'UPDATE `binary_packages_in_repositories`'
-        mysql_join_binary_packages_in_repositories_binary_packages
-        mysql_join_binary_packages_build_assignments
-        mysql_join_build_assignments_package_sources
-        printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=0'
-        printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \
-          "$(
-            printf '%s' "${package}" | \
-              base64 -w0
-          )"
-      } | \
-        mysql_run_query
-      printf '%s\n' "${package}" >> "${tmp_dir}/modified-packages"
-      # shellcheck disable=SC2154
-      printf '%s ' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" >&2
-# TODO: sometimes we schedule the identical package twice (once "normal"
-# and once, because we found the deleted package elsewhere) - this could
-# be avoided
-      mysql_generate_package_metadata "${repository_ids__any_to_be_decided}" "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
-      printf '\n' >&2
+    printf '%s %s %s %s\n' "${pkgbase}" "${repository}" "${git_revision}" "${mod_git_revision}"
+  done
+  git -C "${repo_paths__archlinux32}" diff "${old_repo_revisions__archlinux32}^^" "${new_repo_revisions__archlinux32}" --name-status \
+  | tr '\t/' ' ' \
+  | cut -d' ' -f1,2,3 \
+  | sed '
+    s/^[AM]/>/
+    t
+    s/^D/</
+    t
+    d
+  ' \
+  | while read -r mode repository pkgbase; do
+    if check_if_package_should_be_deleted "${pkgbase}" "${repository}"; then
+      delete_package 'any' "${pkgbase}"
       continue
+    else
+      mode='>'
     fi
-
-    >&2 echo "unknown git diff mode '${mode}'"
-    exit 1
+    git_revision=$(
+      git -C "${repo_paths__state}" archive "${new_repo_revisions__state}" -- $(
+        printf '%s\n' "${directories}" \
+        | grep '^'"${repository}"'-'
+      ) \
+      | tar -Ox \
+      | sort -k1,1 \
+      | join -1 1 -2 1 -o 1.4 - <(echo "${pkgbase}")
+    )
+    if [ -z "${git_revision}" ]; then
+      git_revision='0000000000000000000000000000000000000000'
+    fi
+    if [ ${#git_revision} != 40 ] \
+    || printf '%s\n' "${git_revision}" \
+    | grep -vq '[0-9]'; then
+      >&2 printf 'invalid git revision "%s"\n'
+      exit 1
+    fi
+    printf '%s %s %s %s\n' "${pkgbase}" "${repository}" "${git_revision}" "${new_repo_revisions__archlinux32}"
   done
-
-if [ -s "${tmp_dir}/modified-packages" ] || ! ${pull}; then
-  echo 'Delete black-listed packages.'
-  if [ -s "${tmp_dir}/modified-packages" ]; then
-    sort -u "${tmp_dir}/modified-packages" --output "${tmp_dir}/modified-packages"
-  fi
-  # extract black-listed packages
-  git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
-    tar -t 'blacklist' | \
-    sed '
-      s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1\t\2@
-      t
-      d
-    ' | \
-    expand_blacklist_architectures "${tmp_dir}/architecture-compatibilities" | \
-    if [ -s "${tmp_dir}/modified-packages" ]; then
-      sort -k2,2 | \
-      join -1 1 -2 2 -o 2.1,2.2 "${tmp_dir}/modified-packages" -
-      sed '
-        /^lib32-/ s/^/any /
-        t
-        d
-      ' "${tmp_dir}/modified-packages"
-    else
-      cat
-    fi | \
-    sort -u | \
-    while read -r arch pkgbase; do
-      delete_package "${arch}" "${pkgbase}"
-    done
-  rm "${tmp_dir}/architecture-compatibilities"
+} \
+| sort -u \
+| sort -k1,1 -k2,2 \
+> "${tmp_dir}/modified-packages"
+
+errors=$(
+  awk '{print $3 " " $4 " " $1 " " $2}' \
+  < "${tmp_dir}/modified-packages" \
+  | uniq -Df2
+)
+if [ -n "${errors}" ]; then
+  >&2 printf 'Some packages are scheduled with different versions - this should not happen:\n'
+  >&2 printf '%s\n' "${errors}"
+  exit 1
 fi
 
+# TODO: do not use head, here
+grep -v '^lib32-' "${tmp_dir}/modified-packages" \
+| head -n3 \
+| sponge "${tmp_dir}/modified-packages"
+
+while read -r pkgbase repository git_revision mod_git_revision; do
+  # shellcheck disable=SC2016
+  {
+    # delete old binary packages which are not yet built or on the
+    # deletion list
+    mysql_query_delete_packages \
+      '`package_sources`.`pkgbase`=from_base64("'"$(
+        printf '%s' "${pkgbase}" \
+        | base64 -w0
+      )"'")' \
+      '`repositories`.`stability` IN ('"${repository_stability_ids__unbuilt}"','"${repository_stability_ids__forbidden}"')'
+    # remove is-to-be-deleted marker from old binary packages
+    printf 'UPDATE `binary_packages_in_repositories`'
+    mysql_join_binary_packages_in_repositories_binary_packages
+    mysql_join_binary_packages_build_assignments
+    mysql_join_build_assignments_package_sources
+    printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=0'
+    printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s");\n' \
+      "$(
+        printf '%s' "${pkgbase}" \
+        | base64 -w0
+      )"
+  } \
+  | mysql_run_query
+  # shellcheck disable=SC2154
+  >&2 printf '%s ' "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}"
+  mysql_generate_package_metadata "${repository_ids__any_to_be_decided}" "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}"
+  >&2 printf '\n'
+done \
+< "${tmp_dir}/modified-packages"
+
+# extract black-listed packages
+git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' \
+| tar -t 'blacklist' \
+| sed '
+  s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1\t\2@
+  t
+  d
+' \
+| expand_blacklist_architectures "${tmp_dir}/architecture-compatibilities" \
+| sort -k2,2 \
+| join -1 1 -2 2 -o 2.1,2.2 "${tmp_dir}/modified-packages" - \
+| sort -u \
+| while read -r arch pkgbase; do
+  delete_package "${arch}" "${pkgbase}"
+done
+
 echo 'Done - mark decisions as final.'
 
 # shellcheck disable=SC2016