From 8d3d7cabdad3b873722064632968f3ce916b3e99 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 1 Feb 2018 11:51:58 +0100
Subject: add many mysql-TODOs

---
 bin/cleanup | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'bin/cleanup')

diff --git a/bin/cleanup b/bin/cleanup
index 78daece..15b6fc9 100755
--- a/bin/cleanup
+++ b/bin/cleanup
@@ -5,6 +5,8 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# TODO: clean database, too
+
 # we only clean if run interactive or if no one is logged in
 if ! tty -s && \
   [ -n "$(users)" ]; then
-- 
cgit v1.2.3-70-g09d2


From 4f5f0fafc4be959a9a05adac8bf5162a7c5bab46 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 14:18:41 +0100
Subject: lib/common-functions: remove generate_package_metadata,
 delete_old_metadata - this information is now in the database

---
 bin/cleanup             |   3 -
 bin/get-package-updates |   1 -
 bin/seed-build-list     |   1 -
 lib/common-functions    | 228 +-----------------------------------------------
 4 files changed, 1 insertion(+), 232 deletions(-)

(limited to 'bin/cleanup')

diff --git a/bin/cleanup b/bin/cleanup
index 15b6fc9..02ac98b 100755
--- a/bin/cleanup
+++ b/bin/cleanup
@@ -99,9 +99,6 @@ find "${build_log_directory}/success" -maxdepth 1 -type f -mtime +14 \
   -not -exec zgrep -q '^+.*ELF file .* has text relocations' '{}' \; \
   -delete
 
-# remove old package meta data
-delete_old_metadata
-
 # remove dependency graphs of packages on the deletion list
 sed '
   s|^|'"${webserver_directory}"'/graphs/|
diff --git a/bin/get-package-updates b/bin/get-package-updates
index ce9d57a..a36cb8a 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -353,7 +353,6 @@ done | \
       } | \
         mysql_run_query
       mysql_generate_package_metadata 'to-be-decided' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
-      generate_package_metadata "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
       continue
     fi
 
diff --git a/bin/seed-build-list b/bin/seed-build-list
index 0db9cdf..41707dd 100755
--- a/bin/seed-build-list
+++ b/bin/seed-build-list
@@ -373,7 +373,6 @@ printf 'CREATE TEMPORARY TABLE `pkgbases` (`pkgbase` VARCHAR(64), `repository` V
       printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >&2
       printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" | \
         sed 's/ $/\n/'
-      generate_package_metadata "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
       mysql_generate_package_metadata 'build-list' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}"
       printf '%s %s %s %s\n' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" >> \
         "${work_dir}/build-list"
diff --git a/lib/common-functions b/lib/common-functions
index e3aab4c..0a5ef1c 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -6,7 +6,7 @@
 
 # TODO: have full information (currently in files) in database
 
-# TODO: remove state files / metadata files
+# TODO: remove state files
 
 if [ -z "${base_dir}" ]; then
   # just to make shellcheck happy
@@ -123,232 +123,6 @@ find_git_repository_to_package_repository() {
 
 }
 
-# generate_package_metadata $package $git_revision $mod_git_revision $repository
-# or
-# generate_package_metadata $package.$git_revision.$mod_git_revision.$repository
-# generate the meta data files of a package (dependencies, built packages, ...)
-
-generate_package_metadata() {
-
-  local package="$1"
-  local git_revision="$2"
-  local mod_git_revision="$3"
-  local repository="$4"
-  local file_prefix
-  local file
-  local PKGBUILD
-
-  if [ $# -eq 1 ]; then
-    # second form
-    repository="${package##*.}"
-    package="${package%.*}"
-    mod_git_revision="${package##*.}"
-    package="${package%.*}"
-    git_revision="${package##*.}"
-    package="${package%.*}"
-  fi
-
-  file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}"
-
-  if [ -e "${file_prefix}.builds" ] && \
-    [ -e "${file_prefix}.build-depends" ] && \
-    [ -e "${file_prefix}.run-depends" ] && \
-    [ -e "${file_prefix}.groups" ] && \
-    [ -e "${file_prefix}.packages" ]; then
-    return 0
-  fi
-
-  if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"; then
-    printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"
-    exit 1
-  fi
-  if [ ! -s "${file_prefix}.SRCINFO" ]; then
-    >&2 printf '"%s" not created by "make_source_info" - eh, what?' "${file_prefix}.SRCINFO"
-    exit 1
-  fi
-
-  # otherwise this just calls for trouble
-  sed -i '
-    /^[^=]*=\s*$/d
-    s/_i686\(\s*=\)/\1/
-  ' "${file_prefix}.SRCINFO"
-
-  # extract "groups" = groups \cup provides
-  grep "$(printf '^\t\\(groups\\|provides\\) = ')" "${file_prefix}.SRCINFO" | \
-    cut -d= -f2 | \
-    sed 's|^\s\+||; s|[<>]$||' | \
-    sort -u > \
-    "${file_prefix}.groups"
-
-  # extract "packages" = pkgname
-  grep '^pkgname = ' "${file_prefix}.SRCINFO" | \
-    cut -d= -f2 | \
-    sed 's|^\s\+||; s|[<>]$||' | \
-    sort -u > \
-    "${file_prefix}.packages"
-
-  # extract "builds" = provides \cup pkgname \cup groups
-  cat "${file_prefix}.groups" "${file_prefix}.packages" | \
-    sort -u > \
-    "${file_prefix}.builds"
-
-  # extract "build-depends" = makedepends \cup depends \cup \{ base, base-devel \} \setminus "builds"
-  {
-    {
-      printf 'all_depend = %s\n' 'base' 'base-devel'
-      sed -n "$(
-        printf '/^\t%s = /p\n' \
-          'depends' \
-          'makedepends'
-      )" "${file_prefix}.SRCINFO"
-    } | \
-      cut -d= -f2 | \
-      sed 's|^\s\+||; s|[<>]$||' | \
-      sort -u
-    sed 'p' "${file_prefix}.builds"
-  } | \
-    sort | \
-    uniq -u > \
-    "${file_prefix}.build-depends"
-
-  # extract "run-depends" = depends \cup \{ base \} \setminus "builds"
-  {
-    {
-      printf 'all_depend = %s\n' 'base'
-      sed -n "$(printf '/^\tdepends = /p')" "${file_prefix}.SRCINFO"
-    } | \
-      cut -d= -f2 | \
-      sed 's|^\s\+||; s|[<>]$||' | \
-      sort -u
-    sed 'p' "${file_prefix}.builds"
-  } | \
-    sort | \
-    uniq -u > \
-    "${file_prefix}.run-depends"
-
-  rm "${file_prefix}.SRCINFO"
-
-}
-
-# delete_old_metadata
-# delete old (=unneeded) meta data of packages
-
-delete_old_metadata() {
-
-  local current_metadata
-
-  current_metadata=$(
-    find "${work_dir}/package-infos" -mindepth 1 -maxdepth 1 -printf '%f\n' | \
-      sed '
-        s|\.[^.]\+$||
-        s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3|
-      ' | \
-      sort -u
-  )
-
-  ( # the new shell is intentional
-    # what we have
-    echo "${current_metadata}"
-
-    # package-states should stay
-    find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -printf '%f\n' | \
-      sed '
-        s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
-      ' | \
-      sort -u | \
-      sed 'p'
-
-    # build-list items should stay
-    sed 'p' "${work_dir}/build-list"
-
-    tmp_dir=$(mktemp -d 'tmp.common-functions.delete_old_metadata.XXXXXXXXXX' --tmpdir)
-    trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-    echo "${current_metadata}" | \
-      sort -k1,1 > \
-      "${tmp_dir}/current-metadata"
-
-    # the newest of the following should stay:
-    {
-      # deletion-list items
-      cat "${work_dir}/deletion-list"
-      # all packages in the repos
-      for repo in ${repo_names}; do
-        eval 'git -C "${repo_paths__'"${repo}"'}" archive '"$(cat "${work_dir}/${repo}.revision")" | \
-          tar -t | \
-          sed '
-            s|/$||
-            /\//d
-          '
-      done
-    } | \
-      sort -u | \
-      join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \
-      sort -k4,4 > \
-      "${tmp_dir}/find-newest-revisions"
-
-    uniq -uf3 < \
-      "${tmp_dir}/find-newest-revisions" | \
-      awk '{print $4 " " $1 " " $2 " " $3}' | \
-      sed 'p'
-
-    uniq -Df3 < \
-      "${tmp_dir}/find-newest-revisions" | \
-      uniq --group=append -f3 | \
-      {
-        revs=''
-        mod_revs=''
-        opkg=''
-        orepo=''
-        while read -r rev mod_rev repo pkg; do
-
-          if [ -z "${rev}" ] && \
-            [ -z "${mod_rev}" ] && \
-            [ -z "${repo}" ] && \
-            [ -z "${pkg}" ]; then
-
-            printf '%s %s %s %s\n' \
-              "$(
-                printf '%s\n' ${revs} | \
-                  find_newest_of_git_revisions
-              )" \
-              "$(
-                printf '%s\n' ${mod_revs} | \
-                  find_newest_of_git_revisions
-              )" \
-              "${orepo}" \
-              "${opkg}"
-
-            revs=''
-            mod_revs=''
-            orepo=''
-            opkg=''
-            continue
-          fi
-          revs=$(
-            # shellcheck disable=SC2086
-            printf '%s\n' ${revs} ${rev} | \
-              sort -u
-          )
-          mod_revs=$(
-            # shellcheck disable=SC2086
-            printf '%s\n' ${mod_revs} ${mod_rev} | \
-              sort -u
-          )
-          orepo="${repo}"
-          opkg="${pkg}"
-        done
-      } | \
-      awk '{print $4 " " $1 " " $2 " " $3}' | \
-      sed 'p'
-  ) | \
-    sort | \
-    uniq -u | \
-    while read -r pkg rev mod_rev repo; do
-      rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."*
-    done
-}
-
 # repository_of_package $package.$repo_revision.$mod_repo_revision.$repository
 # print which (stable) repository a package belongs to
 
-- 
cgit v1.2.3-70-g09d2


From 89841624b9eff687dec43c43efba1b5fd54c789e Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 14:36:52 +0100
Subject: bin/cleanup: take info about "recent" logfiles from database

---
 bin/cleanup | 38 ++++++++++++++------------------------
 1 file changed, 14 insertions(+), 24 deletions(-)

(limited to 'bin/cleanup')

diff --git a/bin/cleanup b/bin/cleanup
index 02ac98b..a0bf790 100755
--- a/bin/cleanup
+++ b/bin/cleanup
@@ -42,31 +42,21 @@ fi
 # remove logs where package is not broken/locked anymore
 
 {
-  find "${build_log_directory}/error" -maxdepth 1 -type f -printf '1 %f %f\n' | \
-    sed 's|\.[^. ]\+\.build-log\.gz$||'
-  find "${work_dir}/package-states" -maxdepth 1 \( \
-    -name '*.broken' -o \
-    -name '*.done' -o \
-    -name '*.locked' -o \
-    -name '*.testing' \
-  \) -printf '0 0 %f\n' | \
-    sed 's|\.[^.]\+$||'
+  find "${build_log_directory}/error" -maxdepth 1 -type f -printf 'file %f\n'
+  # shellcheck disable=SC2016
+  printf 'SELECT "mysql",`failed_builds`.`log_file` FROM `failed_builds`;\n' | \
+    mysql_run_query | \
+    tr '\t' ' '
 } | \
-  sort -k3,3 -k1,2 | \
-  uniq --group=prepend -f2 | \
-  while read -r num file _; do
-    if [ "${num}" = '0' ]; then
-      while read -r line; do
-        if [ -z "${line}" ]; then
-          break
-        fi
-      done
-      continue
-    fi
-    if [ -z "${num}${file}" ]; then
-      continue
-    fi
-    rm -f "${build_log_directory}/error/${file}"
+  sort -k2,2 -k1,1 | \
+  uniq -uf 1 | \
+  sed -n '
+    s/^file //
+    T
+    p
+  ' | \
+  while read -r file; do
+    rm "${build_log_directory}/error/${file}"
   done
 
 # only keep 10 newest logs per failed package
-- 
cgit v1.2.3-70-g09d2


From c772be57eca3a2176ac3931c02fb582f85c0d575 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 15:19:23 +0100
Subject: decommission package-states directory - we rely on the database now

---
 bin/bootstrap-mysql      | 60 ++++------------------------------------------
 bin/cleanup              | 20 ----------------
 bin/db-update            | 62 ------------------------------------------------
 bin/get-assignment       | 27 +++------------------
 bin/get-package-updates  |  6 -----
 bin/modify-package-state | 38 -----------------------------
 bin/return-assignment    | 38 -----------------------------
 lib/common-functions     |  4 ----
 8 files changed, 7 insertions(+), 248 deletions(-)

(limited to 'bin/cleanup')

diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql
index 4c1938f..fc347d7 100755
--- a/bin/bootstrap-mysql
+++ b/bin/bootstrap-mysql
@@ -3,6 +3,10 @@
 # shellcheck source=conf/default.conf
 . "${0%/*}/../conf/default.conf"
 
+# TODO: this should become some sort of mysql dump to bootstrap the
+# tables and stored functions only - when the database is gone, we're
+# pretty much screwed anyway :-D
+
 tmp_dir=$(mktemp -d 'tmp.bootstrap-mysql.XXXXXXXXXX' --tmpdir)
 trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 
@@ -792,68 +796,12 @@ fi
 } | \
   mysql_run_query
 
-find "${work_dir}/package-states" \
-  -maxdepth 1 \
-  -mindepth 1 \
-  \( \
-    -name '*.done' -o \
-    -name '*.testing' -o \
-    -name '*.tested' \
-  \) \
-  -exec sed '
-    1!d
-    s@^.*-\([^-]\+\)-\([^-.]\+\)\.pkg\.tar\.xz$@{} \1 \2 \0@
-    s@^\S*/@@
-    s/^\(\S\+\)\.\(done\|testing\|tested\) /\1 /
-    s/ \([0-9]\+\) / \1.0 /
-    s/ [0-9]\+\.\([0-9]\+\) / \1 /
-  ' {} \; | \
-  while read -r state_file sub_pkgrel arch package_file; do
-    printf '%s ' "${state_file}"
-    repository=$(
-      grep '^\S\+ '"$(str_to_regex "${package_file}")"'$' "${tmp_dir}/master-mirror-listing" | \
-        cut -d' ' -f1
-    )
-    if [ "$(echo "${repository}" | grep -cx '\S\+')" -ne 1 ]; then
-      printf 'not found exactly once on the master mirror.\n'
-      >&2 printf '"%s"\n' "${state_file}" "${sub_pkgrel}" "${arch}" "${package_file}"
-      continue
-    fi
-    mysql_generate_package_metadata "${sub_pkgrel}" "${repository}" "${state_file}"
-    printf ' ok\n'
-  done
-
 tr ' ' '.' < \
   "${work_dir}/build-list" | \
   while read -r state_file; do
     mysql_generate_package_metadata 'build-list' "${state_file}"
   done
 
-# shellcheck disable=SC2016
-find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \
-  sed '
-    s|\.\([^. ]\+\)\.\([^. ]\+\)\.\([^. ]\+\)\.blocked$| \1 \2 \3|
-  ' | \
-  while read -r state_file pkgbase git_revision mod_git_revision repository; do
-    printf 'UPDATE `build_assignments`'
-    mysql_join_build_assignments_package_sources
-    mysql_join_package_sources_upstream_repositories
-    printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \
-      "$(
-        tr -d '\n' < \
-          "${state_file}" | \
-          base64 -w0
-      )"
-    printf ' WHERE'
-    printf ' `package_sources`.`%s`=from_base64("%s") AND' \
-      'pkgbase'          "$(printf '%s' "${pkgbase}" |          base64 -w0)" \
-      'git_revision'     "$(printf '%s' "${git_revision}" |     base64 -w0)" \
-      'mod_git_revision' "$(printf '%s' "${mod_git_revision}" | base64 -w0)"
-    printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \
-      "$(printf '%s' "${repository}" | base64 -w0)"
-  done | \
-  mysql_run_query
-
 grep '^\('"$(
   # shellcheck disable=SC2086
   printf '%s\\|' ${standalone_package_repositories} ${stable_package_repositories}
diff --git a/bin/cleanup b/bin/cleanup
index a0bf790..7aaa39f 100755
--- a/bin/cleanup
+++ b/bin/cleanup
@@ -19,26 +19,6 @@ if [ -s "${work_dir}/build-master-sanity" ]; then
   exit
 fi
 
-# remove blocked/broken/locked markes of packages not on the buildlist anymore
-
-{
-  find "${work_dir}/package-states" -maxdepth 1 \( -name '*.broken' -o -name '*.locked' -o -name '*.blocked' \) -printf '%f\n' | \
-    sed '
-      s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$|state \0 \1 \2 \3 \4 \5|
-    '
-  awk '{
-    print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " broken"
-    print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " blocked"
-    print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " locked"
-  }' "${work_dir}/build-list"
-} | \
-  sort -k3 | \
-  uniq -uf2 | \
-  grep '^state ' | \
-  awk '{print $2}' | \
-  sed "s|^|${work_dir}/package-states/|" | \
-  xargs -rn1 rm
-
 # remove logs where package is not broken/locked anymore
 
 {
diff --git a/bin/db-update b/bin/db-update
index 6ecf9c9..6722f24 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -207,12 +207,6 @@ for source_stability in 'testing' 'staging'; do
       mysql_join_repositories_repository_stabilities
       printf ' AND `repository_stabilities`.`name`="%s"' \
         "${source_stability}"
-      # TODO: we may relax this condition, if we a) got rid of state files
-      # or b) improved keeping them in sync (*.testing -> /dev/null is not
-      # implemented, only .tested -> /dev/null)
-      if [ ! "${source_stability}" = 'staging' ]; then
-        printf ' AND `binary_packages`.`is_tested`'
-      fi
       mysql_join_binary_packages_build_assignments
       mysql_join_build_assignments_package_sources
       mysql_join_package_sources_upstream_repositories
@@ -401,62 +395,6 @@ for source_stability in 'testing' 'staging'; do
     printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`;\n'
   } | \
     mysql_run_query
-
-  sed_apply_rm=$(
-    while read -r pkg _ _; do
-      printf '/^%s$/d\n' "$(str_to_regex "${pkg}")"
-    done < \
-      "${tmp_dir}/rm"
-  )
-
-  sed_apply_remove_part_of_mv=$(
-    while read -r pkg _ _; do
-      printf '/^%s$/d\n' "$(str_to_regex "${pkg}")"
-    done < \
-      "${tmp_dir}/mv"
-  )
-
-  sed_apply_move_part_of_mv=$(
-    while read -r pkg _ _; do
-      printf '/^%s$/{\n' "$(str_to_regex "${pkg}")"
-      printf 'w %%s\n'
-      printf 'd\n'
-      printf '}\n'
-    done < \
-      "${tmp_dir}/mv"
-  )
-
-  find "${work_dir}/package-states/" \
-    \( \
-      -name '*.done' -o \
-      -name '*.testing' -o \
-      -name '*.tested' \
-    \) \
-    -exec sed -i "${sed_apply_rm}" '{}' \;
-  find "${work_dir}/package-states/" \
-    -name '*.tested' \
-    -exec sed -i "${sed_apply_remove_part_of_mv}" '{}' \;
-  find "${work_dir}/package-states/" \
-    -name '*.done' \
-    -exec grep -qxF "$(cut -d' ' -f1 < "${tmp_dir}/mv")" '{}' \; \
-    -printf '%p\n' | \
-    while read -r sf; do
-      sed -i "$(
-        printf '%s\n' "${sed_apply_move_part_of_mv}" | \
-          sed '
-            s,%s,'"${sf%.done}.testing"',g
-          '
-      )" "${sf}"
-    done
-  find "${work_dir}/package-states/" \
-    \( \
-      -name '*.done' -o \
-      -name '*.testing' -o \
-      -name '*.tested' \
-    \) \
-    -type f \
-    -not -exec test -s '{}' \; \
-    -delete
 done
 
 trigger_mirror_refreshs
diff --git a/bin/get-assignment b/bin/get-assignment
index 3357fc6..a82e1f8 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -18,17 +18,8 @@
 # TODO: honor manual build order of tool-chain:
 # toolchain build order: linux-api-headers->glibc->binutils->gcc->binutils->glibc
 
-mkdir -p "${work_dir}/package-states"
-
 hand_out_assignment() {
 
-  # we don't care anymore if an older version of this package was
-  # "locked" or "broken"
-  find "${work_dir}/package-states" -maxdepth 1 -regextype grep \
-    -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{3\}\.\(locked\|broken\)' \
-    -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4.")"'[^.]\+' \
-    -delete
-
   # move that build order to the end of the build-list
   sed -i '
     /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ {
@@ -57,21 +48,7 @@ hand_out_assignment() {
     printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \
       "$(printf '%s' "$4" | base64 -w0)"
     printf ' LIMIT 1;\n'
-  } | \
-    mysql_run_query | \
-    tr '\t' ' '
 
-  {
-    # shellcheck disable=SC2154
-    echo "${slave}"
-    if [ -r "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then
-      cat "${work_dir}/package-states/$1.$2.$3.$4.locked"
-    fi
-  } | \
-    sort -u | \
-    sponge "${work_dir}/package-states/$1.$2.$3.$4.locked"
-  # shellcheck disable=SC2016
-  {
     printf 'UPDATE `build_slaves`'
     printf ' SET `currently_building` = ('
     printf ' SELECT `build_assignments`.`id`'
@@ -86,6 +63,7 @@ hand_out_assignment() {
     printf ' `upstream_repositories`.`name` = from_base64("%s")' \
       "$(printf '%s' "$4" | base64 -w0)"
     printf ')'
+    # shellcheck disable=SC2154
     printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \
       "$(printf '%s' "${slave}" | base64 -w0)"
 
@@ -101,7 +79,8 @@ hand_out_assignment() {
     printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \
       "$(printf '%s' "$4" | base64 -w0)"
   } | \
-    mysql_run_query
+    mysql_run_query | \
+    tr '\t' ' '
 
   exit 0
 
diff --git a/bin/get-package-updates b/bin/get-package-updates
index a36cb8a..3e9edf9 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -111,9 +111,6 @@ delete_package() {
     mysql_run_query >> \
     "${work_dir}/deletion-list.new"
   sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new"
-  find "${work_dir}/package-states" -maxdepth 1 -mindepth 1 -regextype grep \
-    -regex '.*/'"$(str_to_regex "${1}")"'\(\.[^.]\+\)\{3\}\.\(broken\|blocked\|locked\)' \
-    -delete
   # TODO: Once we want to rely on the database for test_exclusion, we
   # need to run the command below unconditionally, but with some
   # changes, so we can easily revert.
@@ -317,9 +314,6 @@ done | \
     if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then
       # new or modified PKGBUILD
       sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new"
-      find "${work_dir}/package-states" -regextype grep \
-        -regex '.*/'"$(str_to_regex "${package}")"'\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.\(blocked\|broken\)' \
-        -delete
       # shellcheck disable=SC2154
       echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \
         "${work_dir}/build-list.new"
diff --git a/bin/modify-package-state b/bin/modify-package-state
index 76b8c1e..4a2f5ac 100755
--- a/bin/modify-package-state
+++ b/bin/modify-package-state
@@ -243,46 +243,8 @@ while read -r package reason; do
     {
       printf 'UPDATE %s SET %s %s' "${combiner}" "${modifier}" "${selector}"
       printf ';\n'
-      # TODO: remove the below part, once the database if fully functional
-      printf 'UPDATE `binary_packages` AS `a`'
-      printf ' JOIN `binary_packages` AS `b`'
-      printf ' ON `a`.`build_assignment`=`b`.`build_assignment`'
-      mysql_join_binary_packages_repositories 'b'
-      printf ' SET `b`.`is_tested`=1'
-      printf ' WHERE `a`.`is_tested`'
-      printf ' AND NOT `b`.`is_tested`'
-      printf ' AND NOT `repositories`.`name`="deletion-list";\n'
     } | \
       mysql_run_query
-    case "${action}" in
-      'block')
-        printf '%s' "${reason}" > \
-          "${work_dir}/package-states/${package}.blocked"
-      ;;
-      'unblock')
-        rm -f "${work_dir}/package-states/${package}.blocked"
-      ;;
-      'faulty')
-        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
-          -name '*.tested' \
-          -exec grep -lxF "${package}" '{}' \; | \
-          while read -r sf; do
-            mv "${sf}" "${sf%ed}ing"
-          done
-      ;;
-      'tested')
-        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \
-          -name '*.testing' \
-          -exec grep -lxF "${package}" '{}' \; | \
-          while read -r sf; do
-            mv "${sf}" "${sf%ing}ed"
-          done
-      ;;
-      *)
-        >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}"
-        exit 42
-      ;;
-    esac
     printf '%s %s\n' "${action}" "${package}"
   fi
 done \
diff --git a/bin/return-assignment b/bin/return-assignment
index 02bd491..da9e762 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -89,9 +89,6 @@ if [ "$5" = 'ERROR' ]; then
       '*.build-log.gz'
   )
 
-  # shellcheck disable=SC2154
-  echo "${slave}" >> \
-    "${work_dir}/package-states/$1.$2.$3.$4.broken"
   # shellcheck disable=SC2016
   {
     if [ -n "${saved_build_logs}" ]; then
@@ -162,23 +159,6 @@ if [ "$5" = 'ERROR' ]; then
   } | \
     mysql_run_query
 
-  # shellcheck disable=SC2154
-  sed -i '
-    /^'"$(str_to_regex "${slave}")"'$/d
-  ' "${work_dir}/package-states/$1.$2.$3.$4.locked"
-  if [ ! -s "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then
-    rm "${work_dir}/package-states/$1.$2.$3.$4.locked"
-
-    # move that build order to the end of the build-list
-    sed -i '
-      /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ {
-        $ b
-        d
-      }
-      $ a '"$1 $2 $3 $4" \
-      "${work_dir}/build-list"
-  fi
-
   # release lock on build-list - otherwise seed-build-list won't run
   flock -u 9
 
@@ -517,24 +497,9 @@ for package in ${packages}; do
   remove_old_package_versions 'i686' "${destination}" "${package}"
 done
 
-# remove old state files (these should be only "done" markers, but
-# actually we don't care what it is) - as long as it's not "testing" or "tested"
-find "${work_dir}/package-states" -maxdepth 1 -regextype grep \
-  -not -name '*.testing' \
-  -not -name '*.tested' \
-  -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{4\}' \
-  -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4")"'\.[^.]\+' \
-  -exec rm '{}' \;
-
 # remove package from build list
 sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list"
 
-# remove package lock file
-if ! [ "${destination}" = 'build-support' ]; then
-  # shellcheck disable=SC2086
-  printf '%s\n' ${packages} > \
-    "${work_dir}/package-states/$1.$2.$3.$4.done"
-fi
 # shellcheck disable=SC2016
 {
   printf 'UPDATE `build_assignments`'
@@ -578,6 +543,3 @@ fi
   printf 'DROP TABLE `loops_to_delete`;\n'
 } | \
   mysql_run_query
-rm -f \
-  "${work_dir}/package-states/$1.$2.$3.$4.locked" \
-  "${work_dir}/package-states/$1.$2.$3.$4.broken"
diff --git a/lib/common-functions b/lib/common-functions
index 0a5ef1c..f66ccd1 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -4,10 +4,6 @@
 
 # shellcheck disable=SC2039
 
-# TODO: have full information (currently in files) in database
-
-# TODO: remove state files
-
 if [ -z "${base_dir}" ]; then
   # just to make shellcheck happy
   . 'conf/default.conf'
-- 
cgit v1.2.3-70-g09d2


From ba6f43ca62a719d075cd23ccba35dbbab5f43951 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Thu, 22 Mar 2018 15:45:47 +0100
Subject: decommission deletion-list

---
 bin/build-master-status-from-mysql | 20 --------------------
 bin/cleanup                        |  7 -------
 bin/get-package-updates            | 35 ++++-------------------------------
 conf/default.conf                  |  1 -
 4 files changed, 4 insertions(+), 59 deletions(-)

(limited to 'bin/cleanup')

diff --git a/bin/build-master-status-from-mysql b/bin/build-master-status-from-mysql
index c6df53b..962b12e 100755
--- a/bin/build-master-status-from-mysql
+++ b/bin/build-master-status-from-mysql
@@ -20,28 +20,8 @@ fi
 # shellcheck disable=SC2119
 mysql_cleanup
 
-# shellcheck disable=SC2016
-{
-  printf 'SELECT DISTINCT `package_sources`.`pkgbase`'
-  printf ' FROM `package_sources`'
-  mysql_join_package_sources_build_assignments
-  mysql_join_build_assignments_binary_packages
-  mysql_join_binary_packages_repositories
-  printf ' WHERE `repositories`.`name`="deletion-list"'
-  printf ' OR `binary_packages`.`is_to_be_deleted`;\n'
-} | \
-  mysql_run_query | \
-  sort > \
-  "${tmp_dir}/deletion-list.mysql"
-sort "${work_dir}/deletion-list" > \
-  "${tmp_dir}/deletion-list.file"
-
 {
   mysql_sanity_check || true
-  diff -u \
-    "${tmp_dir}/deletion-list.file" \
-    "${tmp_dir}/deletion-list.mysql" || \
-    true
 } | \
   sed '
     s,^-.*$,<font color="#FF0000">\0</font>,
diff --git a/bin/cleanup b/bin/cleanup
index 7aaa39f..c9a8d2b 100755
--- a/bin/cleanup
+++ b/bin/cleanup
@@ -69,11 +69,4 @@ find "${build_log_directory}/success" -maxdepth 1 -type f -mtime +14 \
   -not -exec zgrep -q '^+.*ELF file .* has text relocations' '{}' \; \
   -delete
 
-# remove dependency graphs of packages on the deletion list
-sed '
-  s|^|'"${webserver_directory}"'/graphs/|
-  s|$|.png|
-' "${work_dir}/deletion-list" | \
-  xargs -rn1 rm -f
-
 exit 0
diff --git a/bin/get-package-updates b/bin/get-package-updates
index dcc9f3b..f4c53b4 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -99,17 +99,6 @@ fi
 # delete_package package
 # mark $package for deletion
 delete_package() {
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT DISTINCT `package_sources`.`pkgbase`'
-    printf ' FROM `package_sources`'
-    mysql_join_package_sources_build_assignments
-    mysql_join_build_assignments_binary_packages
-    printf ' WHERE `package_sources`.`pkgbase`=from_base64("%s")' \
-      "$(printf '%s' "$1" | base64 -w0)"
-  } | \
-    mysql_run_query >> \
-    "${work_dir}/deletion-list.new"
   # TODO: Once we want to rely on the database for test_exclusion, we
   # need to run the command below unconditionally, but with some
   # changes, so we can easily revert.
@@ -200,10 +189,6 @@ echo 'Check modified packages from the last update, and put them to the build li
 # If a package is deleted, remove from the rebuild list, and add it to the deletion list.
 # If a new package is added, then ensure that it's not on the deletion list.
 
-cp \
-  "${work_dir}/deletion-list" \
-  "${work_dir}/deletion-list.new"
-
 for repo in ${repo_names}; do
   eval repo_path='"${repo_paths__'"${repo}"'}"'
   eval old_repo_revision='"${old_repo_revisions__'"${repo}"'}"'
@@ -291,7 +276,6 @@ done | \
       fi
     fi
     if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then
-      sed -i "/^$(str_to_regex "${package}")\$/d" "${work_dir}/deletion-list.new"
       # shellcheck disable=SC2016
       {
         # delete old build assignment and associated binary packages
@@ -329,9 +313,6 @@ done | \
     exit 1
   done
 
-sort -u "${work_dir}/deletion-list.new" | \
-  sponge "${work_dir}/deletion-list.new"
-
 echo 'apply blacklisting'
 # ignore blacklisted packages and dependent packages
 # this is the first time when all the information is available and up to date
@@ -462,15 +443,10 @@ echo "${black_listed}" | \
     delete_package "${package}"
   done
 
-sort -u "${work_dir}/deletion-list.new" | \
-  sponge "${work_dir}/deletion-list.new"
-
 
 if [ -n "${test_exclusion}" ]; then
-  # we should not actually update the build-list et. al, but solely print this difference:
-  if diff --color -u "${work_dir}/deletion-list" "${work_dir}/deletion-list.new"; then
-    printf 'If you put "%s" on the blacklist, no additional packages will end up on the deletion list.\n' "${test_exclusion}"
-  fi
+  # TODO: reimplement test_exclusion with information from the database
+  >&2 echo 'sry, not yet done.'
   exit 0
 fi
 
@@ -504,11 +480,8 @@ mysql_find_build_assignment_loops
 
 # Move the .new-files to the actual files
 
-{
-  printf '%s\n' "deletion-list"
-  # shellcheck disable=SC2086
-  printf '%s.revision\n' ${repo_names}
-} | \
+# shellcheck disable=SC2086
+printf '%s.revision\n' ${repo_names} | \
   while read -r file; do
     mv "${work_dir}/${file}.new" "${work_dir}/${file}"
   done
diff --git a/conf/default.conf b/conf/default.conf
index e8e0169..11483a0 100755
--- a/conf/default.conf
+++ b/conf/default.conf
@@ -133,7 +133,6 @@ if [ -z "${package_database_lock_file}" ]; then
 fi
 
 mkdir -p "${work_dir}"
-touch "${work_dir}/deletion-list"
 
 for repo in ${repo_names}; do
 
-- 
cgit v1.2.3-70-g09d2