From 8e6b99b7ee6bd8589c4f005315e4657c95f182b5 Mon Sep 17 00:00:00 2001
From: Erich Eckner <git@eckner.net>
Date: Fri, 28 Jul 2017 10:45:14 +0200
Subject: files in package-infos contain the repository now, too

---
 bin/build-master-status |   2 +-
 bin/common-functions    |  36 ++++++++------
 bin/db-update           |   4 +-
 bin/get-assignment      |   4 +-
 bin/get-package-updates | 126 ++++++++++++++++++++++++++++++++++++++----------
 bin/return-assignment   |   2 +-
 bin/show-dependencies   |   6 +--
 bin/why_dont_you        |  37 ++++++--------
 8 files changed, 144 insertions(+), 73 deletions(-)

diff --git a/bin/build-master-status b/bin/build-master-status
index c8d4fab..4024a2d 100755
--- a/bin/build-master-status
+++ b/bin/build-master-status
@@ -107,7 +107,7 @@ pending_packages="$(
     tr ' ' '.' | \
     while read -r package; do
       generate_package_metadata "${package}" &> /dev/null
-      cat "${work_dir}/package-infos/${package%.*}.packages"
+      cat "${work_dir}/package-infos/${package}.packages"
     done |
     wc -l
 )"
diff --git a/bin/common-functions b/bin/common-functions
index e59cac0..bab3757 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -130,7 +130,7 @@ generate_package_metadata() {
     package="${package%.*}"
   fi
 
-  file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}"
+  file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}"
 
   if [ -e "${file_prefix}.builds" ] && \
     [ -e "${file_prefix}.depends" ] && \
@@ -207,7 +207,7 @@ delete_old_metadata() {
     ls "${work_dir}/package-infos" | \
       sed '
         s|\.[^.]\+$||
-        s|\.\([^.]\+\)\.\([^.]\+\)$| \1 \2|
+        s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3|
       ' | \
       sort -u
   )"
@@ -219,14 +219,13 @@ delete_old_metadata() {
     # package-states should stay
     ls -1 "${work_dir}/package-states" | \
       sed '
-        s|\.\([^.]\+\)\.\([^.]\+\)\(\.[^.]\+\)\{2\}$| \1 \2|
+        s|\.\([^.]\+\)\.\([^.]\+\)\(\.[^.]\+\)\.\([^.]\+\)$| \1 \2 \3|
       ' | \
       sort -u | \
       sed 'p'
 
     # build-list items should stay
-    cut -d' ' -f1,2,3 "${work_dir}/build-list" | \
-      sed 'p'
+    sed 'p' "${work_dir}/build-list"
 
     tmp_dir="$(mktemp -d)"
     trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
@@ -250,29 +249,31 @@ delete_old_metadata() {
       done
     ) | \
       sort -u | \
-      join -j 1 -o 2.2,2.3,2.1 - "${tmp_dir}/current-metadata" | \
-      sort -k3,3 > \
+      join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \
+      sort -k4,4 > \
       "${tmp_dir}/find-newest-revisions"
 
-    uniq -uf2 < \
+    uniq -uf3 < \
       "${tmp_dir}/find-newest-revisions" | \
-      awk '{print $3 " " $1 " " $2}' | \
+      awk '{print $4 " " $1 " " $2 " " $3}' | \
       sed 'p'
 
-    uniq -Df2 < \
+    uniq -Df3 < \
       "${tmp_dir}/find-newest-revisions" | \
-      uniq --group=append -f2 | \
+      uniq --group=append -f3 | \
       (
         revs=''
         mod_revs=''
         opkg=''
-        while read -r rev mod_rev pkg; do
+        orepo=''
+        while read -r rev mod_rev repo pkg; do
 
           if [ -z "${rev}" ] && \
             [ -z "${mod_rev}" ] && \
+            [ -z "${repo}" ] && \
             [ -z "${pkg}" ]; then
 
-            printf '%s %s %s\n' \
+            printf '%s %s %s %s\n' \
               "$(
                 printf '%s\n' ${revs} | \
                   find_newest_of_git_revisions
@@ -281,10 +282,12 @@ delete_old_metadata() {
                 printf '%s\n' ${mod_revs} | \
                   find_newest_of_git_revisions
               )" \
+              "${orepo}" \
               "${opkg}"
 
             revs=''
             mod_revs=''
+            orepo=''
             opkg=''
             continue
           fi
@@ -296,16 +299,17 @@ delete_old_metadata() {
             printf '%s\n' ${mod_revs} ${mod_rev} | \
               sort -u
           )"
+          orepo="${repo}"
           opkg="${pkg}"
         done
       ) | \
-      awk '{print $3 " " $1 " " $2}' | \
+      awk '{print $4 " " $1 " " $2 " " $3}' | \
       sed 'p'
   ) | \
     sort | \
     uniq -u | \
-    while read -r pkg rev mod_rev; do
-      rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}".*
+    while read -r pkg rev mod_rev repo; do
+      rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."*
     done
 }
 
diff --git a/bin/db-update b/bin/db-update
index 39d972f..af70c10 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -444,7 +444,7 @@ done
 grep -vxF 'break_loops' "${work_dir}/build-list" | \
   while read -r pkg pkg_rev mod_rev repo; do
     generate_package_metadata "${pkg}" "${pkg_rev}" "${mod_rev}" "${repo}"
-    cat "${work_dir}/package-infos/${pkg}.${pkg_rev}.${mod_rev}.depends"
+    cat "${work_dir}/package-infos/${pkg}.${pkg_rev}.${mod_rev}.${repo}.depends"
   done | \
   sort -u > \
   "${tmp_dir}/keep_packages"
@@ -491,7 +491,7 @@ while [ -s "${tmp_dir}/new_keep_packages" ]; do
 
   new_keep_packages="$(
     while read -r package; do
-      cat "${work_dir}/package-infos/${package%.*}.depends"
+      cat "${work_dir}/package-infos/${package}.depends"
     done < \
       "${tmp_dir}/new_keep_packages" | \
       sort -u
diff --git a/bin/get-assignment b/bin/get-assignment
index 68d2658..d426988 100755
--- a/bin/get-assignment
+++ b/bin/get-assignment
@@ -146,9 +146,9 @@ for hand_out_broken in false true; do
 
     [ -z "$(
       (
-        cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.needs"
+        cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.needs"
         grep -vxF 'break_loops' "${work_dir}/build-list" | \
-          awk '{print $1 "." $2 "." $3}' | \
+          awk '{print $1 "." $2 "." $3 "." $4}' | \
           sed "
             s|^|${work_dir}/package-infos/|
             s|\$|\.builds|
diff --git a/bin/get-package-updates b/bin/get-package-updates
index b59ae5b..2fec570 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -228,37 +228,113 @@ echo 'apply blacklisting'
 
 black_listed=''
 black_listed_new="$(
-  cat "${repo_paths__archlinux32}/blacklist"
-  ls "${work_dir}/package-infos" | \
-    grep '\.depends$' | \
-    sed "s|^|${work_dir}/package-infos/|" | \
-    xargs -r grep -l '^lib32-' | \
-    sed '
-      s|^.*/||
-      s|\(\.[^.]\+\)\{3\}$||
-    '
+  (
+    git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- 'blacklist' | \
+      tar -Ox 'blacklist'
+    ls "${work_dir}/package-infos" | \
+      grep '^lib32-' | \
+      sed 's|\(\.[^.]\+\)\{4\}$||' | \
+      uniq
+  ) | \
+    sort -u
 )"
 
+ls "${work_dir}/package-infos/" | \
+  grep '\.builds$' | \
+  sed '
+    s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$|\2 \3 \4 \1|
+  ' | \
+  sort -k4,4 | \
+  uniq -f3 --group=append | \
+  while read rev mod_rev repo pkg; do
+    if [ -z "${fs}" ] && \
+      [ -z "${rev}" ] && \
+      [ -z "${mod_rev}" ] && \
+      [ -z "${repo}" ] && \
+      [ -z "${pkg}" ]; then
+      printf '%s %s.%s.%s.%s\n' \
+        "${opkg}" \
+        "${opkg}" \
+        "$(
+          printf '%s\n' ${revs} | \
+            sort -u | \
+            find_newest_of_git_revisions
+        )" \
+        "$(
+          printf '%s\n' ${mod_revs} | \
+            sort -u | \
+            find_newest_of_git_revisions
+        )" \
+        "${orepo}"
+      revs=''
+      mod_revs=''
+      continue
+    fi
+    revs="$(
+      printf '%s\n' \
+        ${revs} \
+        "${rev}"
+    )"
+    mod_revs="$(
+      printf '%s\n' \
+        ${mod_revs} \
+        "${mod_rev}"
+    )"
+    opkg="${pkg}"
+    orepo="${repo}"
+  done | \
+  sort -k1,1 > \
+  "${work_dir}/newest-revisions"
+
+sed "
+  s|^\S\+ |${work_dir}/package-infos/|
+  s|\$|.builds|
+" "${work_dir}/newest-revisions" | \
+  xargs -r cat | \
+  sort | \
+  uniq -c > \
+  "${work_dir}/built-packages"
+
+sed "
+  s|^\S\+ |${work_dir}/package-infos/|
+  s|\$|.needs|
+" "${work_dir}/newest-revisions" | \
+  xargs -r grep -HF '' | \
+  sed '
+    s|\(\.[^.]\+\)\{4\}:\([^:]\+\)$| \2|
+    s|^.*/||
+  ' | \
+  sort -k2,2 > \
+  "${work_dir}/newest-dependencies"
+
 while [ -n "${black_listed_new}" ]; do
   black_listed="$(
-    printf '%s\n%s' "${black_listed}" "${black_listed_new}"
+    printf '%s\n' ${black_listed} ${black_listed_new} | \
+      sort -u
   )"
   black_listed_new="$(
-    echo "${black_listed_new}" | \
-      while read -r bl_package; do
-        ls "${work_dir}/package-infos" | \
-          grep '\.depends$' | \
-          sed "s|^|${work_dir}/package-infos/|" | \
-          xargs -r grep -lxF "${bl_package}" | \
-          sed '
-            s|^.*/||
-            s|\(\.[^.]\+\)\{3\}$||
-          '
-      done | \
+    (
+      printf '%s\n' ${black_listed} | \
+        sort -k1,1 | \
+        join -j1 -o 2.2 - "${work_dir}/newest-revisions" | \
+        sed "
+          s|^|${work_dir}/package-infos/|
+          s|\$|.builds|
+        " | \
+        xargs -r cat | \
+        sort | \
+        uniq -c
+      cat "${work_dir}/built-packages"
+    ) | \
+      sort | \
+      uniq -d | \
+      awk '{print $2}' | \
+      sort -k2,2 | \
+      join -1 1 -2 2 -o 2.1 - "${work_dir}/newest-dependencies" | \
       while read -r pkg; do
         if [ "${pkg}" = 'python-pysocks' ] || \
           [ "${pkg}" = 'gens' ]; then
-          >&2 printf 'blacklisting: %s would be black listed, but is ignored.' "${pkg}"
+          >&2 printf 'blacklisting: %s would be black listed, but is ignored.\n' "${pkg}"
         else
           echo "${pkg}"
         fi
@@ -266,7 +342,7 @@ while [ -n "${black_listed_new}" ]; do
       sort -u
   )"
   black_listed_new="$(
-    printf '%s\n' "${black_listed}" "${black_listed}" "${black_listed_new}" | \
+    printf '%s\n' ${black_listed} ${black_listed} ${black_listed_new} | \
       sort | \
       uniq -u
   )"
@@ -284,9 +360,9 @@ sort -u "${work_dir}/deletion-list.new" | \
 
 while read -r package git_revision mod_git_revision repository; do
   # add "$pkgname -> $build-target" to build-order list
-  sed "s|^|${package} |" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.builds"
+  sed "s|^|${package} |" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.builds"
   # add "$dependency -> $pkgname" to build-order list
-  sed "s|\$| ${package}|" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.needs"
+  sed "s|\$| ${package}|" "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.needs"
   # add "base/base-devel -> $pkgname" to build-order list
   printf "%s ${package}\n" 'base' 'base-devel'
 done \
diff --git a/bin/return-assignment b/bin/return-assignment
index 2020394..a057799 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -166,7 +166,7 @@ package_errors="$(
     sed '
       s|$| i686|
       s|^|expected: |
-    ' "${work_dir}/package-infos/$1.$2.$3.packages"
+    ' "${work_dir}/package-infos/$1.$2.$3.$4.packages"
   ) | \
     sort -k2 | \
     uniq -u -f1
diff --git a/bin/show-dependencies b/bin/show-dependencies
index e729f57..abd9ac3 100755
--- a/bin/show-dependencies
+++ b/bin/show-dependencies
@@ -53,7 +53,7 @@ sort -u "${work_dir}/build-order" | \
   sponge "${tmp_dir}/original-build-order"
 
 ls "${work_dir}/package-infos" | \
-  sed 's|\(\.[^.]\+\)\{3\}$||' | \
+  sed 's|\(\.[^.]\+\)\{4\}$||' | \
   sort -u > \
   "${tmp_dir}/known-packages"
 
@@ -67,8 +67,8 @@ ls "${work_dir}/package-infos" | \
 ls "${work_dir}/package-infos" | \
   grep '\.packages$' | \
   while read -r pf; do
-    grep -vxF "${pf%.*.*.*}" "${work_dir}/package-infos/${pf}" | \
-      xargs -r printf "${pf%.*.*.*} %s\n" >> \
+    grep -vxF "${pf%.*.*.*.*}" "${work_dir}/package-infos/${pf}" | \
+      xargs -r printf "${pf%.*.*.*.*} %s\n" >> \
       "${tmp_dir}/known-split-packages"
   done
 
diff --git a/bin/why_dont_you b/bin/why_dont_you
index aa4c4b0..443a803 100755
--- a/bin/why_dont_you
+++ b/bin/why_dont_you
@@ -26,11 +26,11 @@ case "${action}" in
             continue
           fi
 
-          if [ -n "$(
+          unmet_dependencies="$(
             (
-              cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.needs"
+              cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.needs"
               grep -vxF 'break_loops' "${work_dir}/build-list" | \
-                awk '{print $1 "." $2 "." $3}' | \
+                awk '{print $1 "." $2 "." $3 "." $4}' | \
                 sed "
                   s|^|${work_dir}/package-infos/|
                   s|\$|\.builds|
@@ -40,26 +40,15 @@ case "${action}" in
             ) | \
               sort | \
               uniq -d
-            )" ]; then
+          )"
+          if [ -n "${unmet_dependencies}" ]; then
             printf '"%s" has unmet dependencies:\n' "${package}"
-            (
-              cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.needs"
-              grep -vxF 'break_loops' "${work_dir}/build-list" | \
-                awk '{print $1 "." $2 "." $3}' | \
-                sed "
-                  s|^|${work_dir}/package-infos/|
-                  s|\$|\.builds|
-                " | \
-                xargs -r cat | \
-                sort -u
-            ) | \
-              sort | \
-              uniq -d | \
+            echo "${unmet_dependencies}" | \
               while read -r dep; do
                 grep -lxF "${dep}" "${work_dir}/package-infos/"*".builds" | \
                   sed '
                     s|^.*/||
-                    s|\(\.[^.]\+\)\{3\}||
+                    s|\(\.[^.]\+\)\{4\}||
                   '
               done | \
               sort -u
@@ -103,7 +92,7 @@ case "${action}" in
       dependent_packages_file="$(mktemp)"
 
       grep -vxF 'break_loops' "${work_dir}/build-list" | \
-        awk '{print $1 "." $2 "." $3 " " $4}' | \
+        awk '{print $1 "." $2 "." $3 "." $4}' | \
         sort -u > \
         "${build_list_items_file}"
 
@@ -112,7 +101,7 @@ case "${action}" in
         "${dependent_packages_file}"
 
       dependent_still_on_build_list="$(
-        join -1 1 -2 1 -o 1.1,1.2 \
+        join -1 1 -2 1 -o 1.1 \
           "${build_list_items_file}" \
           "${dependent_packages_file}"
       )"
@@ -131,7 +120,7 @@ case "${action}" in
       dependent_still_in_staging="$(
         echo "${dependent_packages}" | \
           while read -r sf; do
-            if [ -f "${work_dir}/package-states/$sf."*".done" ]; then
+            if [ -f "${work_dir}/package-states/${sf}.done" ]; then
               echo "${sf}"
             fi
           done
@@ -159,9 +148,11 @@ case "${action}" in
 
       if builds_file="$(
         ls "${work_dir}/package-infos" | \
-          grep -m1 "^$(str_to_regex "${pkg}")\.[^.]\+\.[^.]\+\.builds\$"
+          grep -m1 "^$(str_to_regex "${pkg}")\(\.[^.]\+\)\{3\}\.builds\$"
         )"; then
 
+        builds_file="${builds_file%.*}"
+        prepo="${builds_file##*.}"
         builds_file="${builds_file%.*}"
         mod_rev="${builds_file##*.}"
         builds_file="${builds_file%.*}"
@@ -211,7 +202,7 @@ case "${action}" in
 
       fi
 
-      sed "s|^|${pkg} builds |" "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.builds" >> \
+      sed "s|^|${pkg} builds |" "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${prepo}.builds" >> \
         "${tmp_file}"
 
     done < \
-- 
cgit v1.2.3-54-g00ecf