[archiso] Use dm-snapshot instead of aufs2 (A.K.A. "The Big Commit")

* Use device mapper + snapshot module, instead union layer filesystem.
  * A block-level approach vs vfs-level.
  * No more unofficial (Linux) things.
  * More memory is needed.
* Refactor mkarchiso.
* Refactor hooks/archiso.
* Fix install/archiso_pxe_nbd
  (due recent change in mkinitcpio-0.6.15 on checked_modules()/all_modules())
  [Thanks Dave for the improved workaround]
* New configs/releng to build official images.
  * Works with a Bash script instead of Makefile.
    (better control and easy to maintain)
* Remove configs/syslinux-iso.
* Remove archiso2dual script. Integrate functionality in configs/releng.
* New configs/baseline to build the most basic live medium or use as template.
* New README (draft). [Thanks Dieter for fixing english grammar]

Signed-off-by: Gerardo Exequiel Pozzi <vmlinuz386@yahoo.com.ar>

4 files changed, 138 insertions, 116 deletions

diff --git a/archiso/Makefile b/archiso/Makefile
index 93d68af..74f9ff2 100644
--- a/archiso/Makefile
+++ b/archiso/Makefile
@@ -1,6 +1,8 @@
 all:
 
-install: all
+install: install-program install-examples install-doc
+
+install-program:
 	# install to sbin since script only usable by root
 	install -D -m 755 mkarchiso $(DESTDIR)/usr/sbin/mkarchiso
 	# testiso can be used by anyone
@@ -12,11 +14,16 @@ install: all
 	install -D -m 644 install/archiso_pxe_nbd $(DESTDIR)/lib/initcpio/install/archiso_pxe_nbd
 	install -D -m 644 hooks/archiso_loop_mnt $(DESTDIR)/lib/initcpio/hooks/archiso_loop_mnt
 	install -D -m 644 install/archiso_loop_mnt $(DESTDIR)/lib/initcpio/install/archiso_loop_mnt
-	# install docs and examples
+
+install-examples:
+	# install examples
 	install -d -m 755 $(DESTDIR)/usr/share/archiso/
 	cp -r ../configs $(DESTDIR)/usr/share/archiso/configs
-	install -D -m 644 ../README $(DESTDIR)/usr/share/doc/archiso/README
 
+install-doc:
+	install -d -m 755 $(DESTDIR)/usr/share/archiso/
+	install -D -m 644 ../README $(DESTDIR)/usr/share/doc/archiso/README
+	
 uninstall:
 	rm -f $(DESTDIR)/usr/sbin/mkarchiso
 	rm -f $(DESTDIR)/usr/bin/testiso
@@ -25,4 +32,5 @@ uninstall:
 	rm -f $(DESTDIR)/lib/initcpio/hooks/archiso_pxe_nbd
 	rm -f $(DESTDIR)/lib/initcpio/install/archiso_pxe_nbd
 	rm -rf $(DESTDIR)/usr/share/archiso/
-	rm -rf $(DESTDIR)/usr/share/doc/archiso/
+
+.PHONY: install install-program install-examples install-doc uninstall
diff --git a/archiso/hooks/archiso b/archiso/hooks/archiso
index a129b20..b81bbf7 100644
--- a/archiso/hooks/archiso
+++ b/archiso/hooks/archiso
@@ -1,158 +1,165 @@
-# args: source, mountpoint
-_mnt_aufs() {
-    src="${1}"
-    mnt="${2}"
-    msg "::: Adding new aufs branch: ${src} to ${mnt}"
-    mkdir -p "${mnt}"
-    /bin/mount -t aufs -o remount,append:"${src}"=ro none "${mnt}"
+# Initialize loopback device logic (we using on-demand mode)
+# args: none
+_init_loop_dev() {
+    loop_dev_cnt=99
+}
+
+# Call this function before _make_loop_dev() each time.
+# args: none
+_next_loop_dev() {
+    loop_dev_cnt=$((loop_dev_cnt+1))
+}
+
+# Setup a loopback device for image passed as arguemnt and echo the path to loopback device used.
+# args: /path/to/image_file
+_make_loop_dev() {
+    local img="${1}"
+    mknod /dev/loop${loop_dev_cnt} b 7 ${loop_dev_cnt} &> /dev/null
+    losetup /dev/loop${loop_dev_cnt} "${img}" &> /dev/null
+    echo /dev/loop${loop_dev_cnt}
 }
 
 # args: source, mountpoint
-_mnt_bind() {
-    src="${1}"
-    mnt="${2}"
-    msg "::: Binding ${src} to ${mnt}"
+_mnt_fs() {
+    local img="${1}"
+    local mnt="${2}"
+    local img_fullname="${img##*/}";
+    local img_name="${img_fullname%%.*}"
+    local ro_dev ro_dev_size ro_dev_fs_type rw_dev
+
     mkdir -p "${mnt}"
-    /bin/mount -o bind "${src}" "${mnt}"
+
+    _next_loop_dev
+    ro_dev=$(_make_loop_dev "${img}")
+    ro_dev_size=$(blockdev --getsz ${ro_dev})
+    ro_dev_fs_type=$(blkid -o value -s TYPE -p ${ro_dev} 2> /dev/null)
+
+    dd of="/cowspace/${img_name}.cow" count=0 seek=${ro_dev_size} &> /dev/null
+    _next_loop_dev
+    rw_dev=$(_make_loop_dev "/cowspace/${img_name}.cow")
+
+    echo "0 ${ro_dev_size} snapshot ${ro_dev} ${rw_dev} N 8" | dmsetup create ${img_name}
+
+    msg ":: Mounting '/dev/mapper/${img_name}' (${ro_dev_fs_type}) to '${mnt}'"
+    if ! mount -t "${ro_dev_fs_type}" "/dev/mapper/${img_name}" "${mnt}" ; then
+        echo "ERROR: while mounting '/dev/mapper/${img_name}' to '${mnt}'"
+        launch_interactive_shell
+    fi
 }
 
 # args: /path/to/image_file, mountpoint
-_mnt_squashfs() {
-    img="${1}"
-    mnt="${2}"
-    img_fullname="${img##*/}";
-    img_name="${img_fullname%.*}"
-    tmp_mnt="/ro_branch/${img_name}"
-
-    if [ "${copytoram}" = "y" ]; then
+_mnt_sfs() {
+    local img="${1}"
+    local mnt="${2}"
+    local img_fullname="${img##*/}";
+
+    mkdir -p "${mnt}"
+
+    if [[ "${copytoram}" == "y" ]]; then
         msg -n ":: Copying squashfs image to RAM..."
-        /bin/cp "${img}" "/copytoram/${img_fullname}"
-        if [ $? -ne 0 ]; then
-            echo "ERROR: while copy ${img} to /copytoram/${img_fullname}"
+        if ! cp "${img}" "/copytoram/${img_fullname}" ; then
+            echo "ERROR: while copy '${img}' to '/copytoram/${img_fullname}'"
             launch_interactive_shell
         fi
         img="/copytoram/${img_fullname}"
         msg "done."
     fi
-
-    mkdir -p "${tmp_mnt}"
-    /bin/mount -r -t squashfs "${img}" "${tmp_mnt}"
-    if [ $? -ne 0 ]; then
-        echo "ERROR: while mounting ${img} to ${tmp_mnt}"
+    _next_loop_dev
+    msg ":: Mounting '${img}' (SquashFS) to '${mnt}'"
+    if ! mount -r -t squashfs $(_make_loop_dev "${img}") "${mnt}" &> /dev/null ; then
+        echo "ERROR: while mounting '${img}' to '${mnt}'"
         launch_interactive_shell
     fi
-
-    if [ "/${mnt#/*/}" = "/" ]; then
-        _mnt_aufs "${tmp_mnt}" "${mnt}"
-    else
-        _mnt_bind "${tmp_mnt}" "${mnt}"
-    fi
 }
 
 run_hook() {
-    if [ "x${arch}" = "x" ]; then
-        arch="$(uname -m)"
-    fi
-
-    if [ "x${rw_branch_size}" = "x" ]; then
-        rw_branch_size="75%"
-    fi
-
-    if [ "x${copytoram_size}" = "x" ]; then
-        copytoram_size="75%"
-    fi
-
-    if [ "x${archisobasedir}" = "x" ]; then
-        archisobasedir="arch"
-    fi
-
-    if [ "x${isomounts}" != "x" ]; then
-        isomounts="/bootmnt/${isomounts}"
+    [[ -z "${arch}" ]] && arch="$(uname -m)"
+    [[ -z "${cowspace_size}" ]] && cowspace_size="75%"
+    [[ -z "${copytoram_size}" ]] && copytoram_size="75%"
+    [[ -z "${archisobasedir}" ]] && archisobasedir="arch"
+    [[ -z "${archisodevice}" ]] && archisodevice="/dev/disk/by-label/${archisolabel}"
+    if [[ -z "${aitab}" ]]; then
+        aitab="/bootmnt/${archisobasedir}/aitab"
     else
-        isomounts="/bootmnt/${archisobasedir}/isomounts"
+        aitab="/bootmnt/${aitab}"
     fi
-
-    if [ "x${archisodevice}" = "x" ]; then
-        archisodevice="/dev/disk/by-label/${archisolabel}"
-    fi
-
     # set mount handler for archiso
     mount_handler="archiso_mount_handler"
 }
 
+# This function is called normally from init script, but it can be called
+# as chain from other mount handlers.
+# args: /path/to/newroot
 archiso_mount_handler() {
-    newroot="${1}"
+    local newroot="${1}"
+    local fstype fserror
+
+    _init_loop_dev
 
     msg ":: Waiting for boot device..."
-    while ! poll_device ${archisodevice} 30; do
+    while ! poll_device "${archisodevice}" 30; do
         echo "ERROR: boot device didn't show up after 30 seconds..."
         echo "   Falling back to interactive prompt"
         echo "   You can try to fix the problem manually, log out when you are finished"
         launch_interactive_shell
     done
 
-    FSTYPE=$(blkid -o value -s TYPE -p ${archisodevice} 2> /dev/null)
-    if [ -n "${FSTYPE}" ]; then
-        if mount -r -t "${FSTYPE}" ${archisodevice} /bootmnt > /dev/null 2>&1; then
-            if [ -e "${isomounts}" ]; then
-                echo "SUCCESS: Mounted archiso volume successfully."
-                fserror="0"
+    fstype=$(blkid -o value -s TYPE -p "${archisodevice}" 2> /dev/null)
+    if [[ -n "${fstype}" ]]; then
+        if mount -r -t "${fstype}" "${archisodevice}" /bootmnt; then
+            if [[ -f "${aitab}" ]]; then
+                msg ":: Mounted archiso volume successfully."
+                fserror=0
             else
-                echo "ERROR: Mounting was successful, but the ${isomounts} file does not exist."
-                fserror="1"
+                echo "ERROR: Mounting was successful, but the '${aitab}' file does not exist."
+                fserror=1
             fi
         else
-            echo "ERROR; Failed to mount ${archisodevice} (FS is ${FSTYPE})"
-            fserror="1"
+            echo "ERROR; Failed to mount '${archisodevice}' (FS is ${fstype})"
+            fserror=1
         fi
     else
-        echo "ERROR: ${archisodevice} found, but the filesystem type is unknown."
-        fserror="1"
+        echo "ERROR: '${archisodevice}' found, but the filesystem type is unknown."
+        fserror=1
     fi
 
-    if [ "${fserror}" = "1" ]; then
+    if [[ ${fserror} -eq 1 ]]; then
         echo "   Falling back to interactive prompt"
         echo "   You can try to fix the problem manually, log out when you are finished"
         launch_interactive_shell
     fi
 
-    if [ "${copytoram}" = "y" ]; then
+    if [[ "${copytoram}" == "y" ]]; then
         msg -n ":: Mounting /copytoram (tmpfs) filesystem, size=${copytoram_size}..."
         mount -t tmpfs -o "size=${copytoram_size}",mode=0755 copytoram /copytoram
         msg "done."
     fi
 
-    msg -n ":: Mounting rw_branch (tmpfs) filesystem, size=${rw_branch_size}..."
-    mount -t tmpfs -o "size=${rw_branch_size}",mode=0755 rw_branch /rw_branch
+    msg -n ":: Mounting /cowspace (tmpfs) filesystem, size=${cowspace_size}..."
+    mount -t tmpfs -o "size=${cowspace_size}",mode=0755 cowspace /cowspace
     msg "done."
 
-    msg ":: Mounting root (aufs) filesystem"
-    /bin/mount -t aufs -o dirs=/rw_branch=rw union "${newroot}"
-    if [ $? -ne 0 ]; then
-        echo "ERROR: while mounting root (aufs) filesystem."
-        launch_interactive_shell
-    fi
-
-    msg ":: Mounting images"
-    while read img imgarch mountpoint type; do
-        # check if this line is a comment (starts with #)
-        [ "${img#"#"}" != "${img}" ] && continue
-
-        [ "$imgarch" != "$arch" ] && continue
-
-        [ ! -r "/bootmnt/${archisobasedir}/${img}" ] && continue
-
-        if [ "${type}" = "bind" ]; then
-            _mnt_bind "/bootmnt/${archisobasedir}/${img}" "${newroot}${mountpoint}"
-        elif [ "${type}" = "squashfs" ]; then
-            _mnt_squashfs "/bootmnt/${archisobasedir}/${img}" "${newroot}${mountpoint}"
+    local aitab_img aitab_mnt aitab_arch aitab_sfs_comp aitab_fs_type aitab_fs_size
+    while read aitab_img aitab_mnt aitab_arch aitab_sfs_comp aitab_fs_type aitab_fs_size; do
+        [[ "${aitab_img#\#}" != "${aitab_img}" ]] && continue
+        [[ "${aitab_arch}" != "any" && "${aitab_arch}" != "${arch}" ]] && continue
+        if [[ "${aitab_fs_type}" != "none" ]]; then
+            if [[ "${aitab_sfs_comp}" != "none" ]]; then
+                _mnt_sfs "/bootmnt/${archisobasedir}/${aitab_arch}/${aitab_img}.fs.sfs" "/sfs/${aitab_img}"
+                _mnt_fs "/sfs/${aitab_img}/${aitab_img}.fs" "${newroot}${aitab_mnt}"
+            else
+                _mnt_fs "/bootmnt/${archisobasedir}/${aitab_arch}/${aitab_img}.fs" "${newroot}${aitab_mnt}"
+            fi
+        else
+            _mnt_sfs "/bootmnt/${archisobasedir}/${aitab_arch}/${aitab_img}.sfs" "${newroot}${aitab_mnt}"
         fi
-    done < "${isomounts}"
+    done < "${aitab}"
 
-    if [ "${copytoram}" = "y" ]; then
-        /bin/umount /bootmnt
+    if [[ "${copytoram}" == "y" ]]; then
+        umount /bootmnt
     else
-        _mnt_bind /bootmnt "${newroot}/bootmnt"
+        mkdir "${newroot}/bootmnt"
+        mount --bind /bootmnt "${newroot}/bootmnt"
     fi
 }
 
diff --git a/archiso/install/archiso b/archiso/install/archiso
index 7ad9d06..3ac6246 100644
--- a/archiso/install/archiso
+++ b/archiso/install/archiso
@@ -1,25 +1,28 @@
 install ()
 {
-    MODULES="cdrom ide-cd_mod ide-core ide-generic aufs squashfs isofs loop $(all_modules '/kernel/fs' | grep -v "nls") "
-    MODULES="${MODULES} $(checked_modules "/usb/host" | grep -ve "_cs" -e "sl811-hcd" -e "isp116x-hcd")"
-    MODULES=$(echo ${MODULES}) #trim whitespace
-    if [ "x${MODULES}" != "x" ]; then
-        MODULES="${MODULES} usb_storage sd_mod sr_mod"
-    fi
+    MODULES="cdrom ide-cd_mod ide-core ide-generic loop dm-mod dm-snapshot squashfs isofs $(all_modules '/kernel/fs' | grep -v "nls")"
+    MODULES="${MODULES} $(checked_modules "/usb/host" | grep -ve "_cs" -e "sl811_hcd" -e "isp116x_hcd")"
+    MODULES="${MODULES} usb_storage sd_mod sr_mod"
     MODULES="${MODULES} virtio_pci virtio_blk"
+    MODULES=$(echo ${MODULES}) #trim whitespace
 
     BINARIES=""
     FILES=""
-    add_dir /rw_branch
-    add_dir /ro_branch
+
+    add_dir /cowspace
     add_dir /copytoram
     add_dir /bootmnt
 
-    add_device /lib/udev/devices/loop0 b 7 0
-
     add_binary /lib/udev/cdrom_id
+    add_binary /sbin/blockdev
+    add_binary /sbin/lvm
+    add_binary /sbin/dmsetup
 
     add_file /lib/udev/rules.d/60-cdrom_id.rules
+    add_file /lib/udev/rules.d/10-dm.rules
+    add_file /lib/udev/rules.d/13-dm-disk.rules
+    add_file /lib/udev/rules.d/95-dm-notify.rules
+    add_file /lib/udev/rules.d/11-dm-lvm.rules
 
     SCRIPT="archiso"
 }
diff --git a/archiso/install/archiso_pxe_nbd b/archiso/install/archiso_pxe_nbd
index fbf43c6..189a7b1 100644
--- a/archiso/install/archiso_pxe_nbd
+++ b/archiso/install/archiso_pxe_nbd
@@ -2,8 +2,12 @@
 
 install ()
 {
-    MODULES="nbd $(checked_modules "/drivers/net/" | grep -v -e "/irda/" -e "/phy/" -e "/plip" -e "/ppp" -e "/wimax/" -e "/wireless/") "
-
+    MODULES="nbd"
+    MODULES="${MODULES} $(comm -2 -3 <(checked_modules "/drivers/net/" | sort) \
+                                     <(find $MODULEDIR/kernel/drivers/net/{irda,phy,wimax,wireless} \
+                                            -name '*.ko*' \
+                                            -exec bash -c 'printf "%s\n" "${@%%.ko*}" | sed "s@.*/@@;s@-@_@" | sort' _ {} +) \
+                                     | grep -v -e 'ppp_' -e 'plip' -e 'pppoe')"
     BINARIES=""
     FILES=""
     SCRIPT="archiso_pxe_nbd"