From 80cee500e0ef0cf4de84b8b60b35c25f667e7a34 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Sun, 8 May 2022 18:36:32 +0200 Subject: SysCommand now sets working_directory on SysCommandWorker. Also made it so the parent process moves back to the original working directory, leaving the child process in the target working directory. (#1142) --- archinstall/lib/general.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'archinstall/lib/general.py') diff --git a/archinstall/lib/general.py b/archinstall/lib/general.py index 174acb8a..cf925de3 100644 --- a/archinstall/lib/general.py +++ b/archinstall/lib/general.py @@ -352,7 +352,6 @@ class SysCommandWorker: # only way to get the traceback without loosing it. self.pid, self.child_fd = pty.fork() - os.chdir(old_dir) # https://stackoverflow.com/questions/4022600/python-pty-fork-how-does-it-work if not self.pid: @@ -371,6 +370,9 @@ class SysCommandWorker: log(f"{self.cmd[0]} does not exist.", level=logging.ERROR, fg="red") self.exit_code = 1 return False + else: + # Only parent process moves back to the original working directory + os.chdir(old_dir) self.started = time.time() self.poll_object.register(self.child_fd, EPOLLIN | EPOLLHUP) @@ -457,7 +459,14 @@ class SysCommand: if self.session: return self.session - with SysCommandWorker(self.cmd, callbacks=self._callbacks, peak_output=self.peak_output, environment_vars=self.environment_vars, remove_vt100_escape_codes_from_lines=self.remove_vt100_escape_codes_from_lines) as session: + with SysCommandWorker( + self.cmd, + callbacks=self._callbacks, + peak_output=self.peak_output, + environment_vars=self.environment_vars, + remove_vt100_escape_codes_from_lines=self.remove_vt100_escape_codes_from_lines, + working_directory=self.working_directory) as session: + if not self.session: self.session = session -- cgit v1.2.3-70-g09d2 From 81b7524b53fe21664ec6da3b69e507638b1ed253 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Thu, 12 May 2022 10:46:33 +0200 Subject: Made sure remote sourcing works for --creds, --config and --disk-layout. (#1170) * Made sure remote sourcing works for --creds, --config and --disk-layout. * Spelling error when moving around source code. --- archinstall/__init__.py | 21 ++++++-------------- archinstall/lib/general.py | 49 ++++++++++++++++++++++++++++------------------ 2 files changed, 36 insertions(+), 34 deletions(-) (limited to 'archinstall/lib/general.py') diff --git a/archinstall/__init__.py b/archinstall/__init__.py index 536db370..aa644d48 100644 --- a/archinstall/__init__.py +++ b/archinstall/__init__.py @@ -149,22 +149,13 @@ def get_arguments() -> Dict[str, Any]: # preprocess the json files. # TODO Expand the url access to the other JSON file arguments ? if args.config is not None: - try: - # First, let's check if this is a URL scheme instead of a filename - parsed_url = urllib.parse.urlparse(args.config) - - if not parsed_url.scheme: # The Profile was not a direct match on a remote URL, it must be a local file. - if not json_stream_to_structure('--config',args.config,config): - exit(1) - else: # Attempt to load the configuration from the URL. - with urllib.request.urlopen(urllib.request.Request(args.config, headers={'User-Agent': 'ArchInstall'})) as response: - config.update(json.loads(response.read())) - except Exception as e: - raise ValueError(f"Could not load --config because: {e}") + if not json_stream_to_structure('--config', args.config, config): + exit(1) - if args.creds is not None: - if not json_stream_to_structure('--creds',args.creds,config): - exit(1) + if args.creds is not None: + if not json_stream_to_structure('--creds', args.creds, config): + exit(1) + # load the parameters. first the known, then the unknowns config.update(vars(args)) config.update(parse_unspecified_argument_list(unknowns)) diff --git a/archinstall/lib/general.py b/archinstall/lib/general.py index cf925de3..a4e2a365 100644 --- a/archinstall/lib/general.py +++ b/archinstall/lib/general.py @@ -10,6 +10,9 @@ import string import sys import time import re +import urllib.parse +import urllib.request +import pathlib from datetime import datetime, date from typing import Callable, Optional, Dict, Any, List, Union, Iterator, TYPE_CHECKING # https://stackoverflow.com/a/39757388/929999 @@ -532,32 +535,40 @@ def run_custom_user_commands(commands :List[str], installation :Installer) -> No log(execution_output) os.unlink(f"{installation.target}/var/tmp/user-command.{index}.sh") -def json_stream_to_structure(id : str, stream :str, target :dict) -> bool : - """ Function to load a stream (file (as name) or valid JSON string into an existing dictionary +def json_stream_to_structure(configuration_identifier : str, stream :str, target :dict) -> bool : + """ + Function to load a stream (file (as name) or valid JSON string into an existing dictionary Returns true if it could be done Return false if operation could not be executed - +id is just a parameter to get meaningful, but not so long messages + +configuration_identifier is just a parameter to get meaningful, but not so long messages """ - from pathlib import Path - if Path(stream).exists(): - try: - with open(Path(stream)) as fh: - target.update(json.load(fh)) - except Exception as e: - log(f"{id} = {stream} does not contain a valid JSON format: {e}",level=logging.ERROR) - return False + + parsed_url = urllib.parse.urlparse(stream) + + if parsed_url.scheme: # The stream is in fact a URL that should be grabed + with urllib.request.urlopen(urllib.request.Request(stream, headers={'User-Agent': 'ArchInstall'})) as response: + target.update(json.loads(response.read())) else: - log(f"{id} = {stream} does not exists in the filesystem. Trying as JSON stream",level=logging.DEBUG) - # NOTE: failure of this check doesn't make stream 'real' invalid JSON, just it first level entry is not an object (i.e. dict), so it is not a format we handle. - if stream.strip().startswith('{') and stream.strip().endswith('}'): + if pathlib.Path(stream).exists(): try: - target.update(json.loads(stream)) - except Exception as e: - log(f" {id} Contains an invalid JSON format : {e}",level=logging.ERROR) + with pathlib.Path(stream).open() as fh: + target.update(json.load(fh)) + except Exception as error: + log(f"{configuration_identifier} = {stream} does not contain a valid JSON format: {error}", level=logging.ERROR, fg="red") return False else: - log(f" {id} is neither a file nor is a JSON string:",level=logging.ERROR) - return False + # NOTE: This is a rudimentary check if what we're trying parse is a dict structure. + # Which is the only structure we tolerate anyway. + if stream.strip().startswith('{') and stream.strip().endswith('}'): + try: + target.update(json.loads(stream)) + except Exception as e: + log(f" {configuration_identifier} Contains an invalid JSON format : {e}",level=logging.ERROR, fg="red") + return False + else: + log(f" {configuration_identifier} is neither a file nor is a JSON string:",level=logging.ERROR, fg="red") + return False + return True def secret(x :str): -- cgit v1.2.3-70-g09d2 From 493cccc18fa8c77c362b6abee2c3dc89d331c792 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Wed, 18 May 2022 11:28:59 +0200 Subject: Added a HSM menu entry (#1196) * Added a HSM menu entry, but also a safety check to make sure a FIDO device is connected * flake8 complaints * Adding FIDO lookup using cryptenroll listing * Added systemd-cryptenroll --fido2-device=list * Removed old _select_hsm call * Fixed flake8 complaints * Added support for locking and unlocking with a HSM * Removed hardcoded paths in favor of PR merge * Removed hardcoded paths in favor of PR merge * Fixed mypy complaint * Flake8 issue * Added sd-encrypt for HSM and revert back to encrypt when HSM is not used (stability reason) * Added /etc/vconsole.conf and tweaked fido2_enroll() to use the proper paths * Spelling error * Using UUID instead of PARTUUID when using HSM. I can't figure out how to get sd-encrypt to use PARTUUID instead. Added a Partition().part_uuid function. Actually renamed .uuid to .part_uuid and created a .uuid instead. * Adding missing package libfido2 and removed tpm2-device=auto as it overrides everything and forces password prompt to be used over FIDO2, no matter the order of the options. * Added some notes to clarify some choices. * Had to move libfido2 package install to later in the chain, as there's not even a base during mounting :P --- .flake8 | 2 +- archinstall/__init__.py | 5 +++ archinstall/lib/configuration.py | 27 ++++++++---- archinstall/lib/disk/blockdevice.py | 2 +- archinstall/lib/disk/filesystem.py | 8 ++-- archinstall/lib/disk/partition.py | 44 ++++++++++++++++++- archinstall/lib/general.py | 2 + archinstall/lib/hsm/__init__.py | 4 ++ archinstall/lib/hsm/fido.py | 47 ++++++++++++++++++++ archinstall/lib/installer.py | 67 +++++++++++++++++++++++------ archinstall/lib/menu/global_menu.py | 6 +++ archinstall/lib/menu/selection_menu.py | 24 +++++++++++ archinstall/lib/udev/__init__.py | 1 + archinstall/lib/udev/udevadm.py | 17 ++++++++ archinstall/locales/en/LC_MESSAGES/base.mo | Bin 148 -> 261 bytes archinstall/locales/en/LC_MESSAGES/base.po | 7 +++ examples/guided.py | 6 +++ 17 files changed, 242 insertions(+), 27 deletions(-) create mode 100644 archinstall/lib/hsm/__init__.py create mode 100644 archinstall/lib/hsm/fido.py create mode 100644 archinstall/lib/udev/__init__.py create mode 100644 archinstall/lib/udev/udevadm.py (limited to 'archinstall/lib/general.py') diff --git a/.flake8 b/.flake8 index 39310a6c..d69ec92e 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,7 @@ [flake8] count = True # Several of the following could be autofixed or improved by running the code through psf/black -ignore = E123,E126,E128,E203,E231,E261,E302,E402,E722,F541,W191,W292,W293 +ignore = E123,E126,E128,E203,E231,E261,E302,E402,E722,F541,W191,W292,W293,W503 max-complexity = 40 max-line-length = 236 show-source = True diff --git a/archinstall/__init__.py b/archinstall/__init__.py index aa644d48..da3deb35 100644 --- a/archinstall/__init__.py +++ b/archinstall/__init__.py @@ -45,6 +45,11 @@ from .lib.menu.selection_menu import ( from .lib.translation import Translation, DeferredTranslation from .lib.plugins import plugins, load_plugin # This initiates the plugin loading ceremony from .lib.configuration import * +from .lib.udev import udevadm_info +from .lib.hsm import ( + get_fido2_devices, + fido2_enroll +) parser = ArgumentParser() __version__ = "2.4.2" diff --git a/archinstall/lib/configuration.py b/archinstall/lib/configuration.py index c971768f..f3fe1e1c 100644 --- a/archinstall/lib/configuration.py +++ b/archinstall/lib/configuration.py @@ -1,12 +1,23 @@ import json import logging -from pathlib import Path +import pathlib from typing import Optional, Dict from .storage import storage from .general import JSON, UNSAFE_JSON from .output import log - +from .exceptions import RequirementError +from .hsm import get_fido2_devices + +def configuration_sanity_check(): + if storage['arguments'].get('HSM'): + if not get_fido2_devices(): + raise RequirementError( + f"In order to use HSM to pair with the disk encryption," + + f" one needs to be accessible through /dev/hidraw* and support" + + f" the FIDO2 protocol. You can check this by running" + + f" 'systemd-cryptenroll --fido2-device=list'." + ) class ConfigurationOutput: def __init__(self, config: Dict): @@ -21,7 +32,7 @@ class ConfigurationOutput: self._user_credentials = {} self._disk_layout = None self._user_config = {} - self._default_save_path = Path(storage.get('LOG_PATH', '.')) + self._default_save_path = pathlib.Path(storage.get('LOG_PATH', '.')) self._user_config_file = 'user_configuration.json' self._user_creds_file = "user_credentials.json" self._disk_layout_file = "user_disk_layout.json" @@ -84,7 +95,7 @@ class ConfigurationOutput: print() - def _is_valid_path(self, dest_path :Path) -> bool: + def _is_valid_path(self, dest_path :pathlib.Path) -> bool: if (not dest_path.exists()) or not (dest_path.is_dir()): log( 'Destination directory {} does not exist or is not a directory,\n Configuration files can not be saved'.format(dest_path.resolve()), @@ -93,26 +104,26 @@ class ConfigurationOutput: return False return True - def save_user_config(self, dest_path :Path = None): + def save_user_config(self, dest_path :pathlib.Path = None): if self._is_valid_path(dest_path): with open(dest_path / self._user_config_file, 'w') as config_file: config_file.write(self.user_config_to_json()) - def save_user_creds(self, dest_path :Path = None): + def save_user_creds(self, dest_path :pathlib.Path = None): if self._is_valid_path(dest_path): if user_creds := self.user_credentials_to_json(): target = dest_path / self._user_creds_file with open(target, 'w') as config_file: config_file.write(user_creds) - def save_disk_layout(self, dest_path :Path = None): + def save_disk_layout(self, dest_path :pathlib.Path = None): if self._is_valid_path(dest_path): if disk_layout := self.disk_layout_to_json(): target = dest_path / self._disk_layout_file with target.open('w') as config_file: config_file.write(disk_layout) - def save(self, dest_path :Path = None): + def save(self, dest_path :pathlib.Path = None): if not dest_path: dest_path = self._default_save_path diff --git a/archinstall/lib/disk/blockdevice.py b/archinstall/lib/disk/blockdevice.py index 4978f19c..995ca355 100644 --- a/archinstall/lib/disk/blockdevice.py +++ b/archinstall/lib/disk/blockdevice.py @@ -275,7 +275,7 @@ class BlockDevice: count = 0 while count < 5: for partition_uuid, partition in self.partitions.items(): - if partition.uuid.lower() == uuid.lower(): + if partition.part_uuid.lower() == uuid.lower(): return partition else: log(f"uuid {uuid} not found. Waiting for {count +1} time",level=logging.DEBUG) diff --git a/archinstall/lib/disk/filesystem.py b/archinstall/lib/disk/filesystem.py index db97924f..31929b63 100644 --- a/archinstall/lib/disk/filesystem.py +++ b/archinstall/lib/disk/filesystem.py @@ -150,7 +150,7 @@ class Filesystem: if partition.get('boot', False): log(f"Marking partition {partition['device_instance']} as bootable.") - self.set(self.partuuid_to_index(partition['device_instance'].uuid), 'boot on') + self.set(self.partuuid_to_index(partition['device_instance'].part_uuid), 'boot on') prev_partition = partition @@ -193,7 +193,7 @@ class Filesystem: def add_partition(self, partition_type :str, start :str, end :str, partition_format :Optional[str] = None) -> Partition: log(f'Adding partition to {self.blockdevice}, {start}->{end}', level=logging.INFO) - previous_partition_uuids = {partition.uuid for partition in self.blockdevice.partitions.values()} + previous_partition_uuids = {partition.part_uuid for partition in self.blockdevice.partitions.values()} if self.mode == MBR: if len(self.blockdevice.partitions) > 3: @@ -210,7 +210,7 @@ class Filesystem: count = 0 while count < 10: new_uuid = None - new_uuid_set = (previous_partition_uuids ^ {partition.uuid for partition in self.blockdevice.partitions.values()}) + new_uuid_set = (previous_partition_uuids ^ {partition.part_uuid for partition in self.blockdevice.partitions.values()}) if len(new_uuid_set) > 0: new_uuid = new_uuid_set.pop() @@ -236,7 +236,7 @@ class Filesystem: # TODO: This should never be able to happen log(f"Could not find the new PARTUUID after adding the partition.", level=logging.ERROR, fg="red") log(f"Previous partitions: {previous_partition_uuids}", level=logging.ERROR, fg="red") - log(f"New partitions: {(previous_partition_uuids ^ {partition.uuid for partition in self.blockdevice.partitions.values()})}", level=logging.ERROR, fg="red") + log(f"New partitions: {(previous_partition_uuids ^ {partition.part_uuid for partition in self.blockdevice.partitions.values()})}", level=logging.ERROR, fg="red") raise DiskError(f"Could not add partition using: {parted_string}") def set_name(self, partition: int, name: str) -> bool: diff --git a/archinstall/lib/disk/partition.py b/archinstall/lib/disk/partition.py index e7568258..c52ca434 100644 --- a/archinstall/lib/disk/partition.py +++ b/archinstall/lib/disk/partition.py @@ -184,7 +184,7 @@ class Partition: return device['pttype'] @property - def uuid(self) -> Optional[str]: + def part_uuid(self) -> Optional[str]: """ Returns the PARTUUID as returned by lsblk. This is more reliable than relying on /dev/disk/by-partuuid as @@ -197,6 +197,26 @@ class Partition: time.sleep(max(0.1, storage['DISK_TIMEOUTS'] * i)) + partuuid = self._safe_part_uuid + if partuuid: + return partuuid + + raise DiskError(f"Could not get PARTUUID for {self.path} using 'blkid -s PARTUUID -o value {self.path}'") + + @property + def uuid(self) -> Optional[str]: + """ + Returns the UUID as returned by lsblk for the **partition**. + This is more reliable than relying on /dev/disk/by-uuid as + it doesn't seam to be able to detect md raid partitions. + For bind mounts all the subvolumes share the same uuid + """ + for i in range(storage['DISK_RETRY_ATTEMPTS']): + if not self.partprobe(): + raise DiskError(f"Could not perform partprobe on {self.device_path}") + + time.sleep(max(0.1, storage['DISK_TIMEOUTS'] * i)) + partuuid = self._safe_uuid if partuuid: return partuuid @@ -216,6 +236,28 @@ class Partition: log(f"Could not reliably refresh PARTUUID of partition {self.device_path} due to partprobe error.", level=logging.DEBUG) + try: + return SysCommand(f'blkid -s UUID -o value {self.device_path}').decode('UTF-8').strip() + except SysCallError as error: + if self.block_device.info.get('TYPE') == 'iso9660': + # Parent device is a Optical Disk (.iso dd'ed onto a device for instance) + return None + + log(f"Could not get PARTUUID of partition using 'blkid -s UUID -o value {self.device_path}': {error}") + + @property + def _safe_part_uuid(self) -> Optional[str]: + """ + A near copy of self.uuid but without any delays. + This function should only be used where uuid is not crucial. + For instance when you want to get a __repr__ of the class. + """ + if not self.partprobe(): + if self.block_device.info.get('TYPE') == 'iso9660': + return None + + log(f"Could not reliably refresh PARTUUID of partition {self.device_path} due to partprobe error.", level=logging.DEBUG) + try: return SysCommand(f'blkid -s PARTUUID -o value {self.device_path}').decode('UTF-8').strip() except SysCallError as error: diff --git a/archinstall/lib/general.py b/archinstall/lib/general.py index a4e2a365..44b78777 100644 --- a/archinstall/lib/general.py +++ b/archinstall/lib/general.py @@ -135,6 +135,8 @@ class JsonEncoder: return obj.isoformat() elif isinstance(obj, (list, set, tuple)): return [json.loads(json.dumps(item, cls=JSON)) for item in obj] + elif isinstance(obj, (pathlib.Path)): + return str(obj) else: return obj diff --git a/archinstall/lib/hsm/__init__.py b/archinstall/lib/hsm/__init__.py new file mode 100644 index 00000000..c0888b04 --- /dev/null +++ b/archinstall/lib/hsm/__init__.py @@ -0,0 +1,4 @@ +from .fido import ( + get_fido2_devices, + fido2_enroll +) \ No newline at end of file diff --git a/archinstall/lib/hsm/fido.py b/archinstall/lib/hsm/fido.py new file mode 100644 index 00000000..69f42890 --- /dev/null +++ b/archinstall/lib/hsm/fido.py @@ -0,0 +1,47 @@ +import typing +import pathlib +from ..general import SysCommand, SysCommandWorker, clear_vt100_escape_codes +from ..disk.partition import Partition + +def get_fido2_devices() -> typing.Dict[str, typing.Dict[str, str]]: + """ + Uses systemd-cryptenroll to list the FIDO2 devices + connected that supports FIDO2. + Some devices might show up in udevadm as FIDO2 compliant + when they are in fact not. + + The drawback of systemd-cryptenroll is that it uses human readable format. + That means we get this weird table like structure that is of no use. + + So we'll look for `MANUFACTURER` and `PRODUCT`, we take their index + and we split each line based on those positions. + """ + worker = clear_vt100_escape_codes(SysCommand(f"systemd-cryptenroll --fido2-device=list").decode('UTF-8')) + + MANUFACTURER_POS = 0 + PRODUCT_POS = 0 + devices = {} + for line in worker.split('\r\n'): + if '/dev' not in line: + MANUFACTURER_POS = line.find('MANUFACTURER') + PRODUCT_POS = line.find('PRODUCT') + continue + + path = line[:MANUFACTURER_POS].rstrip() + manufacturer = line[MANUFACTURER_POS:PRODUCT_POS].rstrip() + product = line[PRODUCT_POS:] + + devices[path] = { + 'manufacturer' : manufacturer, + 'product' : product + } + + return devices + +def fido2_enroll(hsm_device_path :pathlib.Path, partition :Partition, password :str) -> bool: + worker = SysCommandWorker(f"systemd-cryptenroll --fido2-device={hsm_device_path} {partition.real_device}", peak_output=True) + pw_inputted = False + while worker.is_alive(): + if pw_inputted is False and bytes(f"please enter current passphrase for disk {partition.real_device}", 'UTF-8') in worker._trace_log.lower(): + worker.write(bytes(password, 'UTF-8')) + pw_inputted = True diff --git a/archinstall/lib/installer.py b/archinstall/lib/installer.py index e94a00c4..292b2c8e 100644 --- a/archinstall/lib/installer.py +++ b/archinstall/lib/installer.py @@ -23,6 +23,7 @@ from .profiles import Profile from .disk.btrfs import manage_btrfs_subvolumes from .disk.partition import get_mount_fs_type from .exceptions import DiskError, ServiceException, RequirementError, HardwareIncompatibilityError, SysCallError +from .hsm import fido2_enroll if TYPE_CHECKING: _: Any @@ -126,7 +127,9 @@ class Installer: self.MODULES = [] self.BINARIES = [] self.FILES = [] - self.HOOKS = ["base", "udev", "autodetect", "keyboard", "keymap", "modconf", "block", "filesystems", "fsck"] + # systemd, sd-vconsole and sd-encrypt will be replaced by udev, keymap and encrypt + # if HSM is not used to encrypt the root volume. Check mkinitcpio() function for that override. + self.HOOKS = ["base", "systemd", "autodetect", "keyboard", "sd-vconsole", "modconf", "block", "filesystems", "fsck"] self.KERNEL_PARAMS = [] self._zram_enabled = False @@ -241,10 +244,10 @@ class Installer: # open the luks device and all associate stuff if not (password := partition.get('!password', None)): raise RequirementError(f"Missing partition {partition['device_instance'].path} encryption password in layout: {partition}") - # i change a bit the naming conventions for the loop device loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(partition['mountpoint']).name}loop" else: loopdev = f"{storage.get('ENC_IDENTIFIER', 'ai')}{pathlib.Path(partition['device_instance'].path).name}" + # note that we DON'T auto_unmount (i.e. close the encrypted device so it can be used with (luks_handle := luks2(partition['device_instance'], loopdev, password, auto_unmount=False)) as unlocked_device: if partition.get('generate-encryption-key-file',False) and not self._has_root(partition): @@ -252,6 +255,10 @@ class Installer: # this way all the requesrs will be to the dm_crypt device and not to the physical partition partition['device_instance'] = unlocked_device + if self._has_root(partition) and partition.get('generate-encryption-key-file', False) is False: + hsm_device_path = storage['arguments']['HSM'] + fido2_enroll(hsm_device_path, partition['device_instance'], password) + # we manage the btrfs partitions for partition in [entry for entry in list_part if entry.get('btrfs', {}).get('subvolumes', {})]: if partition.get('filesystem',{}).get('mount_options',[]): @@ -609,6 +616,15 @@ class Installer: mkinit.write(f"MODULES=({' '.join(self.MODULES)})\n") mkinit.write(f"BINARIES=({' '.join(self.BINARIES)})\n") mkinit.write(f"FILES=({' '.join(self.FILES)})\n") + + if not storage['arguments']['HSM']: + # For now, if we don't use HSM we revert to the old + # way of setting up encryption hooks for mkinitcpio. + # This is purely for stability reasons, we're going away from this. + # * systemd -> udev + # * sd-vconsole -> keymap + self.HOOKS = [hook.replace('systemd', 'udev').replace('sd-vconsole', 'keymap') for hook in self.HOOKS] + mkinit.write(f"HOOKS=({' '.join(self.HOOKS)})\n") return SysCommand(f'/usr/bin/arch-chroot {self.target} mkinitcpio {" ".join(flags)}').exit_code == 0 @@ -643,8 +659,15 @@ class Installer: self.HOOKS.remove('fsck') if self.detect_encryption(partition): - if 'encrypt' not in self.HOOKS: - self.HOOKS.insert(self.HOOKS.index('filesystems'), 'encrypt') + if storage['arguments']['HSM']: + # Required bby mkinitcpio to add support for fido2-device options + self.pacstrap('libfido2') + + if 'sd-encrypt' not in self.HOOKS: + self.HOOKS.insert(self.HOOKS.index('filesystems'), 'sd-encrypt') + else: + if 'encrypt' not in self.HOOKS: + self.HOOKS.insert(self.HOOKS.index('filesystems'), 'encrypt') if not has_uefi(): self.base_packages.append('grub') @@ -700,6 +723,14 @@ class Installer: # TODO: Use python functions for this SysCommand(f'/usr/bin/arch-chroot {self.target} chmod 700 /root') + if storage['arguments']['HSM']: + # TODO: + # A bit of a hack, but we need to get vconsole.conf in there + # before running `mkinitcpio` because it expects it in HSM mode. + if (vconsole := pathlib.Path(f"{self.target}/etc/vconsole.conf")).exists() is False: + with vconsole.open('w') as fh: + fh.write(f"KEYMAP={storage['arguments']['keyboard-layout']}\n") + self.mkinitcpio('-P') self.helper_flags['base'] = True @@ -814,11 +845,23 @@ class Installer: if real_device := self.detect_encryption(root_partition): # TODO: We need to detect if the encrypted device is a whole disk encryption, # or simply a partition encryption. Right now we assume it's a partition (and we always have) - log(f"Identifying root partition by PART-UUID on {real_device}: '{real_device.uuid}'.", level=logging.DEBUG) - entry.write(f'options cryptdevice=PARTUUID={real_device.uuid}:luksdev root=/dev/mapper/luksdev {options_entry}') + log(f"Identifying root partition by PART-UUID on {real_device}: '{real_device.uuid}/{real_device.part_uuid}'.", level=logging.DEBUG) + + kernel_options = f"options" + + if storage['arguments']['HSM']: + # Note: lsblk UUID must be used, not PARTUUID for sd-encrypt to work + kernel_options += f" rd.luks.name={real_device.uuid}=luksdev" + # Note: tpm2-device and fido2-device don't play along very well: + # https://github.com/archlinux/archinstall/pull/1196#issuecomment-1129715645 + kernel_options += f" rd.luks.options=fido2-device=auto,password-echo=no" + else: + kernel_options += f" cryptdevice=PARTUUID={real_device.part_uuid}:luksdev" + + entry.write(f'{kernel_options} root=/dev/mapper/luksdev {options_entry}') else: - log(f"Identifying root partition by PART-UUID on {root_partition}, looking for '{root_partition.uuid}'.", level=logging.DEBUG) - entry.write(f'options root=PARTUUID={root_partition.uuid} {options_entry}') + log(f"Identifying root partition by PARTUUID on {root_partition}, looking for '{root_partition.part_uuid}'.", level=logging.DEBUG) + entry.write(f'options root=PARTUUID={root_partition.part_uuid} {options_entry}') self.helper_flags['bootloader'] = "systemd" @@ -903,11 +946,11 @@ class Installer: if real_device := self.detect_encryption(root_partition): # TODO: We need to detect if the encrypted device is a whole disk encryption, # or simply a partition encryption. Right now we assume it's a partition (and we always have) - log(f"Identifying root partition by PART-UUID on {real_device}: '{real_device.uuid}'.", level=logging.DEBUG) - kernel_parameters.append(f'cryptdevice=PARTUUID={real_device.uuid}:luksdev root=/dev/mapper/luksdev rw intel_pstate=no_hwp rootfstype={root_fs_type} {" ".join(self.KERNEL_PARAMS)}') + log(f"Identifying root partition by PART-UUID on {real_device}: '{real_device.part_uuid}'.", level=logging.DEBUG) + kernel_parameters.append(f'cryptdevice=PARTUUID={real_device.part_uuid}:luksdev root=/dev/mapper/luksdev rw intel_pstate=no_hwp rootfstype={root_fs_type} {" ".join(self.KERNEL_PARAMS)}') else: - log(f"Identifying root partition by PART-UUID on {root_partition}, looking for '{root_partition.uuid}'.", level=logging.DEBUG) - kernel_parameters.append(f'root=PARTUUID={root_partition.uuid} rw intel_pstate=no_hwp rootfstype={root_fs_type} {" ".join(self.KERNEL_PARAMS)}') + log(f"Identifying root partition by PART-UUID on {root_partition}, looking for '{root_partition.part_uuid}'.", level=logging.DEBUG) + kernel_parameters.append(f'root=PARTUUID={root_partition.part_uuid} rw intel_pstate=no_hwp rootfstype={root_fs_type} {" ".join(self.KERNEL_PARAMS)}') SysCommand(f'efibootmgr --disk {boot_partition.path[:-1]} --part {boot_partition.path[-1]} --create --label "{label}" --loader {loader} --unicode \'{" ".join(kernel_parameters)}\' --verbose') diff --git a/archinstall/lib/menu/global_menu.py b/archinstall/lib/menu/global_menu.py index 13d385ef..d807433c 100644 --- a/archinstall/lib/menu/global_menu.py +++ b/archinstall/lib/menu/global_menu.py @@ -85,6 +85,12 @@ class GlobalMenu(GeneralMenu): lambda x: self._select_encrypted_password(), display_func=lambda x: secret(x) if x else 'None', dependencies=['harddrives']) + self._menu_options['HSM'] = Selector( + description=_('Use HSM to unlock encrypted drive'), + func=lambda preset: self._select_hsm(preset), + dependencies=['!encryption-password'], + default=None + ) self._menu_options['swap'] = \ Selector( _('Swap'), diff --git a/archinstall/lib/menu/selection_menu.py b/archinstall/lib/menu/selection_menu.py index 35057e9c..26be4cc7 100644 --- a/archinstall/lib/menu/selection_menu.py +++ b/archinstall/lib/menu/selection_menu.py @@ -2,12 +2,14 @@ from __future__ import annotations import logging import sys +import pathlib from typing import Callable, Any, List, Iterator, Tuple, Optional, Dict, TYPE_CHECKING from .menu import Menu, MenuSelectionType from ..locale_helpers import set_keyboard_language from ..output import log from ..translation import Translation +from ..hsm.fido import get_fido2_devices if TYPE_CHECKING: _: Any @@ -466,3 +468,25 @@ class GeneralMenu: return language return preset_value + + def _select_hsm(self, preset :Optional[pathlib.Path] = None) -> Optional[pathlib.Path]: + title = _('Select which partitions to mark for formatting:') + title += '\n' + + fido_devices = get_fido2_devices() + + indexes = [] + for index, path in enumerate(fido_devices.keys()): + title += f"{index}: {path} ({fido_devices[path]['manufacturer']} - {fido_devices[path]['product']})" + indexes.append(f"{index}|{fido_devices[path]['product']}") + + title += '\n' + + choice = Menu(title, indexes, multi=False).run() + + match choice.type_: + case MenuSelectionType.Esc: return preset + case MenuSelectionType.Selection: + return pathlib.Path(list(fido_devices.keys())[int(choice.value.split('|',1)[0])]) + + return None \ No newline at end of file diff --git a/archinstall/lib/udev/__init__.py b/archinstall/lib/udev/__init__.py new file mode 100644 index 00000000..86c8cc29 --- /dev/null +++ b/archinstall/lib/udev/__init__.py @@ -0,0 +1 @@ +from .udevadm import udevadm_info \ No newline at end of file diff --git a/archinstall/lib/udev/udevadm.py b/archinstall/lib/udev/udevadm.py new file mode 100644 index 00000000..84ec9cfd --- /dev/null +++ b/archinstall/lib/udev/udevadm.py @@ -0,0 +1,17 @@ +import typing +import pathlib +from ..general import SysCommand + +def udevadm_info(path :pathlib.Path) -> typing.Dict[str, str]: + if path.resolve().exists() is False: + return {} + + result = SysCommand(f"udevadm info {path.resolve()}") + data = {} + for line in result: + if b': ' in line and b'=' in line: + _, obj = line.split(b': ', 1) + key, value = obj.split(b'=', 1) + data[key.decode('UTF-8').lower()] = value.decode('UTF-8').strip() + + return data \ No newline at end of file diff --git a/archinstall/locales/en/LC_MESSAGES/base.mo b/archinstall/locales/en/LC_MESSAGES/base.mo index c89651e7..e6ac80c2 100644 Binary files a/archinstall/locales/en/LC_MESSAGES/base.mo and b/archinstall/locales/en/LC_MESSAGES/base.mo differ diff --git a/archinstall/locales/en/LC_MESSAGES/base.po b/archinstall/locales/en/LC_MESSAGES/base.po index d883553c..531e20a9 100644 --- a/archinstall/locales/en/LC_MESSAGES/base.po +++ b/archinstall/locales/en/LC_MESSAGES/base.po @@ -1,8 +1,15 @@ msgid "" msgstr "" +"Project-Id-Version: \n" +"POT-Creation-Date: \n" +"PO-Revision-Date: \n" +"Last-Translator: \n" +"Language-Team: \n" +"Language: en\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" +"X-Generator: Poedit 3.0.1\n" msgid "[!] A log file has been created here: {} {}" msgstr "" diff --git a/examples/guided.py b/examples/guided.py index f104b7e3..3b762a8b 100644 --- a/examples/guided.py +++ b/examples/guided.py @@ -57,6 +57,10 @@ def ask_user_questions(): # Get disk encryption password (or skip if blank) global_menu.enable('!encryption-password') + if archinstall.arguments.get('advanced', False) or archinstall.arguments.get('HSM', None): + # Enables the use of HSM + global_menu.enable('HSM') + # Ask which boot-loader to use (will only ask if we're in UEFI mode, otherwise will default to GRUB) global_menu.enable('bootloader') @@ -130,6 +134,7 @@ def perform_installation(mountpoint): Only requirement is that the block devices are formatted and setup prior to entering this function. """ + with archinstall.Installer(mountpoint, kernels=archinstall.arguments.get('kernels', ['linux'])) as installation: # Mount all the drives to the desired mountpoint # This *can* be done outside of the installation, but the installer can deal with it. @@ -301,5 +306,6 @@ if archinstall.arguments.get('dry_run'): if not archinstall.arguments.get('silent'): input(str(_('Press Enter to continue.'))) +archinstall.configuration_sanity_check() perform_filesystem_operations() perform_installation(archinstall.storage.get('MOUNT_POINT', '/mnt')) -- cgit v1.2.3-70-g09d2 From c93482a8b943a593608d8bae7156e357ed0002d5 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Thu, 26 May 2022 18:46:10 +0200 Subject: Rework btrfs handling (#1234) * Restructuring btrfs.py into lib/btrfs/*.py * Reworking how BTRFS subvolumes get represented, and worked with. Subvolumes are now their own entity which can be used to access it's information, parents or mount location. * Added BtrfsSubvolume.partition and other stuff. * Reworking the way luks2().unlock and .format() returns device instances. They should now return BTRFSSubvolume where appropriate. * Fixed a missing import * Fixed an issue where mkfs.btrfs wouldn't trigger due to busy disk. * Fixing subvol mounting without creating a fake instance. * Added creation of mountpint for btrfs subvolume * Fixed root detection * Re-worked mounting into a queue system using frozen mounting calls using lambda * Removed old mount_subvolume() function * Removed get_subvolumes_from_findmnt() * Fixed Partition().subvolumes iteration * Adding .root to BtrfsSubvolume * Fixed issue in SysCommandWorker where log output would break and crash execution due to cmd being a string vs list * Changed return-value from MapperDev.mountpoint to pathlib.Path --- archinstall/lib/disk/__init__.py | 2 +- archinstall/lib/disk/btrfs.py | 232 --------------------------- archinstall/lib/disk/btrfs/__init__.py | 182 +++++++++++++++++++++ archinstall/lib/disk/btrfs/btrfs_helpers.py | 132 +++++++++++++++ archinstall/lib/disk/btrfs/btrfspartition.py | 116 ++++++++++++++ archinstall/lib/disk/btrfs/btrfssubvolume.py | 191 ++++++++++++++++++++++ archinstall/lib/disk/filesystem.py | 13 ++ archinstall/lib/disk/helpers.py | 31 +++- archinstall/lib/disk/mapperdev.py | 10 +- archinstall/lib/disk/partition.py | 120 ++++++++------ archinstall/lib/exceptions.py | 4 + archinstall/lib/general.py | 2 +- archinstall/lib/installer.py | 77 ++++++--- archinstall/lib/luks.py | 16 +- archinstall/lib/output.py | 6 +- archinstall/lib/systemd.py | 13 +- 16 files changed, 829 insertions(+), 318 deletions(-) delete mode 100644 archinstall/lib/disk/btrfs.py create mode 100644 archinstall/lib/disk/btrfs/__init__.py create mode 100644 archinstall/lib/disk/btrfs/btrfs_helpers.py create mode 100644 archinstall/lib/disk/btrfs/btrfspartition.py create mode 100644 archinstall/lib/disk/btrfs/btrfssubvolume.py (limited to 'archinstall/lib/general.py') diff --git a/archinstall/lib/disk/__init__.py b/archinstall/lib/disk/__init__.py index bb6eb815..352d04b9 100644 --- a/archinstall/lib/disk/__init__.py +++ b/archinstall/lib/disk/__init__.py @@ -4,4 +4,4 @@ from .blockdevice import BlockDevice from .filesystem import Filesystem, MBR, GPT from .partition import * from .user_guides import * -from .validators import * +from .validators import * \ No newline at end of file diff --git a/archinstall/lib/disk/btrfs.py b/archinstall/lib/disk/btrfs.py deleted file mode 100644 index 33f59721..00000000 --- a/archinstall/lib/disk/btrfs.py +++ /dev/null @@ -1,232 +0,0 @@ -from __future__ import annotations -import pathlib -import glob -import logging -import re -from typing import Union, Dict, TYPE_CHECKING, Any, Iterator -from dataclasses import dataclass - -# https://stackoverflow.com/a/39757388/929999 -if TYPE_CHECKING: - from ..installer import Installer -from .helpers import get_mount_info -from ..exceptions import DiskError -from ..general import SysCommand -from ..output import log -from ..exceptions import SysCallError - -@dataclass -class BtrfsSubvolume: - target :str - source :str - fstype :str - name :str - options :str - root :bool = False - -def get_subvolumes_from_findmnt(struct :Dict[str, Any], index=0) -> Iterator[BtrfsSubvolume]: - if '[' in struct['source']: - subvolume = re.findall(r'\[.*?\]', struct['source'])[0][1:-1] - struct['source'] = struct['source'].replace(f"[{subvolume}]", "") - yield BtrfsSubvolume( - target=struct['target'], - source=struct['source'], - fstype=struct['fstype'], - name=subvolume, - options=struct['options'], - root=index == 0 - ) - index += 1 - - for child in struct.get('children', []): - for item in get_subvolumes_from_findmnt(child, index=index): - yield item - index += 1 - -def get_subvolume_info(path :pathlib.Path) -> Dict[str, Any]: - try: - output = SysCommand(f"btrfs subvol show {path}").decode() - except SysCallError as error: - print('Error:', error) - - result = {} - for line in output.replace('\r\n', '\n').split('\n'): - if ':' in line: - key, val = line.replace('\t', '').split(':', 1) - result[key.strip().lower().replace(' ', '_')] = val.strip() - - return result - -def mount_subvolume(installation :Installer, subvolume_location :Union[pathlib.Path, str], force=False) -> bool: - """ - This function uses mount to mount a subvolume on a given device, at a given location with a given subvolume name. - - @installation: archinstall.Installer instance - @subvolume_location: a localized string or path inside the installation / or /boot for instance without specifying /mnt/boot - @force: overrides the check for weither or not the subvolume mountpoint is empty or not - - This function is DEPRECATED. you can get the same result creating a partition dict like any other partition, and using the standard mount procedure. - Only change partition['device_instance'].path with the apropriate bind name: real_partition_path[/subvolume_name] - """ - log("[Deprecated] function btrfs.mount_subvolume is deprecated. See code for alternatives",fg="yellow",level=logging.WARNING) - installation_mountpoint = installation.target - if type(installation_mountpoint) == str: - installation_mountpoint = pathlib.Path(installation_mountpoint) - # Set up the required physical structure - if type(subvolume_location) == str: - subvolume_location = pathlib.Path(subvolume_location) - - target = installation_mountpoint / subvolume_location.relative_to(subvolume_location.anchor) - - if not target.exists(): - target.mkdir(parents=True) - - if glob.glob(str(target / '*')) and force is False: - raise DiskError(f"Cannot mount subvolume to {target} because it contains data (non-empty folder target)") - - log(f"Mounting {target} as a subvolume", level=logging.INFO) - # Mount the logical volume to the physical structure - mount_information, mountpoint_device_real_path = get_mount_info(target, traverse=True, return_real_path=True) - if mountpoint_device_real_path == str(target): - log(f"Unmounting non-subvolume {mount_information['source']} previously mounted at {target}") - SysCommand(f"umount {mount_information['source']}") - - return SysCommand(f"mount {mount_information['source']} {target} -o subvol=@{subvolume_location}").exit_code == 0 - -def create_subvolume(installation :Installer, subvolume_location :Union[pathlib.Path, str]) -> bool: - """ - This function uses btrfs to create a subvolume. - - @installation: archinstall.Installer instance - @subvolume_location: a localized string or path inside the installation / or /boot for instance without specifying /mnt/boot - """ - - installation_mountpoint = installation.target - if type(installation_mountpoint) == str: - installation_mountpoint = pathlib.Path(installation_mountpoint) - # Set up the required physical structure - if type(subvolume_location) == str: - subvolume_location = pathlib.Path(subvolume_location) - - target = installation_mountpoint / subvolume_location.relative_to(subvolume_location.anchor) - - # Difference from mount_subvolume: - # We only check if the parent exists, since we'll run in to "target path already exists" otherwise - if not target.parent.exists(): - target.parent.mkdir(parents=True) - - if glob.glob(str(target / '*')): - raise DiskError(f"Cannot create subvolume at {target} because it contains data (non-empty folder target)") - - # Remove the target if it exists - if target.exists(): - target.rmdir() - - log(f"Creating a subvolume on {target}", level=logging.INFO) - if (cmd := SysCommand(f"btrfs subvolume create {target}")).exit_code != 0: - raise DiskError(f"Could not create a subvolume at {target}: {cmd}") - -def _has_option(option :str,options :list) -> bool: - """ auxiliary routine to check if an option is present in a list. - we check if the string appears in one of the options, 'cause it can appear in severl forms (option, option=val,...) - """ - if not options: - return False - for item in options: - if option in item: - return True - return False - -def manage_btrfs_subvolumes(installation :Installer, - partition :Dict[str, str],) -> list: - from copy import deepcopy - """ we do the magic with subvolumes in a centralized place - parameters: - * the installation object - * the partition dictionary entry which represents the physical partition - returns - * mountpoinst, the list which contains all the "new" partititon to be mounted - - We expect the partition has been mounted as / , and it to be unmounted after the processing - Then we create all the subvolumes inside btrfs as demand - We clone then, both the partition dictionary and the object inside it and adapt it to the subvolume needs - Then we return a list of "new" partitions to be processed as "normal" partitions - # TODO For encrypted devices we need some special processing prior to it - """ - # We process each of the pairs - # th mount info dict has an entry for the path of the mountpoint (named 'mountpoint') and 'options' which is a list - # of mount options (or similar used by brtfs) - mountpoints = [] - subvolumes = partition['btrfs']['subvolumes'] - for name, right_hand in subvolumes.items(): - try: - # we normalize the subvolume name (getting rid of slash at the start if exists. In our implemenation has no semantic load - every subvolume is created from the top of the hierarchy- and simplifies its further use - if name.startswith('/'): - name = name[1:] - # renormalize the right hand. - location = None - subvol_options = [] - # no contents, so it is not to be mounted - if not right_hand: - location = None - # just a string. per backward compatibility the mount point - elif isinstance(right_hand,str): - location = right_hand - # a dict. two elements 'mountpoint' (obvious) and and a mount options list ¿? - elif isinstance(right_hand,dict): - location = right_hand.get('mountpoint',None) - subvol_options = right_hand.get('options',[]) - # we create the subvolume - create_subvolume(installation,name) - # Make the nodatacow processing now - # It will be the main cause of creation of subvolumes which are not to be mounted - # it is not an options which can be established by subvolume (but for whole file systems), and can be - # set up via a simple attribute change in a directory (if empty). And here the directories are brand new - if 'nodatacow' in subvol_options: - if (cmd := SysCommand(f"chattr +C {installation.target}/{name}")).exit_code != 0: - raise DiskError(f"Could not set nodatacow attribute at {installation.target}/{name}: {cmd}") - # entry is deleted so nodatacow doesn't propagate to the mount options - del subvol_options[subvol_options.index('nodatacow')] - # Make the compress processing now - # it is not an options which can be established by subvolume (but for whole file systems), and can be - # set up via a simple attribute change in a directory (if empty). And here the directories are brand new - # in this way only zstd compression is activaded - # TODO WARNING it is not clear if it should be a standard feature, so it might need to be deactivated - if 'compress' in subvol_options: - if not _has_option('compress',partition.get('filesystem',{}).get('mount_options',[])): - if (cmd := SysCommand(f"chattr +c {installation.target}/{name}")).exit_code != 0: - raise DiskError(f"Could not set compress attribute at {installation.target}/{name}: {cmd}") - # entry is deleted so compress doesn't propagate to the mount options - del subvol_options[subvol_options.index('compress')] - # END compress processing. - # we do not mount if THE basic partition will be mounted or if we exclude explicitly this subvolume - if not partition['mountpoint'] and location is not None: - # we begin to create a fake partition entry. First we copy the original -the one that corresponds to - # the primary partition. We make a deepcopy to avoid altering the original content in any case - fake_partition = deepcopy(partition) - # we start to modify entries in the "fake partition" to match the needs of the subvolumes - # to avoid any chance of entering in a loop (not expected) we delete the list of subvolumes in the copy - del fake_partition['btrfs'] - fake_partition['encrypted'] = False - fake_partition['generate-encryption-key-file'] = False - # Mount destination. As of now the right hand part - fake_partition['mountpoint'] = location - # we load the name in an attribute called subvolume, but i think it is not needed anymore, 'cause the mount logic uses a different path. - fake_partition['subvolume'] = name - # here we add the special mount options for the subvolume, if any. - # if the original partition['options'] is not a list might give trouble - if fake_partition.get('filesystem',{}).get('mount_options',[]): - fake_partition['filesystem']['mount_options'].extend(subvol_options) - else: - fake_partition['filesystem']['mount_options'] = subvol_options - # Here comes the most exotic part. The dictionary attribute 'device_instance' contains an instance of Partition. This instance will be queried along the mount process at the installer. - # As the rest will query there the path of the "partition" to be mounted, we feed it with the bind name needed to mount subvolumes - # As we made a deepcopy we have a fresh instance of this object we can manipulate problemless - fake_partition['device_instance'].path = f"{partition['device_instance'].path}[/{name}]" - - # Well, now that this "fake partition" is ready, we add it to the list of the ones which are to be mounted, - # as "normal" ones - mountpoints.append(fake_partition) - except Exception as e: - raise e - return mountpoints diff --git a/archinstall/lib/disk/btrfs/__init__.py b/archinstall/lib/disk/btrfs/__init__.py new file mode 100644 index 00000000..84b9c0f6 --- /dev/null +++ b/archinstall/lib/disk/btrfs/__init__.py @@ -0,0 +1,182 @@ +from __future__ import annotations +import pathlib +import glob +import logging +import re +from typing import Union, Dict, TYPE_CHECKING, Any, Iterator + +# https://stackoverflow.com/a/39757388/929999 +if TYPE_CHECKING: + from ...installer import Installer + +from .btrfs_helpers import ( + subvolume_info_from_path as subvolume_info_from_path, + find_parent_subvolume as find_parent_subvolume, + setup_subvolumes as setup_subvolumes, + mount_subvolume as mount_subvolume +) +from .btrfssubvolume import BtrfsSubvolume as BtrfsSubvolume +from .btrfspartition import BTRFSPartition as BTRFSPartition + +from ..helpers import get_mount_info +from ...exceptions import DiskError, Deprecated +from ...general import SysCommand +from ...output import log +from ...exceptions import SysCallError + +def get_subvolume_info(path :pathlib.Path) -> Dict[str, Any]: + try: + output = SysCommand(f"btrfs subvol show {path}").decode() + except SysCallError as error: + print('Error:', error) + + result = {} + for line in output.replace('\r\n', '\n').split('\n'): + if ':' in line: + key, val = line.replace('\t', '').split(':', 1) + result[key.strip().lower().replace(' ', '_')] = val.strip() + + return result + +def create_subvolume(installation :Installer, subvolume_location :Union[pathlib.Path, str]) -> bool: + """ + This function uses btrfs to create a subvolume. + + @installation: archinstall.Installer instance + @subvolume_location: a localized string or path inside the installation / or /boot for instance without specifying /mnt/boot + """ + + installation_mountpoint = installation.target + if type(installation_mountpoint) == str: + installation_mountpoint = pathlib.Path(installation_mountpoint) + # Set up the required physical structure + if type(subvolume_location) == str: + subvolume_location = pathlib.Path(subvolume_location) + + target = installation_mountpoint / subvolume_location.relative_to(subvolume_location.anchor) + + # Difference from mount_subvolume: + # We only check if the parent exists, since we'll run in to "target path already exists" otherwise + if not target.parent.exists(): + target.parent.mkdir(parents=True) + + if glob.glob(str(target / '*')): + raise DiskError(f"Cannot create subvolume at {target} because it contains data (non-empty folder target)") + + # Remove the target if it exists + if target.exists(): + target.rmdir() + + log(f"Creating a subvolume on {target}", level=logging.INFO) + if (cmd := SysCommand(f"btrfs subvolume create {target}")).exit_code != 0: + raise DiskError(f"Could not create a subvolume at {target}: {cmd}") + +def _has_option(option :str,options :list) -> bool: + """ auxiliary routine to check if an option is present in a list. + we check if the string appears in one of the options, 'cause it can appear in severl forms (option, option=val,...) + """ + if not options: + return False + + for item in options: + if option in item: + return True + + return False + +def manage_btrfs_subvolumes(installation :Installer, + partition :Dict[str, str],) -> list: + + raise Deprecated("Use setup_subvolumes() instead.") + + from copy import deepcopy + """ we do the magic with subvolumes in a centralized place + parameters: + * the installation object + * the partition dictionary entry which represents the physical partition + returns + * mountpoinst, the list which contains all the "new" partititon to be mounted + + We expect the partition has been mounted as / , and it to be unmounted after the processing + Then we create all the subvolumes inside btrfs as demand + We clone then, both the partition dictionary and the object inside it and adapt it to the subvolume needs + Then we return a list of "new" partitions to be processed as "normal" partitions + # TODO For encrypted devices we need some special processing prior to it + """ + # We process each of the pairs + # th mount info dict has an entry for the path of the mountpoint (named 'mountpoint') and 'options' which is a list + # of mount options (or similar used by brtfs) + mountpoints = [] + subvolumes = partition['btrfs']['subvolumes'] + for name, right_hand in subvolumes.items(): + try: + # we normalize the subvolume name (getting rid of slash at the start if exists. In our implemenation has no semantic load - every subvolume is created from the top of the hierarchy- and simplifies its further use + if name.startswith('/'): + name = name[1:] + # renormalize the right hand. + location = None + subvol_options = [] + # no contents, so it is not to be mounted + if not right_hand: + location = None + # just a string. per backward compatibility the mount point + elif isinstance(right_hand,str): + location = right_hand + # a dict. two elements 'mountpoint' (obvious) and and a mount options list ¿? + elif isinstance(right_hand,dict): + location = right_hand.get('mountpoint',None) + subvol_options = right_hand.get('options',[]) + # we create the subvolume + create_subvolume(installation,name) + # Make the nodatacow processing now + # It will be the main cause of creation of subvolumes which are not to be mounted + # it is not an options which can be established by subvolume (but for whole file systems), and can be + # set up via a simple attribute change in a directory (if empty). And here the directories are brand new + if 'nodatacow' in subvol_options: + if (cmd := SysCommand(f"chattr +C {installation.target}/{name}")).exit_code != 0: + raise DiskError(f"Could not set nodatacow attribute at {installation.target}/{name}: {cmd}") + # entry is deleted so nodatacow doesn't propagate to the mount options + del subvol_options[subvol_options.index('nodatacow')] + # Make the compress processing now + # it is not an options which can be established by subvolume (but for whole file systems), and can be + # set up via a simple attribute change in a directory (if empty). And here the directories are brand new + # in this way only zstd compression is activaded + # TODO WARNING it is not clear if it should be a standard feature, so it might need to be deactivated + if 'compress' in subvol_options: + if not _has_option('compress',partition.get('filesystem',{}).get('mount_options',[])): + if (cmd := SysCommand(f"chattr +c {installation.target}/{name}")).exit_code != 0: + raise DiskError(f"Could not set compress attribute at {installation.target}/{name}: {cmd}") + # entry is deleted so compress doesn't propagate to the mount options + del subvol_options[subvol_options.index('compress')] + # END compress processing. + # we do not mount if THE basic partition will be mounted or if we exclude explicitly this subvolume + if not partition['mountpoint'] and location is not None: + # we begin to create a fake partition entry. First we copy the original -the one that corresponds to + # the primary partition. We make a deepcopy to avoid altering the original content in any case + fake_partition = deepcopy(partition) + # we start to modify entries in the "fake partition" to match the needs of the subvolumes + # to avoid any chance of entering in a loop (not expected) we delete the list of subvolumes in the copy + del fake_partition['btrfs'] + fake_partition['encrypted'] = False + fake_partition['generate-encryption-key-file'] = False + # Mount destination. As of now the right hand part + fake_partition['mountpoint'] = location + # we load the name in an attribute called subvolume, but i think it is not needed anymore, 'cause the mount logic uses a different path. + fake_partition['subvolume'] = name + # here we add the special mount options for the subvolume, if any. + # if the original partition['options'] is not a list might give trouble + if fake_partition.get('filesystem',{}).get('mount_options',[]): + fake_partition['filesystem']['mount_options'].extend(subvol_options) + else: + fake_partition['filesystem']['mount_options'] = subvol_options + # Here comes the most exotic part. The dictionary attribute 'device_instance' contains an instance of Partition. This instance will be queried along the mount process at the installer. + # As the rest will query there the path of the "partition" to be mounted, we feed it with the bind name needed to mount subvolumes + # As we made a deepcopy we have a fresh instance of this object we can manipulate problemless + fake_partition['device_instance'].path = f"{partition['device_instance'].path}[/{name}]" + + # Well, now that this "fake partition" is ready, we add it to the list of the ones which are to be mounted, + # as "normal" ones + mountpoints.append(fake_partition) + except Exception as e: + raise e + return mountpoints diff --git a/archinstall/lib/disk/btrfs/btrfs_helpers.py b/archinstall/lib/disk/btrfs/btrfs_helpers.py new file mode 100644 index 00000000..d529478f --- /dev/null +++ b/archinstall/lib/disk/btrfs/btrfs_helpers.py @@ -0,0 +1,132 @@ +import pathlib +import logging +from typing import Optional + +from ...exceptions import SysCallError, DiskError +from ...general import SysCommand +from ...output import log +from ..helpers import get_mount_info +from .btrfssubvolume import BtrfsSubvolume + + +def mount_subvolume(installation, device, name, subvolume_information): + # we normalize the subvolume name (getting rid of slash at the start if exists. In our implemenation has no semantic load. + # Every subvolume is created from the top of the hierarchy- and simplifies its further use + name = name.lstrip('/') + + # renormalize the right hand. + mountpoint = subvolume_information.get('mountpoint', None) + if not mountpoint: + return None + + if type(mountpoint) == str: + mountpoint = pathlib.Path(mountpoint) + + installation_target = installation.target + if type(installation_target) == str: + installation_target = pathlib.Path(installation_target) + + mountpoint = installation_target / mountpoint.relative_to(mountpoint.anchor) + mountpoint.mkdir(parents=True, exist_ok=True) + + mount_options = subvolume_information.get('options', []) + if not any('subvol=' in x for x in mount_options): + mount_options += [f'subvol={name}'] + + log(f"Mounting subvolume {name} on {device} to {mountpoint}", level=logging.INFO, fg="gray") + SysCommand(f"mount {device.path} {mountpoint} -o {','.join(mount_options)}") + + +def setup_subvolumes(installation, partition_dict): + """ + Taken from: ..user_guides.py + + partition['btrfs'] = { + "subvolumes" : { + "@": "/", + "@home": "/home", + "@log": "/var/log", + "@pkg": "/var/cache/pacman/pkg", + "@.snapshots": "/.snapshots" + } + } + """ + log(f"Setting up subvolumes: {partition_dict['btrfs']['subvolumes']}", level=logging.INFO, fg="gray") + for name, right_hand in partition_dict['btrfs']['subvolumes'].items(): + # we normalize the subvolume name (getting rid of slash at the start if exists. In our implemenation has no semantic load. + # Every subvolume is created from the top of the hierarchy- and simplifies its further use + name = name.lstrip('/') + + # renormalize the right hand. + # mountpoint = None + subvol_options = [] + + match right_hand: + # case str(): # backwards-compatability + # mountpoint = right_hand + case dict(): + # mountpoint = right_hand.get('mountpoint', None) + subvol_options = right_hand.get('options', []) + + # We create the subvolume using the BTRFSPartition instance. + # That way we ensure not only easy access, but also accurate mount locations etc. + partition_dict['device_instance'].create_subvolume(name, installation=installation) + + # Make the nodatacow processing now + # It will be the main cause of creation of subvolumes which are not to be mounted + # it is not an options which can be established by subvolume (but for whole file systems), and can be + # set up via a simple attribute change in a directory (if empty). And here the directories are brand new + if 'nodatacow' in subvol_options: + if (cmd := SysCommand(f"chattr +C {installation.target}/{name}")).exit_code != 0: + raise DiskError(f"Could not set nodatacow attribute at {installation.target}/{name}: {cmd}") + # entry is deleted so nodatacow doesn't propagate to the mount options + del subvol_options[subvol_options.index('nodatacow')] + # Make the compress processing now + # it is not an options which can be established by subvolume (but for whole file systems), and can be + # set up via a simple attribute change in a directory (if empty). And here the directories are brand new + # in this way only zstd compression is activaded + # TODO WARNING it is not clear if it should be a standard feature, so it might need to be deactivated + + if 'compress' in subvol_options: + if not any(['compress' in filesystem_option for filesystem_option in partition_dict.get('filesystem', {}).get('mount_options', [])]): + if (cmd := SysCommand(f"chattr +c {installation.target}/{name}")).exit_code != 0: + raise DiskError(f"Could not set compress attribute at {installation.target}/{name}: {cmd}") + # entry is deleted so compress doesn't propagate to the mount options + del subvol_options[subvol_options.index('compress')] + +def subvolume_info_from_path(path :pathlib.Path) -> Optional[BtrfsSubvolume]: + try: + subvolume_name = None + result = {} + for index, line in enumerate(SysCommand(f"btrfs subvolume show {path}")): + if index == 0: + subvolume_name = line.strip().decode('UTF-8') + continue + + if b':' in line: + key, value = line.strip().decode('UTF-8').split(':', 1) + + # A bit of a hack, until I figure out how @dataclass + # allows for hooking in a pre-processor to do this we have to do it here: + result[key.lower().replace(' ', '_').replace('(s)', 's')] = value.strip() + + return BtrfsSubvolume(**{'full_path' : path, 'name' : subvolume_name, **result}) + + except SysCallError: + pass + + return None + +def find_parent_subvolume(path :pathlib.Path, filters=[]): + # A root path cannot have a parent + if str(path) == '/': + return None + + if found_mount := get_mount_info(str(path.parent), traverse=True, ignore=filters): + if not (subvolume := subvolume_info_from_path(found_mount['target'])): + if found_mount['target'] == '/': + return None + + return find_parent_subvolume(path.parent, traverse=True, filters=[*filters, found_mount['target']]) + + return subvolume \ No newline at end of file diff --git a/archinstall/lib/disk/btrfs/btrfspartition.py b/archinstall/lib/disk/btrfs/btrfspartition.py new file mode 100644 index 00000000..5020133d --- /dev/null +++ b/archinstall/lib/disk/btrfs/btrfspartition.py @@ -0,0 +1,116 @@ +import glob +import pathlib +import logging +from typing import Optional, TYPE_CHECKING + +from ...exceptions import DiskError +from ...storage import storage +from ...output import log +from ...general import SysCommand +from ..partition import Partition +from ..helpers import findmnt +from .btrfs_helpers import ( + subvolume_info_from_path +) + +if TYPE_CHECKING: + from ...installer import Installer + from .btrfssubvolume import BtrfsSubvolume + +class BTRFSPartition(Partition): + def __init__(self, *args, **kwargs): + Partition.__init__(self, *args, **kwargs) + + def __repr__(self, *args :str, **kwargs :str) -> str: + mount_repr = '' + if self.mountpoint: + mount_repr = f", mounted={self.mountpoint}" + elif self.target_mountpoint: + mount_repr = f", rel_mountpoint={self.target_mountpoint}" + + if self._encrypted: + return f'BTRFSPartition(path={self.path}, size={self.size}, PARTUUID={self._safe_uuid}, parent={self.real_device}, fs={self.filesystem}{mount_repr})' + else: + return f'BTRFSPartition(path={self.path}, size={self.size}, PARTUUID={self._safe_uuid}, fs={self.filesystem}{mount_repr})' + + @property + def subvolumes(self): + for filesystem in findmnt(pathlib.Path(self.path), recurse=True).get('filesystems', []): + if '[' in filesystem.get('source', ''): + yield subvolume_info_from_path(filesystem['target']) + + def iterate_children(struct): + for child in struct.get('children', []): + if '[' in child.get('source', ''): + yield subvolume_info_from_path(child['target']) + + for sub_child in iterate_children(child): + yield sub_child + + for child in iterate_children(filesystem): + yield child + + def create_subvolume(self, subvolume :pathlib.Path, installation :Optional['Installer'] = None) -> 'BtrfsSubvolume': + """ + Subvolumes have to be created within a mountpoint. + This means we need to get the current installation target. + After we get it, we need to verify it is a btrfs subvolume filesystem. + Finally, the destination must be empty. + """ + + # Allow users to override the installation session + if not installation: + installation = storage.get('installation_session') + + # Determain if the path given, is an absolute path or a releative path. + # We do this by checking if the path contains a known mountpoint. + if str(subvolume)[0] == '/': + if filesystems := findmnt(subvolume, traverse=True).get('filesystems'): + if (target := filesystems[0].get('target')) and target != '/' and str(subvolume).startswith(target): + # Path starts with a known mountpoint which isn't / + # Which means it's an absolut path to a mounted location. + pass + else: + # Since it's not an absolute position with a known start. + # We omit the anchor ('/' basically) and make sure it's appendable + # to the installation.target later + subvolume = subvolume.relative_to(subvolume.anchor) + # else: We don't need to do anything about relative paths, they should be appendable to installation.target as-is. + + # If the subvolume is not absolute, then we do two checks: + # 1. Check if the partition itself is mounted somewhere, and use that as a root + # 2. Use an active Installer().target as the root, assuming it's filesystem is btrfs + # If both above fail, we need to warn the user that such setup is not supported. + if str(subvolume)[0] != '/': + if self.mountpoint is None and installation is None: + raise DiskError("When creating a subvolume on BTRFSPartition()'s, you need to either initiate a archinstall.Installer() or give absolute paths when creating the subvoulme.") + elif self.mountpoint: + subvolume = self.mountpoint / subvolume + elif installation: + ongoing_installation_destination = installation.target + if type(ongoing_installation_destination) == str: + ongoing_installation_destination = pathlib.Path(ongoing_installation_destination) + + subvolume = ongoing_installation_destination / subvolume + + subvolume.parent.mkdir(parents=True, exist_ok=True) + + # + + log(f'Attempting to create subvolume at {subvolume}', level=logging.DEBUG, fg="grey") + + if glob.glob(str(subvolume / '*')): + raise DiskError(f"Cannot create subvolume at {subvolume} because it contains data (non-empty folder target is not supported by BTRFS)") + elif subvolinfo := subvolume_info_from_path(subvolume): + raise DiskError(f"Destination {subvolume} is already a subvolume: {subvolinfo}") + + SysCommand(f"btrfs subvolume create {subvolume}") + + return subvolume_info_from_path(subvolume) \ No newline at end of file diff --git a/archinstall/lib/disk/btrfs/btrfssubvolume.py b/archinstall/lib/disk/btrfs/btrfssubvolume.py new file mode 100644 index 00000000..a96e2a94 --- /dev/null +++ b/archinstall/lib/disk/btrfs/btrfssubvolume.py @@ -0,0 +1,191 @@ +import pathlib +import datetime +import logging +import string +import random +import shutil +from dataclasses import dataclass +from typing import Optional, List# , TYPE_CHECKING +from functools import cached_property + +# if TYPE_CHECKING: +# from ..blockdevice import BlockDevice + +from ...exceptions import DiskError +from ...general import SysCommand +from ...output import log +from ...storage import storage + +@dataclass +class BtrfsSubvolume: + full_path :pathlib.Path + name :str + uuid :str + parent_uuid :str + creation_time :datetime.datetime + subvolume_id :int + generation :int + gen_at_creation :int + parent_id :int + top_level_id :int + send_transid :int + send_time :datetime.datetime + receive_transid :int + received_uuid :Optional[str] = None + flags :Optional[str] = None + receive_time :Optional[datetime.datetime] = None + snapshots :Optional[List] = None + + def __post_init__(self): + self.full_path = pathlib.Path(self.full_path) + + # Convert "-" entries to `None` + if self.parent_uuid == "-": + self.parent_uuid = None + if self.received_uuid == "-": + self.received_uuid = None + if self.flags == "-": + self.flags = None + if self.receive_time == "-": + self.receive_time = None + if self.snapshots == "": + self.snapshots = [] + + # Convert timestamps into datetime workable objects (and preserve timezone by using ISO formats) + self.creation_time = datetime.datetime.fromisoformat(self.convert_to_ISO_format(self.creation_time)) + self.send_time = datetime.datetime.fromisoformat(self.convert_to_ISO_format(self.send_time)) + if self.receive_time: + self.receive_time = datetime.datetime.fromisoformat(self.convert_to_ISO_format(self.receive_time)) + + @property + def parent_subvolume(self): + from .btrfs_helpers import find_parent_subvolume + + return find_parent_subvolume(self.full_path) + + @property + def root(self) -> bool: + from .btrfs_helpers import subvolume_info_from_path + + # TODO: Make this function traverse storage['MOUNT_POINT'] and find the first + # occurance of a mountpoint that is a btrfs volume instead of lazy assume / is a subvolume. + # It would also be nice if it could use findmnt(self.full_path) and traverse backwards + # finding the last occurance of a subvolume which 'self' belongs to. + if volume := subvolume_info_from_path(storage['MOUNT_POINT']): + return self.full_path == volume.full_path + + return False + + @cached_property + def partition(self): + from ..helpers import findmnt, get_parent_of_partition, all_blockdevices + from ..partition import Partition + from ..blockdevice import BlockDevice + from ..mapperdev import MapperDev + from .btrfspartition import BTRFSPartition + from .btrfs_helpers import subvolume_info_from_path + + try: + # If the subvolume is mounted, it's pretty trivial to lookup the partition (parent) device. + if filesystem := findmnt(self.full_path).get('filesystems', []): + if source := filesystem[0].get('source', None): + # Strip away subvolume definitions from findmnt + if '[' in source: + source = source[:source.find('[')] + + if filesystem[0].get('fstype', '') == 'btrfs': + return BTRFSPartition(source, BlockDevice(get_parent_of_partition(pathlib.Path(source)))) + elif filesystem[0].get('source', '').startswith('/dev/mapper'): + return MapperDev(source) + else: + return Partition(source, BlockDevice(get_parent_of_partition(pathlib.Path(source)))) + except DiskError: + # Subvolume has never been mounted, we have no reliable way of finding where it is. + # But we have the UUID of the partition, and can begin looking for it by mounting + # all blockdevices that we can reliably support.. This is taxing tho and won't cover all devices. + + log(f"Looking up {self}, this might take time.", fg="orange", level=logging.WARNING) + for blockdevice, instance in all_blockdevices(mappers=True, partitions=True, error=True).items(): + if type(instance) in (Partition, MapperDev): + we_mounted_it = False + detection_mountpoint = instance.mountpoint + if not detection_mountpoint: + if type(instance) == Partition and instance.encrypted: + # TODO: Perhaps support unlocking encrypted volumes? + # This will cause a lot of potential user interactions tho. + log(f"Ignoring {blockdevice} because it's encrypted.", fg="gray", level=logging.DEBUG) + continue + + detection_mountpoint = pathlib.Path(f"/tmp/{''.join([random.choice(string.ascii_letters) for x in range(20)])}") + detection_mountpoint.mkdir(parents=True, exist_ok=True) + + instance.mount(str(detection_mountpoint)) + we_mounted_it = True + + if (filesystem := findmnt(detection_mountpoint)) and (filesystem := filesystem.get('filesystems', [])): + if subvolume := subvolume_info_from_path(filesystem[0]['target']): + if subvolume.uuid == self.uuid: + # The top level subvolume matched of ourselves, + # which means the instance we're iterating has the subvol we're looking for. + log(f"Found the subvolume on device {instance}", level=logging.DEBUG, fg="gray") + return instance + + def iterate_children(struct): + for child in struct.get('children', []): + if '[' in child.get('source', ''): + yield subvolume_info_from_path(child['target']) + + for sub_child in iterate_children(child): + yield sub_child + + for child in iterate_children(filesystem[0]): + if child.uuid == self.uuid: + # We found a child within the instance that has the subvol we're looking for. + log(f"Found the subvolume on device {instance}", level=logging.DEBUG, fg="gray") + return instance + + if we_mounted_it: + instance.unmount() + shutil.rmtree(detection_mountpoint) + + @cached_property + def mount_options(self) -> Optional[List[str]]: + from ..helpers import findmnt + + if filesystem := findmnt(self.full_path).get('filesystems', []): + return filesystem[0].get('options').split(',') + + def convert_to_ISO_format(self, time_string): + time_string_almost_done = time_string.replace(' ', 'T', 1).replace(' ', '') + iso_string = f"{time_string_almost_done[:-2]}:{time_string_almost_done[-2:]}" + return iso_string + + def mount(self, mountpoint :pathlib.Path, options=None, include_previously_known_options=True): + from ..helpers import findmnt + + try: + if mnt_info := findmnt(pathlib.Path(mountpoint), traverse=False): + log(f"Unmounting {mountpoint} as it was already mounted using {mnt_info}") + SysCommand(f"umount {mountpoint}") + except DiskError: + # No previously mounted device at the mountpoint + pass + + if not options: + options = [] + + try: + if include_previously_known_options and (cached_options := self.mount_options): + options += cached_options + except DiskError: + pass + + if not any('subvol=' in x for x in options): + options += f'subvol={self.name}' + + SysCommand(f"mount {self.partition.path} {mountpoint} -o {','.join(options)}") + log(f"{self} has successfully been mounted to {mountpoint}", level=logging.INFO, fg="gray") + + def unmount(self, recurse :bool = True): + SysCommand(f"umount {'-R' if recurse else ''} {self.full_path}") + log(f"Successfully unmounted {self}", level=logging.INFO, fg="gray") \ No newline at end of file diff --git a/archinstall/lib/disk/filesystem.py b/archinstall/lib/disk/filesystem.py index 31929b63..8d8f596e 100644 --- a/archinstall/lib/disk/filesystem.py +++ b/archinstall/lib/disk/filesystem.py @@ -65,6 +65,7 @@ class Filesystem: def load_layout(self, layout :Dict[str, Any]) -> None: from ..luks import luks2 + from .btrfs import BTRFSPartition # If the layout tells us to wipe the drive, we do so if layout.get('wipe', False): @@ -142,12 +143,24 @@ class Filesystem: break unlocked_device.format(partition['filesystem']['format'], options=format_options) + elif partition.get('wipe', False): if not partition['device_instance']: raise DiskError(f"Internal error caused us to loose the partition. Please report this issue upstream!") partition['device_instance'].format(partition['filesystem']['format'], options=format_options) + if partition['filesystem']['format'] == 'btrfs': + # We upgrade the device instance to a BTRFSPartition if we format it as such. + # This is so that we can gain access to more features than otherwise available in Partition() + partition['device_instance'] = BTRFSPartition( + partition['device_instance'].path, + block_device=partition['device_instance'].block_device, + encrypted=False, + filesystem='btrfs', + autodetect_filesystem=False + ) + if partition.get('boot', False): log(f"Marking partition {partition['device_instance']} as bootable.") self.set(self.partuuid_to_index(partition['device_instance'].part_uuid), 'boot on') diff --git a/archinstall/lib/disk/helpers.py b/archinstall/lib/disk/helpers.py index 0799cd49..99856aad 100644 --- a/archinstall/lib/disk/helpers.py +++ b/archinstall/lib/disk/helpers.py @@ -291,11 +291,37 @@ def find_mountpoint(device_path :str) -> Dict[str, Any]: except SysCallError: return {} -def get_mount_info(path :Union[pathlib.Path, str], traverse :bool = False, return_real_path :bool = False) -> Dict[str, Any]: +def findmnt(path :pathlib.Path, traverse :bool = False, ignore :List = [], recurse :bool = True) -> Dict[str, Any]: + for traversal in list(map(str, [str(path)] + list(path.parents))): + if traversal in ignore: + continue + + try: + log(f"Getting mount information for device path {traversal}", level=logging.DEBUG) + if (output := SysCommand(f"/usr/bin/findmnt --json {'--submounts' if recurse else ''} {traversal}").decode('UTF-8')): + return json.loads(output) + + except SysCallError as error: + log(f"Could not get mount information on {path} but continuing and ignoring: {error}", level=logging.INFO, fg="gray") + pass + + if not traverse: + break + + raise DiskError(f"Could not get mount information for path {path}") + + +def get_mount_info(path :Union[pathlib.Path, str], traverse :bool = False, return_real_path :bool = False, ignore :List = []) -> Dict[str, Any]: + import traceback + + log(f"Deprecated: archinstall.get_mount_info(). Use archinstall.findmnt() instead, which does not do any automatic parsing. Please change at:\n{''.join(traceback.format_stack())}") device_path, bind_path = split_bind_name(path) output = {} for traversal in list(map(str, [str(device_path)] + list(pathlib.Path(str(device_path)).parents))): + if traversal in ignore: + continue + try: log(f"Getting mount information for device path {traversal}", level=logging.DEBUG) if (output := SysCommand(f'/usr/bin/findmnt --json {traversal}').decode('UTF-8')): @@ -385,9 +411,8 @@ def get_partitions_in_use(mountpoint :str) -> List[Partition]: def get_filesystem_type(path :str) -> Optional[str]: - device_name, bind_name = split_bind_name(path) try: - return SysCommand(f"blkid -o value -s TYPE {device_name}").decode('UTF-8').strip() + return SysCommand(f"blkid -o value -s TYPE {path}").decode('UTF-8').strip() except SysCallError: return None diff --git a/archinstall/lib/disk/mapperdev.py b/archinstall/lib/disk/mapperdev.py index 32e3ac9b..67230012 100644 --- a/archinstall/lib/disk/mapperdev.py +++ b/archinstall/lib/disk/mapperdev.py @@ -51,11 +51,11 @@ class MapperDev: raise ValueError(f"Could not convert {self.mappername} to a real dm-crypt device") @property - def mountpoint(self) -> Optional[str]: + def mountpoint(self) -> Optional[pathlib.Path]: try: data = json.loads(SysCommand(f"findmnt --json -R {self.path}").decode()) for filesystem in data['filesystems']: - return filesystem.get('target') + return pathlib.Path(filesystem.get('target')) except SysCallError as error: # Not mounted anywhere most likely @@ -76,8 +76,8 @@ class MapperDev: @property def subvolumes(self) -> Iterator['BtrfsSubvolume']: - from .btrfs import get_subvolumes_from_findmnt + from .btrfs import subvolume_info_from_path for mountpoint in self.mount_information: - for result in get_subvolumes_from_findmnt(mountpoint): - yield result \ No newline at end of file + if subvolume := subvolume_info_from_path(mountpoint): + yield subvolume \ No newline at end of file diff --git a/archinstall/lib/disk/partition.py b/archinstall/lib/disk/partition.py index c52ca434..e33c600c 100644 --- a/archinstall/lib/disk/partition.py +++ b/archinstall/lib/disk/partition.py @@ -13,7 +13,8 @@ from ..storage import storage from ..exceptions import DiskError, SysCallError, UnknownFilesystemFormat from ..output import log from ..general import SysCommand -from .btrfs import get_subvolumes_from_findmnt, BtrfsSubvolume +from .btrfs.btrfs_helpers import subvolume_info_from_path +from .btrfs.btrfssubvolume import BtrfsSubvolume class Partition: def __init__(self, @@ -96,7 +97,7 @@ class Partition: try: data = json.loads(SysCommand(f"findmnt --json -R {self.path}").decode()) for filesystem in data['filesystems']: - return filesystem.get('target') + return pathlib.Path(filesystem.get('target')) except SysCallError as error: # Not mounted anywhere most likely @@ -304,9 +305,26 @@ class Partition: @property def subvolumes(self) -> Iterator[BtrfsSubvolume]: + from .helpers import findmnt + + def iterate_children_recursively(information): + for child in information.get('children', []): + if target := child.get('target'): + if subvolume := subvolume_info_from_path(pathlib.Path(target)): + yield subvolume + + if child.get('children'): + for subchild in iterate_children_recursively(child): + yield subchild + for mountpoint in self.mount_information: - for result in get_subvolumes_from_findmnt(mountpoint): - yield result + if result := findmnt(pathlib.Path(mountpoint['target'])): + for filesystem in result.get('filesystems', []): + if subvolume := subvolume_info_from_path(pathlib.Path(mountpoint['target'])): + yield subvolume + + for child in iterate_children_recursively(filesystem): + yield child def partprobe(self) -> bool: try: @@ -357,7 +375,7 @@ class Partition: handle = luks2(self, None, None) return handle.encrypt(self, *args, **kwargs) - def format(self, filesystem :Optional[str] = None, path :Optional[str] = None, log_formatting :bool = True, options :List[str] = []) -> bool: + def format(self, filesystem :Optional[str] = None, path :Optional[str] = None, log_formatting :bool = True, options :List[str] = [], retry :bool = True) -> bool: """ Format can be given an overriding path, for instance /dev/null to test the formatting functionality and in essence the support for the given filesystem. @@ -379,63 +397,71 @@ class Partition: if log_formatting: log(f'Formatting {path} -> {filesystem}', level=logging.INFO) - if filesystem == 'btrfs': - options = ['-f'] + options + try: + if filesystem == 'btrfs': + options = ['-f'] + options - if 'UUID:' not in (mkfs := SysCommand(f"/usr/bin/mkfs.btrfs {' '.join(options)} {path}").decode('UTF-8')): - raise DiskError(f'Could not format {path} with {filesystem} because: {mkfs}') - self.filesystem = filesystem + if 'UUID:' not in (mkfs := SysCommand(f"/usr/bin/mkfs.btrfs {' '.join(options)} {path}").decode('UTF-8')): + raise DiskError(f'Could not format {path} with {filesystem} because: {mkfs}') + self.filesystem = filesystem - elif filesystem == 'vfat': - options = ['-F32'] + options + elif filesystem == 'vfat': + options = ['-F32'] + options - if (handle := SysCommand(f"/usr/bin/mkfs.vfat {' '.join(options)} {path}")).exit_code != 0: - raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}") - self.filesystem = filesystem + if (handle := SysCommand(f"/usr/bin/mkfs.vfat {' '.join(options)} {path}")).exit_code != 0: + raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}") + self.filesystem = filesystem - elif filesystem == 'ext4': - options = ['-F'] + options + elif filesystem == 'ext4': + options = ['-F'] + options - if (handle := SysCommand(f"/usr/bin/mkfs.ext4 {' '.join(options)} {path}")).exit_code != 0: - raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}") - self.filesystem = filesystem + if (handle := SysCommand(f"/usr/bin/mkfs.ext4 {' '.join(options)} {path}")).exit_code != 0: + raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}") + self.filesystem = filesystem - elif filesystem == 'ext2': - options = ['-F'] + options + elif filesystem == 'ext2': + options = ['-F'] + options - if (handle := SysCommand(f"/usr/bin/mkfs.ext2 {' '.join(options)} {path}")).exit_code != 0: - raise DiskError(f'Could not format {path} with {filesystem} because: {b"".join(handle)}') - self.filesystem = 'ext2' + if (handle := SysCommand(f"/usr/bin/mkfs.ext2 {' '.join(options)} {path}")).exit_code != 0: + raise DiskError(f'Could not format {path} with {filesystem} because: {b"".join(handle)}') + self.filesystem = 'ext2' - elif filesystem == 'xfs': - options = ['-f'] + options + elif filesystem == 'xfs': + options = ['-f'] + options - if (handle := SysCommand(f"/usr/bin/mkfs.xfs {' '.join(options)} {path}")).exit_code != 0: - raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}") - self.filesystem = filesystem + if (handle := SysCommand(f"/usr/bin/mkfs.xfs {' '.join(options)} {path}")).exit_code != 0: + raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}") + self.filesystem = filesystem - elif filesystem == 'f2fs': - options = ['-f'] + options + elif filesystem == 'f2fs': + options = ['-f'] + options - if (handle := SysCommand(f"/usr/bin/mkfs.f2fs {' '.join(options)} {path}")).exit_code != 0: - raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}") - self.filesystem = filesystem + if (handle := SysCommand(f"/usr/bin/mkfs.f2fs {' '.join(options)} {path}")).exit_code != 0: + raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}") + self.filesystem = filesystem - elif filesystem == 'ntfs3': - options = ['-f'] + options + elif filesystem == 'ntfs3': + options = ['-f'] + options - if (handle := SysCommand(f"/usr/bin/mkfs.ntfs -Q {' '.join(options)} {path}")).exit_code != 0: - raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}") - self.filesystem = filesystem + if (handle := SysCommand(f"/usr/bin/mkfs.ntfs -Q {' '.join(options)} {path}")).exit_code != 0: + raise DiskError(f"Could not format {path} with {filesystem} because: {handle.decode('UTF-8')}") + self.filesystem = filesystem - elif filesystem == 'crypto_LUKS': - # from ..luks import luks2 - # encrypted_partition = luks2(self, None, None) - # encrypted_partition.format(path) - self.filesystem = filesystem + elif filesystem == 'crypto_LUKS': + # from ..luks import luks2 + # encrypted_partition = luks2(self, None, None) + # encrypted_partition.format(path) + self.filesystem = filesystem - else: - raise UnknownFilesystemFormat(f"Fileformat '{filesystem}' is not yet implemented.") + else: + raise UnknownFilesystemFormat(f"Fileformat '{filesystem}' is not yet implemented.") + except SysCallError as error: + log(f"Formatting ran in to an error: {error}", level=logging.WARNING, fg="orange") + if retry is True: + log(f"Retrying in {storage.get('DISK_TIMEOUTS', 1)} seconds.", level=logging.WARNING, fg="orange") + time.sleep(storage.get('DISK_TIMEOUTS', 1)) + + return self.format(filesystem, path, log_formatting, options, retry=False) if get_filesystem_type(path) == 'crypto_LUKS' or get_filesystem_type(self.real_device) == 'crypto_LUKS': self.encrypted = True diff --git a/archinstall/lib/exceptions.py b/archinstall/lib/exceptions.py index f6f58151..a16faa3f 100644 --- a/archinstall/lib/exceptions.py +++ b/archinstall/lib/exceptions.py @@ -48,4 +48,8 @@ class PackageError(BaseException): class TranslationError(BaseException): + pass + + +class Deprecated(BaseException): pass \ No newline at end of file diff --git a/archinstall/lib/general.py b/archinstall/lib/general.py index 44b78777..b99e4a45 100644 --- a/archinstall/lib/general.py +++ b/archinstall/lib/general.py @@ -363,7 +363,7 @@ class SysCommandWorker: try: try: with open(f"{storage['LOG_PATH']}/cmd_history.txt", "a") as cmd_log: - cmd_log.write(f"{' '.join(self.cmd)}\n") + cmd_log.write(f"{self.cmd}\n") except PermissionError: pass diff --git a/archinstall/lib/installer.py b/archinstall/lib/installer.py index e3bd16d3..3e04de59 100644 --- a/archinstall/lib/installer.py +++ b/archinstall/lib/installer.py @@ -20,7 +20,6 @@ from .storage import storage # from .user_interaction import * from .output import log from .profiles import Profile -from .disk.btrfs import manage_btrfs_subvolumes from .disk.partition import get_mount_fs_type from .exceptions import DiskError, ServiceException, RequirementError, HardwareIncompatibilityError, SysCallError from .hsm import fido2_enroll @@ -233,12 +232,17 @@ class Installer: def mount_ordered_layout(self, layouts: Dict[str, Any]) -> None: from .luks import luks2 + from .disk.btrfs import setup_subvolumes, mount_subvolume + # set the partitions as a list not part of a tree (which we don't need anymore (i think) list_part = [] list_luks_handles = [] for blockdevice in layouts: list_part.extend(layouts[blockdevice]['partitions']) + # TODO: Implement a proper mount-queue system that does not depend on return values. + mount_queue = {} + # we manage the encrypted partititons for partition in [entry for entry in list_part if entry.get('encrypted', False)]: # open the luks device and all associate stuff @@ -260,32 +264,61 @@ class Installer: fido2_enroll(hsm_device_path, partition['device_instance'], password) # we manage the btrfs partitions - for partition in [entry for entry in list_part if entry.get('btrfs', {}).get('subvolumes', {})]: - if partition.get('filesystem',{}).get('mount_options',[]): - mount_options = ','.join(partition['filesystem']['mount_options']) - self.mount(partition['device_instance'], "/", options=mount_options) - else: - self.mount(partition['device_instance'], "/") - try: - new_mountpoints = manage_btrfs_subvolumes(self,partition) - except Exception as e: - # every exception unmounts the physical volume. Otherwise we let the system in an unstable state + if any(btrfs_subvolumes := [entry for entry in list_part if entry.get('btrfs', {}).get('subvolumes', {})]): + for partition in btrfs_subvolumes: + if mount_options := ','.join(partition.get('filesystem',{}).get('mount_options',[])): + self.mount(partition['device_instance'], "/", options=mount_options) + else: + self.mount(partition['device_instance'], "/") + + setup_subvolumes( + installation=self, + partition_dict=partition + ) + partition['device_instance'].unmount() - raise e - partition['device_instance'].unmount() - if new_mountpoints: - list_part.extend(new_mountpoints) - # we mount. We need to sort by mountpoint to get a good working order - for partition in sorted([entry for entry in list_part if entry.get('mountpoint',False)],key=lambda part: part['mountpoint']): + # We then handle any special cases, such as btrfs + if any(btrfs_subvolumes := [entry for entry in list_part if entry.get('btrfs', {}).get('subvolumes', {})]): + for partition_information in btrfs_subvolumes: + for name, mountpoint in sorted(partition_information['btrfs']['subvolumes'].items(), key=lambda item: item[1]): + btrfs_subvolume_information = {} + + match mountpoint: + case str(): # backwards-compatability + btrfs_subvolume_information['mountpoint'] = mountpoint + btrfs_subvolume_information['options'] = [] + case dict(): + btrfs_subvolume_information['mountpoint'] = mountpoint.get('mountpoint', None) + btrfs_subvolume_information['options'] = mountpoint.get('options', []) + case _: + continue + + if mountpoint_parsed := btrfs_subvolume_information.get('mountpoint'): + # We cache the mount call for later + mount_queue[mountpoint_parsed] = lambda device=partition_information['device_instance'], \ + name=name, \ + subvolume_information=btrfs_subvolume_information: mount_subvolume( + installation=self, + device=device, + name=name, + subvolume_information=subvolume_information + ) + + # We mount ordinary partitions, and we sort them by the mountpoint + for partition in sorted([entry for entry in list_part if entry.get('mountpoint', False)], key=lambda part: part['mountpoint']): mountpoint = partition['mountpoint'] log(f"Mounting {mountpoint} to {self.target}{mountpoint} using {partition['device_instance']}", level=logging.INFO) if partition.get('filesystem',{}).get('mount_options',[]): mount_options = ','.join(partition['filesystem']['mount_options']) - partition['device_instance'].mount(f"{self.target}{mountpoint}", options=mount_options) + mount_queue[mountpoint] = lambda target=f"{self.target}{mountpoint}", options=mount_options: partition['device_instance'].mount(target, options) else: - partition['device_instance'].mount(f"{self.target}{mountpoint}") + mount_queue[mountpoint] = lambda target=f"{self.target}{mountpoint}": partition['device_instance'].mount(target) + + # We mount everything by sorting on the mountpoint itself. + for mountpoint, frozen_func in sorted(mount_queue.items(), key=lambda item: item[0]): + frozen_func() time.sleep(1) @@ -979,10 +1012,14 @@ class Installer: if plugin.on_add_bootloader(self): return True + if type(self.target) == str: + self.target = pathlib.Path(self.target) + boot_partition = None root_partition = None for partition in self.partitions: - if partition.mountpoint == os.path.join(self.target, 'boot'): + print(partition, [partition.mountpoint], [self.target]) + if partition.mountpoint == self.target / 'boot': boot_partition = partition elif partition.mountpoint == self.target: root_partition = partition diff --git a/archinstall/lib/luks.py b/archinstall/lib/luks.py index 710af01e..ac480b11 100644 --- a/archinstall/lib/luks.py +++ b/archinstall/lib/luks.py @@ -15,7 +15,10 @@ from .general import SysCommand, SysCommandWorker from .output import log from .exceptions import SysCallError, DiskError from .storage import storage +from .disk.helpers import get_filesystem_type from .disk.mapperdev import MapperDev +from .disk.btrfs import BTRFSPartition + class luks2: def __init__(self, @@ -149,7 +152,6 @@ class luks2: :param mountpoint: The name without absolute path, for instance "luksdev" will point to /dev/mapper/luksdev :type mountpoint: str """ - from .disk import get_filesystem_type if '/' in mountpoint: os.path.basename(mountpoint) # TODO: Raise exception instead? @@ -162,14 +164,22 @@ class luks2: if os.path.islink(f'/dev/mapper/{mountpoint}'): self.mapdev = f'/dev/mapper/{mountpoint}' - unlocked_partition = Partition( + if (filesystem_type := get_filesystem_type(pathlib.Path(self.mapdev))) == 'btrfs': + return BTRFSPartition( + self.mapdev, + block_device=MapperDev(mountpoint).partition.block_device, + encrypted=True, + filesystem=filesystem_type, + autodetect_filesystem=False + ) + + return Partition( self.mapdev, block_device=MapperDev(mountpoint).partition.block_device, encrypted=True, filesystem=get_filesystem_type(self.mapdev), autodetect_filesystem=False ) - return unlocked_partition def close(self, mountpoint :Optional[str] = None) -> bool: if not mountpoint: diff --git a/archinstall/lib/output.py b/archinstall/lib/output.py index da41d16d..07747091 100644 --- a/archinstall/lib/output.py +++ b/archinstall/lib/output.py @@ -61,9 +61,11 @@ def stylize_output(text: str, *opts :str, **kwargs) -> str: 'magenta' : '5', 'cyan' : '6', 'white' : '7', - 'orange' : '8;5;208', # Extended 256-bit colors (not always supported) - 'darkorange' : '8;5;202',# https://www.lihaoyi.com/post/BuildyourownCommandLinewithANSIescapecodes.html#256-colors + 'teal' : '8;5;109', # Extended 256-bit colors (not always supported) + 'orange' : '8;5;208', # https://www.lihaoyi.com/post/BuildyourownCommandLinewithANSIescapecodes.html#256-colors + 'darkorange' : '8;5;202', 'gray' : '8;5;246', + 'grey' : '8;5;246', 'darkgray' : '8;5;240', 'lightgray' : '8;5;256' } diff --git a/archinstall/lib/systemd.py b/archinstall/lib/systemd.py index 417870da..3d2f0385 100644 --- a/archinstall/lib/systemd.py +++ b/archinstall/lib/systemd.py @@ -91,20 +91,25 @@ class Boot: log(f"The error above occured in a temporary boot-up of the installation {self.instance}", level=logging.ERROR, fg="red") shutdown = None + shutdown_exit_code = -1 try: shutdown = SysCommand(f'systemd-run --machine={self.container_name} --pty shutdown now') except SysCallError as error: - if error.exit_code == 256: - pass + shutdown_exit_code = error.exit_code + # if error.exit_code == 256: + # pass while self.session.is_alive(): time.sleep(0.25) - if self.session.exit_code == 0 or (shutdown and shutdown.exit_code == 0): + if shutdown: + shutdown_exit_code = shutdown.exit_code + + if self.session.exit_code == 0 or shutdown_exit_code == 0: storage['active_boot'] = None else: - raise SysCallError(f"Could not shut down temporary boot of {self.instance}: {shutdown}", exit_code=shutdown.exit_code) + raise SysCallError(f"Could not shut down temporary boot of {self.instance}: {self.session.exit_code}/{shutdown_exit_code}", exit_code=next(filter(bool, [self.session.exit_code, shutdown_exit_code]))) def __iter__(self) -> Iterator[str]: if self.session: -- cgit v1.2.3-70-g09d2