From 40e4046633d857929b8fd1bfa121b38937add81c Mon Sep 17 00:00:00 2001 From: Daniel Girtler Date: Tue, 4 Oct 2022 01:37:57 +1100 Subject: Exclude archiso devices from selectable blockdevices (#1481) * Exclude liveusb from disk selection * Exclude arch iso from being an option * Update * Update * Update * Mypy Co-authored-by: Daniel Girtler --- archinstall/lib/disk/diskinfo.py | 40 +++++++++++++++++++++++++ archinstall/lib/disk/helpers.py | 27 +++++++++++++---- archinstall/lib/menu/global_menu.py | 2 +- archinstall/lib/user_interaction/system_conf.py | 7 +---- 4 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 archinstall/lib/disk/diskinfo.py (limited to 'archinstall/lib') diff --git a/archinstall/lib/disk/diskinfo.py b/archinstall/lib/disk/diskinfo.py new file mode 100644 index 00000000..b56ba282 --- /dev/null +++ b/archinstall/lib/disk/diskinfo.py @@ -0,0 +1,40 @@ +import dataclasses +import json +from dataclasses import dataclass, field +from typing import Optional, List + +from ..general import SysCommand +from ..exceptions import DiskError + +@dataclass +class LsblkInfo: + size: int = 0 + log_sec: int = 0 + pttype: Optional[str] = None + rota: bool = False + tran: Optional[str] = None + ptuuid: Optional[str] = None + partuuid: Optional[str] = None + uuid: Optional[str] = None + fstype: Optional[str] = None + type: Optional[str] = None + mountpoints: List[str] = field(default_factory=list) + + +def get_lsblk_info(dev_path: str) -> LsblkInfo: + fields = [f.name for f in dataclasses.fields(LsblkInfo)] + lsblk_fields = ','.join([f.upper().replace('_', '-') for f in fields]) + + output = SysCommand(f'lsblk --json -b -o+{lsblk_fields} {dev_path}').decode('UTF-8') + + if output: + block_devices = json.loads(output) + info = block_devices['blockdevices'][0] + lsblk_info = LsblkInfo() + + for f in fields: + setattr(lsblk_info, f, info[f.replace('_', '-')]) + + return lsblk_info + + raise DiskError(f'Failed to read disk "{dev_path}" with lsblk') diff --git a/archinstall/lib/disk/helpers.py b/archinstall/lib/disk/helpers.py index f19125f4..256f7abb 100644 --- a/archinstall/lib/disk/helpers.py +++ b/archinstall/lib/disk/helpers.py @@ -6,13 +6,12 @@ import pathlib import re import time import glob + from typing import Union, List, Iterator, Dict, Optional, Any, TYPE_CHECKING # https://stackoverflow.com/a/39757388/929999 +from .diskinfo import get_lsblk_info from ..models.subvolume import Subvolume -if TYPE_CHECKING: - from .partition import Partition - from .blockdevice import BlockDevice from .dmcryptdev import DMCryptDev from .mapperdev import MapperDev @@ -21,6 +20,10 @@ from ..general import SysCommand from ..output import log from ..storage import storage +if TYPE_CHECKING: + from .partition import Partition + + ROOT_DIR_PATTERN = re.compile('^.*?/devices') GIGA = 2 ** 30 @@ -204,11 +207,18 @@ def get_blockdevice_uevent(dev_name :str) -> Dict[str, Any]: } } + def all_disks() -> List[BlockDevice]: log(f"[Deprecated] archinstall.all_disks() is deprecated. Use archinstall.all_blockdevices() with the appropriate filters instead.", level=logging.WARNING, fg="yellow") return all_blockdevices(partitions=False, mappers=False) -def all_blockdevices(mappers=False, partitions=False, error=False) -> Dict[str, Any]: + +def all_blockdevices( + mappers: bool = False, + partitions: bool = False, + error: bool = False, + exclude_iso_dev: bool = True +) -> Dict[str, Any]: """ Returns BlockDevice() and Partition() objects for all available devices. """ @@ -227,6 +237,13 @@ def all_blockdevices(mappers=False, partitions=False, error=False) -> Dict[str, continue try: + if exclude_iso_dev: + # exclude all devices associated with the iso boot locations + iso_devs = ['/run/archiso/airootfs', '/run/archiso/bootmnt'] + lsblk_info = get_lsblk_info(device_path) + if any([dev in lsblk_info.mountpoints for dev in iso_devs]): + continue + information = blkid(f'blkid -p -o export {device_path}') except SysCallError as ex: if ex.exit_code in (512, 2): @@ -416,7 +433,7 @@ def get_partitions_in_use(mountpoint :str) -> Dict[str, Any]: # Since all_blockdevices() returns PosixPath objects, we need to convert # findmnt paths to pathlib.Path() first: mountpoint = pathlib.Path(mountpoint) - + if mountpoint in block_devices_mountpoints: if mountpoint not in mounts: mounts[mountpoint] = block_devices_mountpoints[mountpoint] diff --git a/archinstall/lib/menu/global_menu.py b/archinstall/lib/menu/global_menu.py index d1bec189..444ba7ee 100644 --- a/archinstall/lib/menu/global_menu.py +++ b/archinstall/lib/menu/global_menu.py @@ -341,7 +341,7 @@ class GlobalMenu(GeneralMenu): return ntp - def _select_harddrives(self, old_harddrives : list) -> List: + def _select_harddrives(self, old_harddrives: List[str] = []) -> List: harddrives = select_harddrives(old_harddrives) if harddrives is not None: diff --git a/archinstall/lib/user_interaction/system_conf.py b/archinstall/lib/user_interaction/system_conf.py index 0416e91f..8b574b2c 100644 --- a/archinstall/lib/user_interaction/system_conf.py +++ b/archinstall/lib/user_interaction/system_conf.py @@ -52,11 +52,6 @@ def select_harddrives(preset: List[str] = []) -> List[str]: hard_drives = all_blockdevices(partitions=False).values() options = {f'{option}': option for option in hard_drives} - if preset: - preset_disks = {f'{option}': option for option in preset} - else: - preset_disks = {} - title = str(_('Select one or more hard drives to use and configure\n')) title += str(_('Any modifications to the existing setting will reset the disk layout!')) @@ -65,7 +60,7 @@ def select_harddrives(preset: List[str] = []) -> List[str]: selected_harddrive = Menu( title, list(options.keys()), - preset_values=list(preset_disks.keys()), + preset_values=preset, multi=True, raise_error_on_interrupt=True, raise_error_warning_msg=warning -- cgit v1.2.3-70-g09d2 From 977976f88e761ced7d5bc9f39b32e2296ee91806 Mon Sep 17 00:00:00 2001 From: codefiles <11915375+codefiles@users.noreply.github.com> Date: Mon, 3 Oct 2022 10:42:10 -0400 Subject: Add hostname and locales as parameters to `minimal_installation()` (#1458) --- README.md | 3 +-- archinstall/lib/installer.py | 8 +++++--- examples/guided.py | 6 +++--- examples/swiss.py | 6 +++--- 4 files changed, 12 insertions(+), 11 deletions(-) (limited to 'archinstall/lib') diff --git a/README.md b/README.md index 72197ada..6bddfa24 100644 --- a/README.md +++ b/README.md @@ -122,8 +122,7 @@ with archinstall.luks2(root, 'luksloop', disk_password) as unlocked_root: boot.mount('/mnt/boot') with archinstall.Installer('/mnt') as installation: - if installation.minimal_installation(): - installation.set_hostname('minimal-arch') + if installation.minimal_installation(hostname='minimal-arch'): installation.add_bootloader() installation.add_additional_packages(['nano', 'wget', 'git']) diff --git a/archinstall/lib/installer.py b/archinstall/lib/installer.py index 1270959e..49ce4d7f 100644 --- a/archinstall/lib/installer.py +++ b/archinstall/lib/installer.py @@ -662,7 +662,9 @@ class Installer: return SysCommand(f'/usr/bin/arch-chroot {self.target} mkinitcpio {" ".join(flags)}').exit_code == 0 - def minimal_installation(self, testing=False, multilib=False) -> bool: + def minimal_installation( + self, testing: bool = False, multilib: bool = False, + hostname: str = 'archinstall', locales: List[str] = ['en_US.UTF-8 UTF-8']) -> bool: # Add necessary packages if encrypting the drive # (encrypted partitions default to btrfs for now, so we need btrfs-progs) # TODO: Perhaps this should be living in the function which dictates @@ -750,8 +752,8 @@ class Installer: # os.remove(f'{self.target}/etc/localtime') # sys_command(f'/usr/bin/arch-chroot {self.target} ln -s /usr/share/zoneinfo/{localtime} /etc/localtime') # sys_command('/usr/bin/arch-chroot /mnt hwclock --hctosys --localtime') - self.set_hostname('archinstall') - self.set_locale('en_US') + self.set_hostname(hostname) + self.set_locale(*locales[0].split()) # TODO: Use python functions for this SysCommand(f'/usr/bin/arch-chroot {self.target} chmod 700 /root') diff --git a/examples/guided.py b/examples/guided.py index 6f289caa..ad178207 100644 --- a/examples/guided.py +++ b/examples/guided.py @@ -192,9 +192,9 @@ def perform_installation(mountpoint): enable_testing = False enable_multilib = False - if installation.minimal_installation(testing=enable_testing, multilib=enable_multilib): - installation.set_locale(archinstall.arguments['sys-language'], archinstall.arguments['sys-encoding'].upper()) - installation.set_hostname(archinstall.arguments['hostname']) + if installation.minimal_installation( + testing=enable_testing, multilib=enable_multilib, hostname=archinstall.arguments['hostname'], + locales=[f"{archinstall.arguments['sys-language']} {archinstall.arguments['sys-encoding'].upper()}"]): if archinstall.arguments.get('mirror-region') is not None: if archinstall.arguments.get("mirrors", None) is not None: installation.set_mirrors(archinstall.arguments['mirror-region']) # Set the mirrors in the installation medium diff --git a/examples/swiss.py b/examples/swiss.py index da45cd18..419bd859 100644 --- a/examples/swiss.py +++ b/examples/swiss.py @@ -402,9 +402,9 @@ def os_setup(installation): # Set mirrors used by pacstrap (outside of installation) if archinstall.arguments.get('mirror-region', None): archinstall.use_mirrors(archinstall.arguments['mirror-region']) # Set the mirrors for the live medium - if installation.minimal_installation(): - installation.set_locale(archinstall.arguments['sys-language'], archinstall.arguments['sys-encoding'].upper()) - installation.set_hostname(archinstall.arguments['hostname']) + if installation.minimal_installation( + hostname=archinstall.arguments['hostname'], + locales=[f"{archinstall.arguments['sys-language']} {archinstall.arguments['sys-encoding'].upper()}"]): if archinstall.arguments['mirror-region'].get("mirrors", None) is not None: installation.set_mirrors( archinstall.arguments['mirror-region']) # Set the mirrors in the installation medium -- cgit v1.2.3-70-g09d2 From eec45643e950a5b982e1478ef57497491ec37f73 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Wed, 12 Oct 2022 14:17:14 +0200 Subject: Updating Partition() information after mount and unmount. (#1508) * Updating Partition() information after mount and unmount. * Cleaned up raw_parted() to gracefully output relevant partition error information. * Adding timestmap to cmd_history.txt as it's impossible to debug properly otherwise * Adding more verbose debugging information * Reinstating the lsblk retry code for PARTUUID and UUID on Partition()'s information * Added error handling for JSON parsing --- .gitignore | 3 +- archinstall/lib/disk/blockdevice.py | 4 +++ archinstall/lib/disk/filesystem.py | 15 ++++++--- archinstall/lib/disk/partition.py | 63 ++++++++++++++++++++++++++++++------- archinstall/lib/general.py | 2 +- 5 files changed, 70 insertions(+), 17 deletions(-) (limited to 'archinstall/lib') diff --git a/.gitignore b/.gitignore index 24b1b697..40e00e87 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,5 @@ venv .DS_Store **/cmd_history.txt **/*.*~ -/*.sig \ No newline at end of file +/*.sig +/*.json \ No newline at end of file diff --git a/archinstall/lib/disk/blockdevice.py b/archinstall/lib/disk/blockdevice.py index 736bacbc..cd0e14a6 100644 --- a/archinstall/lib/disk/blockdevice.py +++ b/archinstall/lib/disk/blockdevice.py @@ -274,12 +274,16 @@ class BlockDevice: if not uuid and not partuuid: raise ValueError(f"BlockDevice.get_partition() requires either a UUID or a PARTUUID for lookups.") + log(f"Retrieving partition PARTUUID={partuuid} or UUID={uuid}", level=logging.INFO, fg="teal") + for count in range(storage.get('DISK_RETRY_ATTEMPTS', 5)): for partition_index, partition in self.partitions.items(): try: if uuid and partition.uuid and partition.uuid.lower() == uuid.lower(): + log(f"Matched UUID={uuid} against {partition.uuid}", level=logging.INFO, fg="teal") return partition elif partuuid and partition.part_uuid and partition.part_uuid.lower() == partuuid.lower(): + log(f"Matched PARTUUID={partuuid} against {partition.part_uuid}", level=logging.INFO, fg="teal") return partition except DiskError as error: # Most likely a blockdevice that doesn't support or use UUID's diff --git a/archinstall/lib/disk/filesystem.py b/archinstall/lib/disk/filesystem.py index 5d5952a0..c8eaf0be 100644 --- a/archinstall/lib/disk/filesystem.py +++ b/archinstall/lib/disk/filesystem.py @@ -189,10 +189,13 @@ class Filesystem: return True def raw_parted(self, string: str) -> SysCommand: - if (cmd_handle := SysCommand(f'/usr/bin/parted -s {string}')).exit_code != 0: - log(f"Parted ended with a bad exit code: {cmd_handle}", level=logging.ERROR, fg="red") - time.sleep(0.5) - return cmd_handle + try: + cmd_handle = SysCommand(f'/usr/bin/parted -s {string}') + time.sleep(0.5) + return cmd_handle + except SysCallError as error: + log(f"Parted ended with a bad exit code: {error.exit_code} ({error})", level=logging.ERROR, fg="red") + return error def parted(self, string: str) -> bool: """ @@ -258,6 +261,9 @@ class Filesystem: new_partition_uuids = [partition.part_uuid for partition in self.blockdevice.partitions.values()] new_partuuid_set = (set(previous_partuuids) ^ set(new_partition_uuids)) + log(f'Old partition set: {previous_partuuids}', level=logging.INFO, fg="teal") + log(f'New partition set: {new_partition_uuids}', level=logging.INFO, fg="teal") + if len(new_partuuid_set) and (new_partuuid := new_partuuid_set.pop()): try: return self.blockdevice.get_partition(partuuid=new_partuuid) @@ -282,6 +288,7 @@ class Filesystem: log(f"Could not find the new PARTUUID after adding the partition.", level=logging.ERROR, fg="red") log(f"Previous partitions: {previous_partuuids}", level=logging.ERROR, fg="red") log(f"New partitions: {total_partitions}", level=logging.ERROR, fg="red") + raise DiskError(f"Could not add partition using: {parted_string}") def set_name(self, partition: int, name: str) -> bool: diff --git a/archinstall/lib/disk/partition.py b/archinstall/lib/disk/partition.py index 56a7d436..04d33453 100644 --- a/archinstall/lib/disk/partition.py +++ b/archinstall/lib/disk/partition.py @@ -5,7 +5,7 @@ import json import os import hashlib import typing -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Optional, Dict, Any, List, Union, Iterator @@ -18,19 +18,51 @@ from ..general import SysCommand from .btrfs.btrfs_helpers import subvolume_info_from_path from .btrfs.btrfssubvolumeinfo import BtrfsSubvolumeInfo - @dataclass class PartitionInfo: - pttype: str - partuuid: str - uuid: str - start: Optional[int] - end: Optional[int] + partition_object: 'Partition' + device_path: str # This would be /dev/sda1 for instance bootable: bool size: float sector_size: int - filesystem_type: str - mountpoints: List[Path] + start: Optional[int] + end: Optional[int] + pttype: Optional[str] + filesystem_type: Optional[str] + partuuid: Optional[str] + uuid: Optional[str] + mountpoints: List[Path] = field(default_factory=list) + + def __post_init__(self): + if not all([self.partuuid, self.uuid]): + for i in range(storage['DISK_RETRY_ATTEMPTS']): + lsblk_info = SysCommand(f"lsblk --json -b -o+LOG-SEC,SIZE,PTTYPE,PARTUUID,UUID,FSTYPE {self.device_path}").decode('UTF-8') + try: + lsblk_info = json.loads(lsblk_info) + except json.decoder.JSONDecodeError: + log(f"Could not decode JSON: {lsblk_info}", fg="red", level=logging.ERROR) + raise DiskError(f'Failed to retrieve information for "{self.device_path}" with lsblk') + + if not (device := lsblk_info.get('blockdevices', [None])[0]): + raise DiskError(f'Failed to retrieve information for "{self.device_path}" with lsblk') + + self.partuuid = device.get('partuuid') + self.uuid = device.get('uuid') + + # Lets build a list of requirements that we would like + # to retry and build (stuff that can take time between partprobes) + requirements = [] + requirements.append(self.partuuid) + + # Unformatted partitions won't have a UUID + if lsblk_info.get('fstype') is not None: + requirements.append(self.uuid) + + if all(requirements): + break + + self.partition_object.partprobe() + time.sleep(max(0.1, storage['DISK_TIMEOUTS'] * i)) def get_first_mountpoint(self) -> Optional[Path]: if len(self.mountpoints) > 0: @@ -154,8 +186,11 @@ class Partition: output = error.worker.decode('UTF-8') if output: - lsblk_info = json.loads(output) - return lsblk_info + try: + lsblk_info = json.loads(output) + return lsblk_info + except json.decoder.JSONDecodeError: + log(f"Could not decode JSON: {output}", fg="red", level=logging.ERROR) raise DiskError(f'Failed to read disk "{self.device_path}" with lsblk') @@ -185,6 +220,8 @@ class Partition: bootable = sfdisk_info.get('bootable', False) or sfdisk_info.get('type', '') == 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B' return PartitionInfo( + partition_object=self, + device_path=self._path, pttype=device['pttype'], partuuid=device['partuuid'], uuid=device['uuid'], @@ -568,6 +605,8 @@ class Partition: except SysCallError as err: raise err + # Update the partition info since the mount info has changed after this call. + self._partition_info = self._fetch_information() return True return False @@ -582,6 +621,8 @@ class Partition: if exit_code and 0 < exit_code < 8000: raise SysCallError(f"Could not unmount {self._path} properly: {worker}", exit_code=exit_code) + # Update the partition info since the mount info has changed after this call. + self._partition_info = self._fetch_information() return True def filesystem_supported(self) -> bool: diff --git a/archinstall/lib/general.py b/archinstall/lib/general.py index d76b7036..8c7aed91 100644 --- a/archinstall/lib/general.py +++ b/archinstall/lib/general.py @@ -379,7 +379,7 @@ class SysCommandWorker: try: with history_logfile.open("a") as cmd_log: - cmd_log.write(f"{self.cmd}\n") + cmd_log.write(f"{time.time()} {self.cmd}\n") if change_perm: os.chmod(str(history_logfile), stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP) -- cgit v1.2.3-70-g09d2 From db32bc4dda8d4632d6816ac09407327c97a64518 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Wed, 12 Oct 2022 15:07:11 +0200 Subject: Muted a log output --- archinstall/lib/disk/blockdevice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'archinstall/lib') diff --git a/archinstall/lib/disk/blockdevice.py b/archinstall/lib/disk/blockdevice.py index cd0e14a6..b7f92a57 100644 --- a/archinstall/lib/disk/blockdevice.py +++ b/archinstall/lib/disk/blockdevice.py @@ -274,7 +274,7 @@ class BlockDevice: if not uuid and not partuuid: raise ValueError(f"BlockDevice.get_partition() requires either a UUID or a PARTUUID for lookups.") - log(f"Retrieving partition PARTUUID={partuuid} or UUID={uuid}", level=logging.INFO, fg="teal") + log(f"Retrieving partition PARTUUID={partuuid} or UUID={uuid}", level=logging.DEBUG, fg="gray") for count in range(storage.get('DISK_RETRY_ATTEMPTS', 5)): for partition_index, partition in self.partitions.items(): -- cgit v1.2.3-70-g09d2 From 14be423a6cf3ae6f9ac25c4da138911efc59f7fd Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Wed, 12 Oct 2022 15:07:44 +0200 Subject: Muted a log output --- archinstall/lib/disk/blockdevice.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'archinstall/lib') diff --git a/archinstall/lib/disk/blockdevice.py b/archinstall/lib/disk/blockdevice.py index b7f92a57..178b786a 100644 --- a/archinstall/lib/disk/blockdevice.py +++ b/archinstall/lib/disk/blockdevice.py @@ -280,10 +280,10 @@ class BlockDevice: for partition_index, partition in self.partitions.items(): try: if uuid and partition.uuid and partition.uuid.lower() == uuid.lower(): - log(f"Matched UUID={uuid} against {partition.uuid}", level=logging.INFO, fg="teal") + log(f"Matched UUID={uuid} against {partition.uuid}", level=logging.DEBUG, fg="gray") return partition elif partuuid and partition.part_uuid and partition.part_uuid.lower() == partuuid.lower(): - log(f"Matched PARTUUID={partuuid} against {partition.part_uuid}", level=logging.INFO, fg="teal") + log(f"Matched PARTUUID={partuuid} against {partition.part_uuid}", level=logging.DEBUG, fg="gray") return partition except DiskError as error: # Most likely a blockdevice that doesn't support or use UUID's -- cgit v1.2.3-70-g09d2 From 4addd2d7cbc015ec19c155d1d4a4bb997c05db00 Mon Sep 17 00:00:00 2001 From: Anton Hvornum Date: Wed, 12 Oct 2022 15:08:48 +0200 Subject: Removed debug output --- archinstall/lib/disk/filesystem.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'archinstall/lib') diff --git a/archinstall/lib/disk/filesystem.py b/archinstall/lib/disk/filesystem.py index c8eaf0be..af5879aa 100644 --- a/archinstall/lib/disk/filesystem.py +++ b/archinstall/lib/disk/filesystem.py @@ -261,9 +261,6 @@ class Filesystem: new_partition_uuids = [partition.part_uuid for partition in self.blockdevice.partitions.values()] new_partuuid_set = (set(previous_partuuids) ^ set(new_partition_uuids)) - log(f'Old partition set: {previous_partuuids}', level=logging.INFO, fg="teal") - log(f'New partition set: {new_partition_uuids}', level=logging.INFO, fg="teal") - if len(new_partuuid_set) and (new_partuuid := new_partuuid_set.pop()): try: return self.blockdevice.get_partition(partuuid=new_partuuid) -- cgit v1.2.3-70-g09d2