import glob, re, os, json, time, hashlib import pathlib, traceback from collections import OrderedDict from .exceptions import DiskError from .general import * from .output import log, LOG_LEVELS from .storage import storage ROOT_DIR_PATTERN = re.compile('^.*?/devices') GPT = 0b00000001 MBR = 0b00000010 #import ctypes #import ctypes.util #libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True) #libc.mount.argtypes = (ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_ulong, ctypes.c_char_p) class BlockDevice(): def __init__(self, path, info=None): if not info: # If we don't give any information, we need to auto-fill it. # Otherwise any subsequent usage will break. info = all_disks()[path].info self.path = path self.info = info self.keep_partitions = True self.part_cache = OrderedDict() # TODO: Currently disk encryption is a BIT misleading. # It's actually partition-encryption, but for future-proofing this # I'm placing the encryption password on a BlockDevice level. self.encryption_password = None def __repr__(self, *args, **kwargs): return f"BlockDevice({self.device})" def __iter__(self): for partition in self.partitions: yield self.partitions[partition] def __getitem__(self, key, *args, **kwargs): if key not in self.info: raise KeyError(f'{self} does not contain information: "{key}"') return self.info[key] def json(self): """ json() has precedence over __dump__, so this is a way to give less/partial information for user readability. """ return { 'path' : self.path, 'size' : self.info['size'] if 'size' in self.info else '', 'model' : self.info['model'] if 'model' in self.info else '' } def __dump__(self): return { 'path': self.path, 'info': self.info, 'partition_cache': self.part_cache } @property def device(self): """ Returns the actual device-endpoint of the BlockDevice. If it's a loop-back-device it returns the back-file, If it's a ATA-drive it returns the /dev/X device And if it's a crypto-device it returns the parent device """ if "type" not in self.info: raise DiskError(f'Could not locate backplane info for "{self.path}"') if self.info['type'] == 'loop': for drive in json.loads(b''.join(sys_command(f'losetup --json', hide_from_log=True)).decode('UTF_8'))['loopdevices']: if not drive['name'] == self.path: continue return drive['back-file'] elif self.info['type'] == 'disk': return self.path elif self.info['type'][:4] == 'raid': # This should catch /dev/md## raid devices return self.path elif self.info['type'] == 'crypt': if 'pkname' not in self.info: raise DiskError(f'A crypt device ({self.path}) without a parent kernel device name.') return f"/dev/{self.info['pkname']}" else: log(f"Unknown blockdevice type for {self.path}: {self.info['type']}", level=LOG_LEVELS.Debug) # if not stat.S_ISBLK(os.stat(full_path).st_mode): # raise DiskError(f'Selected disk "{full_path}" is not a block device.') @property def partitions(self): o = b''.join(sys_command(f'partprobe {self.path}')) #o = b''.join(sys_command('/usr/bin/lsblk -o name -J -b {dev}'.format(dev=dev))) o = b''.join(sys_command(f'/usr/bin/lsblk -J {self.path}')) if b'not a block device' in o: raise DiskError(f'Can not read partitions off something that isn\'t a block device: {self.path}') if not o[:1] == b'{': raise DiskError(f'Error getting JSON output from:', f'/usr/bin/lsblk -J {self.path}') r = json.loads(o.decode('UTF-8')) if len(r['blockdevices']) and 'children' in r['blockdevices'][0]: root_path = f"/dev/{r['blockdevices'][0]['name']}" for part in r['blockdevices'][0]['children']: part_id = part['name'][len(os.path.basename(self.path)):] if part_id not in self.part_cache: ## TODO: Force over-write even if in cache? if part_id not in self.part_cache or self.part_cache[part_id].size != part['size']: self.part_cache[part_id] = Partition(root_path + part_id, self, part_id=part_id, size=part['size']) return {k: self.part_cache[k] for k in sorted(self.part_cache)} @property def partition(self): all_partitions = self.partitions return [all_partitions[k] for k in all_partitions] @property def partition_table_type(self): return GPT @property def uuid(self): log(f'BlockDevice().uuid is untested!', level=LOG_LEVELS.Warning, fg='yellow') """ Returns the disk UUID as returned by lsblk. This is more reliable than relying on /dev/disk/by-partuuid as it doesn't seam to be able to detect md raid partitions. """ lsblk = b''.join(sys_command(f'lsblk -J -o+UUID {self.path}')) for partition in json.loads(lsblk.decode('UTF-8'))['blockdevices']: return partition.get('uuid', None) def has_partitions(self): return len(self.partitions) def has_mount_point(self, mountpoint): for partition in self.partitions: if self.partitions[partition].mountpoint == mountpoint: return True return False def flush_cache(self): self.part_cache = OrderedDict() class Partition(): def __init__(self, path :str, block_device :BlockDevice, part_id=None, size=-1, filesystem=None, mountpoint=None, encrypted=False, autodetect_filesystem=True): if not part_id: part_id = os.path.basename(path) self.block_device = block_device self.path = path self.part_id = part_id self.mountpoint = mountpoint self.target_mountpoint = mountpoint self.filesystem = filesystem self.size = size # TODO: Refresh? self._encrypted = None self.encrypted = encrypted self.allow_formatting = False # A fail-safe for unconfigured partitions, such as windows NTFS partitions. if mountpoint: self.mount(mountpoint) mount_information = get_mount_info(self.path) if self.mountpoint != mount_information.get('target', None) and mountpoint: raise DiskError(f"{self} was given a mountpoint but the actual mountpoint differs: {mount_information.get('target', None)}") if (target := mount_information.get('target', None)): self.mountpoint = target if not self.filesystem and autodetect_filesystem: if (fstype := mount_information.get('fstype', get_filesystem_type(path))): self.filesystem = fstype if self.filesystem == 'crypto_LUKS': self.encrypted = True def __lt__(self, left_comparitor): if type(left_comparitor) == Partition: left_comparitor = left_comparitor.path else: left_comparitor = str(left_comparitor) return self.path < left_comparitor # Not quite sure the order here is correct. But /dev/nvme0n1p1 comes before /dev/nvme0n1p5 so seems correct. def __repr__(self, *args, **kwargs): mount_repr = '' if self.mountpoint: mount_repr = f", mounted={self.mountpoint}" elif self.target_mountpoint: mount_repr = f", rel_mountpoint={self.target_mountpoint}" if self._encrypted: return f'Partition(path={self.path}, real_device={self.real_device}, fs={self.filesystem}{mount_repr})' else: return f'Partition(path={self.path}, fs={self.filesystem}{mount_repr})' @property def uuid(self) -> str: """ Returns the PARTUUID as returned by lsblk. This is more reliable than relying on /dev/disk/by-partuuid as it doesn't seam to be able to detect md raid partitions. """ lsblk = b''.join(sys_command(f'lsblk -J -o+PARTUUID {self.path}')) for partition in json.loads(lsblk.decode('UTF-8'))['blockdevices']: return partition.get('partuuid', None) @property def encrypted(self): return self._encrypted @encrypted.setter def encrypted(self, value :bool): if value: log(f'Marking {self} as encrypted: {value}', level=LOG_LEVELS.Debug) log(f"Callstrack when marking the partition: {''.join(traceback.format_stack())}", level=LOG_LEVELS.Debug) self._encrypted = value @property def parent(self): return self.real_device @property def real_device(self): for blockdevice in json.loads(b''.join(sys_command('lsblk -J')).decode('UTF-8'))['blockdevices']: if (parent := self.find_parent_of(blockdevice, os.path.basename(self.path))): return f"/dev/{parent}" # raise DiskError(f'Could not find appropriate parent for encrypted partition {self}') return self.path def detect_inner_filesystem(self, password): log(f'Trying to detect inner filesystem format on {self} (This might take a while)', level=LOG_LEVELS.Info) from .luks import luks2 try: with luks2(self, 'luksloop', password, auto_unmount=True) as unlocked_device: return unlocked_device.filesystem except SysCallError: return None def has_content(self): if not get_filesystem_type(self.path): return False temporary_mountpoint = '/tmp/'+hashlib.md5(bytes(f"{time.time()}", 'UTF-8')+os.urandom(12)).hexdigest() temporary_path = pathlib.Path(temporary_mountpoint) temporary_path.mkdir(parents=True, exist_ok=True) if (handle := sys_command(f'/usr/bin/mount {self.path} {temporary_mountpoint}')).exit_code != 0: raise DiskError(f'Could not mount and check for content on {self.path} because: {b"".join(handle)}') files = len(glob.glob(f"{temporary_mountpoint}/*")) sys_command(f'/usr/bin/umount {temporary_mountpoint}') temporary_path.rmdir() return True if files > 0 else False def safe_to_format(self): if self.allow_formatting is False: log(f"Partition {self} is not marked for formatting.", level=LOG_LEVELS.Debug) return False elif self.target_mountpoint == '/boot': try: if self.has_content(): log(f"Partition {self} is a boot partition and has content inside.", level=LOG_LEVELS.Debug) return False except SysCallError as err: log(err.message, LOG_LEVELS.Debug) log(f"Partition {self} was identified as /boot but we could not mount to check for content, continuing!", level=LOG_LEVELS.Debug) pass return True def encrypt(self, *args, **kwargs): """ A wrapper function for luks2() instances and the .encrypt() method of that instance. """ from .luks import luks2 if not self._encrypted: raise DiskError(f"Attempting to encrypt a partition that was not marked for encryption: {self}") if not self.safe_to_format(): log(f"Partition {self} was marked as protected but encrypt() was called on it!", level=LOG_LEVELS.Error, fg="red") return False handle = luks2(self, None, None) return handle.encrypt(self, *args, **kwargs) def format(self, filesystem=None, path=None, allow_formatting=None, log_formatting=True): """ Format can be given an overriding path, for instance /dev/null to test the formatting functionality and in essence the support for the given filesystem. """ if filesystem is None: filesystem = self.filesystem if path is None: path = self.path if allow_formatting is None: allow_formatting = self.allow_formatting # To avoid "unable to open /dev/x: No such file or directory" start_wait = time.time() while pathlib.Path(path).exists() is False and time.time() - start_wait < 10: time.sleep(0.025) if not allow_formatting: raise PermissionError(f"{self} is not formatable either because instance is locked ({self.allow_formatting}) or a blocking flag was given ({allow_formatting})") if log_formatting: log(f'Formatting {path} -> {filesystem}', level=LOG_LEVELS.Info) if filesystem == 'btrfs': o = b''.join(sys_command(f'/usr/bin/mkfs.btrfs -f {path}')) if b'UUID' not in o: raise DiskError(f'Could not format {path} with {filesystem} because: {o}') self.filesystem = 'btrfs' elif filesystem == 'vfat': o = b''.join(sys_command(f'/usr/bin/mkfs.vfat -F32 {path}')) if (b'mkfs.fat' not in o and b'mkfs.vfat' not in o) or b'command not found' in o: raise DiskError(f'Could not format {path} with {filesystem} because: {o}') self.filesystem = 'vfat' elif filesystem == 'ext4': if (handle := sys_command(f'/usr/bin/mkfs.ext4 -F {path}')).exit_code != 0: raise DiskError(f'Could not format {path} with {filesystem} because: {b"".join(handle)}') self.filesystem = 'ext4' elif filesystem == 'xfs': if (handle := sys_command(f'/usr/bin/mkfs.xfs -f {path}')).exit_code != 0: raise DiskError(f'Could not format {path} with {filesystem} because: {b"".join(handle)}') self.filesystem = 'xfs' elif filesystem == 'f2fs': if (handle := sys_command(f'/usr/bin/mkfs.f2fs -f {path}')).exit_code != 0: raise DiskError(f'Could not format {path} with {filesystem} because: {b"".join(handle)}') self.filesystem = 'f2fs' elif filesystem == 'crypto_LUKS': # from .luks import luks2 # encrypted_partition = luks2(self, None, None) # encrypted_partition.format(path) self.filesystem = 'crypto_LUKS' else: raise UnknownFilesystemFormat(f"Fileformat '{filesystem}' is not yet implemented.") if get_filesystem_type(path) == 'crypto_LUKS' or get_filesystem_type(self.real_device) == 'crypto_LUKS': self.encrypted = True else: self.encrypted = False return True def find_parent_of(self, data, name, parent=None): if data['name'] == name: return parent elif 'children' in data: for child in data['children']: if (parent := self.find_parent_of(child, name, parent=data['name'])): return parent def mount(self, target, fs=None, options=''): if not self.mountpoint: log(f'Mounting {self} to {target}', level=LOG_LEVELS.Info) if not fs: if not self.filesystem: raise DiskError(f'Need to format (or define) the filesystem on {self} before mounting.') fs = self.filesystem pathlib.Path(target).mkdir(parents=True, exist_ok=True) try: sys_command(f'/usr/bin/mount {self.path} {target}') except SysCallError as err: raise err self.mountpoint = target return True def unmount(self): try: exit_code = sys_command(f'/usr/bin/umount {self.path}').exit_code except SysCallError as err: exit_code = err.exit_code # Without to much research, it seams that low error codes are errors. # And above 8k is indicators such as "/dev/x not mounted.". # So anything in between 0 and 8k are errors (?). if exit_code > 0 and exit_code < 8000: raise err self.mountpoint = None return True def umount(self): return self.unmount() def filesystem_supported(self): """ The support for a filesystem (this partition) is tested by calling partition.format() with a path set to '/dev/null' which returns two exceptions: 1. SysCallError saying that /dev/null is not formattable - but the filesystem is supported 2. UnknownFilesystemFormat that indicates that we don't support the given filesystem type """ try: self.format(self.filesystem, '/dev/null', log_formatting=False, allow_formatting=True) except SysCallError: pass # We supported it, but /dev/null is not formatable as expected so the mkfs call exited with an error code except UnknownFilesystemFormat as err: raise err return True class Filesystem(): # TODO: # When instance of a HDD is selected, check all usages and gracefully unmount them # as well as close any crypto handles. def __init__(self, blockdevice, mode=GPT): self.blockdevice = blockdevice self.mode = mode def __enter__(self, *args, **kwargs): if self.blockdevice.keep_partitions is False: log(f'Wiping {self.blockdevice} by using partition format {self.mode}', level=LOG_LEVELS.Debug) if self.mode == GPT: if self.raw_parted(f'{self.blockdevice.device} mklabel gpt').exit_code == 0: self.blockdevice.flush_cache() return self else: raise DiskError(f'Problem setting the partition format to GPT:', f'/usr/bin/parted -s {self.blockdevice.device} mklabel gpt') else: raise DiskError(f'Unknown mode selected to format in: {self.mode}') # TODO: partition_table_type is hardcoded to GPT at the moment. This has to be changed. elif self.mode == self.blockdevice.partition_table_type: log(f'Kept partition format {self.mode} for {self.blockdevice}', level=LOG_LEVELS.Debug) else: raise DiskError(f'The selected partition table format {self.mode} does not match that of {self.blockdevice}.') return self def __repr__(self): return f"Filesystem(blockdevice={self.blockdevice}, mode={self.mode})" def __exit__(self, *args, **kwargs): # TODO: https://stackoverflow.com/questions/28157929/how-to-safely-handle-an-exception-inside-a-context-manager if len(args) >= 2 and args[1]: raise args[1] b''.join(sys_command(f'sync')) return True def find_partition(self, mountpoint): for partition in self.blockdevice: if partition.target_mountpoint == mountpoint or partition.mountpoint == mountpoint: return partition def raw_parted(self, string:str): x = sys_command(f'/usr/bin/parted -s {string}') log(f"'parted -s {string}' returned: {b''.join(x)}", level=LOG_LEVELS.Debug) return x def parted(self, string:str): """ Performs a parted execution of the given string :param string: A raw string passed to /usr/bin/parted -s :type string: str """ return self.raw_parted(string).exit_code def use_entire_disk(self, root_filesystem_type='ext4'): log(f"Using and formatting the entire {self.blockdevice}.", level=LOG_LEVELS.Debug) self.add_partition('primary', start='1MiB', end='513MiB', format='fat32') self.set_name(0, 'EFI') self.set(0, 'boot on') # TODO: Probably redundant because in GPT mode 'esp on' is an alias for "boot on"? # https://www.gnu.org/software/parted/manual/html_node/set.html self.set(0, 'esp on') self.add_partition('primary', start='513MiB', end='100%') self.blockdevice.partition[0].filesystem = 'vfat' self.blockdevice.partition[1].filesystem = root_filesystem_type log(f"Set the root partition {self.blockdevice.partition[1]} to use filesystem {root_filesystem_type}.", level=LOG_LEVELS.Debug) self.blockdevice.partition[0].target_mountpoint = '/boot' self.blockdevice.partition[1].target_mountpoint = '/' self.blockdevice.partition[0].allow_formatting = True self.blockdevice.partition[1].allow_formatting = True def add_partition(self, type, start, end, format=None): log(f'Adding partition to {self.blockdevice}', level=LOG_LEVELS.Info) previous_partitions = self.blockdevice.partitions if format: partitioning = self.parted(f'{self.blockdevice.device} mkpart {type} {format} {start} {end}') == 0 else: partitioning = self.parted(f'{self.blockdevice.device} mkpart {type} {start} {end}') == 0 if partitioning: start_wait = time.time() while previous_partitions == self.blockdevice.partitions: time.sleep(0.025) # Let the new partition come up in the kernel if time.time() - start_wait > 10: raise DiskError(f"New partition never showed up after adding new partition on {self} (timeout 10 seconds).") return True def set_name(self, partition:int, name:str): return self.parted(f'{self.blockdevice.device} name {partition+1} "{name}"') == 0 def set(self, partition:int, string:str): return self.parted(f'{self.blockdevice.device} set {partition+1} {string}') == 0 def device_state(name, *args, **kwargs): # Based out of: https://askubuntu.com/questions/528690/how-to-get-list-of-all-non-removable-disk-device-names-ssd-hdd-and-sata-ide-onl/528709#528709 if os.path.isfile('/sys/block/{}/device/block/{}/removable'.format(name, name)): with open('/sys/block/{}/device/block/{}/removable'.format(name, name)) as f: if f.read(1) == '1': return path = ROOT_DIR_PATTERN.sub('', os.readlink('/sys/block/{}'.format(name))) hotplug_buses = ("usb", "ieee1394", "mmc", "pcmcia", "firewire") for bus in hotplug_buses: if os.path.exists('/sys/bus/{}'.format(bus)): for device_bus in os.listdir('/sys/bus/{}/devices'.format(bus)): device_link = ROOT_DIR_PATTERN.sub('', os.readlink('/sys/bus/{}/devices/{}'.format(bus, device_bus))) if re.search(device_link, path): return return True # lsblk --json -l -n -o path def all_disks(*args, **kwargs): kwargs.setdefault("partitions", False) drives = OrderedDict() #for drive in json.loads(sys_command(f'losetup --json', *args, **lkwargs, hide_from_log=True)).decode('UTF_8')['loopdevices']: for drive in json.loads(b''.join(sys_command(f'lsblk --json -l -n -o path,size,type,mountpoint,label,pkname,model', *args, **kwargs, hide_from_log=True)).decode('UTF_8'))['blockdevices']: if not kwargs['partitions'] and drive['type'] == 'part': continue drives[drive['path']] = BlockDevice(drive['path'], drive) return drives def convert_to_gigabytes(string): unit = string.strip()[-1] size = float(string.strip()[:-1]) if unit == 'M': size = size/1024 elif unit == 'T': size = size*1024 return size def harddrive(size=None, model=None, fuzzy=False): collection = all_disks() for drive in collection: if size and convert_to_gigabytes(collection[drive]['size']) != size: continue if model and (collection[drive]['model'] is None or collection[drive]['model'].lower() != model.lower()): continue return collection[drive] def get_mount_info(path): try: output = b''.join(sys_command(f'/usr/bin/findmnt --json {path}')) except SysCallError: return {} output = output.decode('UTF-8') output = json.loads(output) if 'filesystems' in output: if len(output['filesystems']) > 1: raise DiskError(f"Path '{path}' contains multiple mountpoints: {output['filesystems']}") return output['filesystems'][0] def get_partitions_in_use(mountpoint): try: output = b''.join(sys_command(f'/usr/bin/findmnt --json -R {mountpoint}')) except SysCallError: return {} mounts = [] output = output.decode('UTF-8') output = json.loads(output) for target in output.get('filesystems', []): mounts.append(Partition(target['source'], None, filesystem=target.get('fstype', None), mountpoint=target['target'])) for child in target.get('children', []): mounts.append(Partition(child['source'], None, filesystem=child.get('fstype', None), mountpoint=child['target'])) return mounts def get_filesystem_type(path): try: handle = sys_command(f"blkid -o value -s TYPE {path}") return b''.join(handle).strip().decode('UTF-8') except SysCallError: return None