Update translations (#1348)

* Show translations in own tongue

* Fix flake8

* Update

* Update

* Update

* Update

* fix mypy

* Update

* Update

Co-authored-by: Daniel Girtler <girtler.daniel@gmail.com>

6 files changed, 196 insertions, 163 deletions

diff --git a/archinstall/lib/menu/global_menu.py b/archinstall/lib/menu/global_menu.py
index 1a292476..1badc052 100644
--- a/archinstall/lib/menu/global_menu.py
+++ b/archinstall/lib/menu/global_menu.py
@@ -50,7 +50,8 @@ class GlobalMenu(GeneralMenu):
 			Selector(
 				_('Archinstall language'),
 				lambda x: self._select_archinstall_language(x),
-				default='English')
+				display_func=lambda x: x.display_name,
+				default=self.translation_handler.get_language('en'))
 		self._menu_options['keyboard-layout'] = \
 			Selector(
 				_('Keyboard layout'),
diff --git a/archinstall/lib/menu/selection_menu.py b/archinstall/lib/menu/selection_menu.py
index c6ac5852..8a08812c 100644
--- a/archinstall/lib/menu/selection_menu.py
+++ b/archinstall/lib/menu/selection_menu.py
@@ -8,9 +8,11 @@ from typing import Callable, Any, List, Iterator, Tuple, Optional, Dict, TYPE_CH
 from .menu import Menu, MenuSelectionType
 from ..locale_helpers import set_keyboard_language
 from ..output import log
-from ..translation import Translation
+from ..translationhandler import TranslationHandler, Language
 from ..hsm.fido import get_fido2_devices
 
+from ..user_interaction.general_conf import select_archinstall_language
+
 if TYPE_CHECKING:
 	_: Any
 
@@ -181,7 +183,7 @@ class GeneralMenu:
 
 		"""
 		self._enabled_order :List[str] = []
-		self._translation = Translation.load_nationalization()
+		self._translation_handler = TranslationHandler()
 		self.is_context_mgr = False
 		self._data_store = data_store if data_store is not None else {}
 		self.auto_cursor = auto_cursor
@@ -213,6 +215,10 @@ class GeneralMenu:
 
 		self.exit_callback()
 
+	@property
+	def translation_handler(self) -> TranslationHandler:
+		return self._translation_handler
+
 	def _setup_selection_menu_options(self):
 		""" Define the menu options.
 			Menu options can be defined here in a subclass or done per program calling self.set_option()
@@ -461,14 +467,10 @@ class GeneralMenu:
 					mandatory_waiting += 1
 		return mandatory_fields, mandatory_waiting
 
-	def _select_archinstall_language(self, preset_value: str) -> str:
-		from ... import select_archinstall_language
-		language = select_archinstall_language(preset_value)
-		if language is not None:
-			self._translation.activate(language)
-			return language
-
-		return preset_value
+	def _select_archinstall_language(self, preset_value: Language) -> Language:
+		language = select_archinstall_language(self.translation_handler.translated_languages, preset_value)
+		self._translation_handler.activate(language)
+		return language
 
 	def _select_hsm(self, preset :Optional[pathlib.Path] = None) -> Optional[pathlib.Path]:
 		title = _('Select which partitions to mark for formatting:')
diff --git a/archinstall/lib/translation.py b/archinstall/lib/translation.py
deleted file mode 100644
index c20a4285..00000000
--- a/archinstall/lib/translation.py
+++ /dev/null
@@ -1,144 +0,0 @@
-from __future__ import annotations
-
-import json
-import logging
-import os
-import gettext
-
-from pathlib import Path
-from typing import List, Dict, Any, TYPE_CHECKING, Tuple
-from .exceptions import TranslationError
-
-if TYPE_CHECKING:
-	_: Any
-
-
-class LanguageDefinitions:
-	_languages = 'languages.json'
-	_cyrillic = 'cyrillic.json'
-
-	def __init__(self):
-		self._mappings = self._get_language_mappings()
-		self._cyrillic_languages = self._get_cyrillic_languages()
-
-	def is_cyrillic(self, language: str) -> bool:
-		return language in self._cyrillic_languages
-
-	def _get_language_mappings(self) -> List[Dict[str, str]]:
-		locales_dir = Translation.get_locales_dir()
-		languages = Path.joinpath(locales_dir, self._languages)
-
-		with open(languages, 'r') as fp:
-			return json.load(fp)
-
-	def get_language(self, abbr: str) -> str:
-		for entry in self._mappings:
-			if entry['abbr'] == abbr:
-				return entry['lang']
-
-		raise ValueError(f'No language with abbreviation "{abbr}" found')
-
-	def _get_cyrillic_languages(self) -> List[str]:
-		locales_dir = Translation.get_locales_dir()
-		languages = Path.joinpath(locales_dir, self._cyrillic)
-
-		with open(languages, 'r') as fp:
-			data = json.load(fp)
-			return data['languages']
-
-
-class DeferredTranslation:
-	def __init__(self, message: str):
-		self.message = message
-
-	def __len__(self) -> int:
-		return len(self.message)
-
-	def __str__(self) -> str:
-		translate = _
-		if translate is DeferredTranslation:
-			return self.message
-		return translate(self.message)
-
-	def __lt__(self, other) -> bool:
-		return self.message < other
-
-	def __gt__(self, other) -> bool:
-		return self.message > other
-
-	def __add__(self, other) -> DeferredTranslation:
-		if isinstance(other, str):
-			other = DeferredTranslation(other)
-
-		concat = self.message + other.message
-		return DeferredTranslation(concat)
-
-	def format(self, *args) -> str:
-		return self.message.format(*args)
-
-	@classmethod
-	def install(cls):
-		import builtins
-		builtins._ = cls
-
-
-class Translation:
-	def __init__(self, locales_dir):
-		self._languages = {}
-
-		for names in self._get_translation_lang():
-			try:
-				self._languages[names[0]] = gettext.translation('base', localedir=locales_dir, languages=names)
-			except FileNotFoundError as error:
-				raise TranslationError(f"Could not locate language file for '{names}': {error}")
-
-	def activate(self, name):
-		if language := self._languages.get(name, None):
-			languages = LanguageDefinitions()
-
-			if languages.is_cyrillic(name):
-				self._set_font('UniCyr_8x16')
-			else:
-				# this will reset a possible previously set font to a default font
-				self._set_font('')
-
-			language.install()
-		else:
-			raise ValueError(f'Language not supported: {name}')
-
-	def _set_font(self, font: str):
-		from archinstall import SysCommand, log
-		try:
-			log(f'Setting new font: {font}', level=logging.DEBUG)
-			SysCommand(f'setfont {font}')
-		except Exception:
-			log(f'Unable to set font {font}', level=logging.ERROR)
-
-	@classmethod
-	def load_nationalization(cls) -> Translation:
-		locales_dir = cls.get_locales_dir()
-		return Translation(locales_dir)
-
-	@classmethod
-	def get_locales_dir(cls) -> Path:
-		cur_path = Path(__file__).parent.parent
-		locales_dir = Path.joinpath(cur_path, 'locales')
-		return locales_dir
-
-	@classmethod
-	def _defined_languages(cls) -> List[str]:
-		locales_dir = cls.get_locales_dir()
-		filenames = os.listdir(locales_dir)
-		return list(filter(lambda x: len(x) == 2, filenames))
-
-	@classmethod
-	def _get_translation_lang(cls) -> List[Tuple[str, str]]:
-		def_languages = cls._defined_languages()
-		languages = LanguageDefinitions()
-		return [(languages.get_language(lang), lang) for lang in def_languages]
-
-	@classmethod
-	def get_available_lang(cls) -> List[str]:
-		def_languages = cls._defined_languages()
-		languages = LanguageDefinitions()
-		return [languages.get_language(lang) for lang in def_languages]
diff --git a/archinstall/lib/translationhandler.py b/archinstall/lib/translationhandler.py
new file mode 100644
index 00000000..12c8da4a
--- /dev/null
+++ b/archinstall/lib/translationhandler.py
@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+import gettext
+from dataclasses import dataclass
+
+from pathlib import Path
+from typing import List, Dict, Any, TYPE_CHECKING, Optional
+from .exceptions import TranslationError
+
+if TYPE_CHECKING:
+	_: Any
+
+
+@dataclass
+class Language:
+	abbr: str
+	lang: str
+	translation: gettext.NullTranslations
+	translation_percent: int
+	translated_lang: Optional[str]
+
+	@property
+	def display_name(self) -> str:
+		if self.translated_lang:
+			name = self.translated_lang
+		else:
+			name = self.lang
+		return f'{name} ({self.translation_percent}%)'
+
+	def is_match(self, lang_or_translated_lang: str) -> bool:
+		if self.lang == lang_or_translated_lang:
+			return True
+		elif self.translated_lang == lang_or_translated_lang:
+			return True
+		return False
+
+
+class TranslationHandler:
+	_base_pot = 'base.pot'
+	_languages = 'languages.json'
+
+	def __init__(self):
+		# to display cyrillic languages correctly
+		self._set_font('UniCyr_8x16')
+
+		self._total_messages = self._get_total_messages()
+		self._translated_languages = self._get_translations()
+
+	@property
+	def translated_languages(self) -> List[Language]:
+		return self._translated_languages
+
+	def _get_translations(self) -> List[Language]:
+		mappings = self._load_language_mappings()
+		defined_languages = self._defined_languages()
+
+		languages = []
+
+		for short_form in defined_languages:
+			mapping_entry: Dict[str, Any] = next(filter(lambda x: x['abbr'] == short_form, mappings))
+			abbr = mapping_entry['abbr']
+			lang = mapping_entry['lang']
+			translated_lang = mapping_entry.get('translated_lang', None)
+
+			try:
+				translation = gettext.translation('base', localedir=self._get_locales_dir(), languages=(abbr, lang))
+
+				if abbr == 'en':
+					percent = 100
+				else:
+					num_translations = self._get_catalog_size(translation)
+					percent = int((num_translations / self._total_messages) * 100)
+
+				language = Language(abbr, lang, translation, percent, translated_lang)
+				languages.append(language)
+			except FileNotFoundError as error:
+				raise TranslationError(f"Could not locate language file for '{lang}': {error}")
+
+		return languages
+
+	def _set_font(self, font: str):
+		from archinstall import SysCommand, log
+		try:
+			log(f'Setting font: {font}', level=logging.DEBUG)
+			SysCommand(f'setfont {font}')
+		except Exception:
+			log(f'Unable to set font {font}', level=logging.ERROR)
+
+	def _load_language_mappings(self) -> List[Dict[str, Any]]:
+		locales_dir = self._get_locales_dir()
+		languages = Path.joinpath(locales_dir, self._languages)
+
+		with open(languages, 'r') as fp:
+			return json.load(fp)
+
+	def _get_catalog_size(self, translation: gettext.NullTranslations) -> int:
+		# this is a ery naughty way of retrieving the data but
+		# there's no alternative method exposed unfortunately
+		catalog = translation._catalog  # type: ignore
+		messages = {k: v for k, v in catalog.items() if k and v}
+		return len(messages)
+
+	def _get_total_messages(self) -> int:
+		locales = self._get_locales_dir()
+		with open(f'{locales}/{self._base_pot}', 'r') as fp:
+			lines = fp.readlines()
+			msgid_lines = [line for line in lines if 'msgid' in line]
+		return len(msgid_lines) - 1  # don't count the first line which contains the metadata
+
+	def get_language(self, abbr: str) -> Language:
+		try:
+			return next(filter(lambda x: x.abbr == abbr, self._translated_languages))
+		except Exception:
+			raise ValueError(f'No language with abbreviation "{abbr}" found')
+
+	def activate(self, language: Language):
+		language.translation.install()
+
+	def _get_locales_dir(self) -> Path:
+		cur_path = Path(__file__).parent.parent
+		locales_dir = Path.joinpath(cur_path, 'locales')
+		return locales_dir
+
+	def _defined_languages(self) -> List[str]:
+		locales_dir = self._get_locales_dir()
+		filenames = os.listdir(locales_dir)
+		return list(filter(lambda x: len(x) == 2 or x == 'pt_BR', filenames))
+
+
+class DeferredTranslation:
+	def __init__(self, message: str):
+		self.message = message
+
+	def __len__(self) -> int:
+		return len(self.message)
+
+	def __str__(self) -> str:
+		translate = _
+		if translate is DeferredTranslation:
+			return self.message
+		return translate(self.message)
+
+	def __lt__(self, other) -> bool:
+		return self.message < other
+
+	def __gt__(self, other) -> bool:
+		return self.message > other
+
+	def __add__(self, other) -> DeferredTranslation:
+		if isinstance(other, str):
+			other = DeferredTranslation(other)
+
+		concat = self.message + other.message
+		return DeferredTranslation(concat)
+
+	def format(self, *args) -> str:
+		return self.message.format(*args)
+
+	@classmethod
+	def install(cls):
+		import builtins
+		builtins._ = cls
diff --git a/archinstall/lib/user_interaction/general_conf.py b/archinstall/lib/user_interaction/general_conf.py
index 15c42b86..bdc602b3 100644
--- a/archinstall/lib/user_interaction/general_conf.py
+++ b/archinstall/lib/user_interaction/general_conf.py
@@ -12,7 +12,7 @@ from ..output import log
 from ..profiles import Profile, list_profiles
 from ..mirrors import list_mirrors
 
-from ..translation import Translation
+from ..translationhandler import Language
 from ..packages.packages import validate_package_list
 
 from ..storage import storage
@@ -118,13 +118,22 @@ def select_mirror_regions(preset_values: Dict[str, Any] = {}) -> Dict[str, Any]:
 		case _: return {selected: mirrors[selected] for selected in selected_mirror.value}
 
 
-def select_archinstall_language(preset_values: str):
-	languages = Translation.get_available_lang()
-	choice = Menu(_('Archinstall language'), languages, default_option=preset_values).run()
+def select_archinstall_language(languages: List[Language], preset_value: Language) -> Language:
+	# these are the displayed language names which can either be
+	# the english name of a language or, if present, the
+	# name of the language in its own language
+	options = {lang.display_name: lang for lang in languages}
+
+	choice = Menu(
+		_('Archinstall language'),
+		list(options.keys()),
+		default_option=preset_value.display_name
+	).run()
 
 	match choice.type_:
-		case MenuSelectionType.Esc: return preset_values
-		case MenuSelectionType.Selection: return choice.value
+		case MenuSelectionType.Esc: return preset_value
+		case MenuSelectionType.Selection:
+			return options[choice.value]
 
 
 def select_profile(preset) -> Optional[Profile]:
diff --git a/archinstall/lib/user_interaction/manage_users_conf.py b/archinstall/lib/user_interaction/manage_users_conf.py
index a97328c2..84ce3556 100644
--- a/archinstall/lib/user_interaction/manage_users_conf.py
+++ b/archinstall/lib/user_interaction/manage_users_conf.py
@@ -57,10 +57,10 @@ class UserList(ListManager):
 			prompt = str(_('Password for user "{}": ').format(entry.username))
 			new_password = get_password(prompt=prompt)
 			if new_password:
-				user = next(filter(lambda x: x == entry, data), 1)
+				user = next(filter(lambda x: x == entry, data))
 				user.password = new_password
 		elif action == self._actions[2]:  # promote/demote
-			user = next(filter(lambda x: x == entry, data), 1)
+			user = next(filter(lambda x: x == entry, data))
 			user.sudo = False if user.sudo else True
 		elif action == self._actions[3]:  # delete
 			data = [d for d in data if d != entry]