diff --git a/PortMaster/pylibs/harbourmaster/platform/base.py b/PortMaster/pylibs/harbourmaster/platform/base.py index e89a36c7..cedf32a7 100644 --- a/PortMaster/pylibs/harbourmaster/platform/base.py +++ b/PortMaster/pylibs/harbourmaster/platform/base.py @@ -107,7 +107,7 @@ def gamelist_backup(self): if broken: logger.debug(f"Creating empty {gamelist_xml}") - with open(gamelist_xml, 'w') as fh: + with open(gamelist_xml, 'w', encoding='utf-8') as fh: print(self.BLANK_GAMELIST_XML, file=fh) try: @@ -193,7 +193,7 @@ def gamelist_add(self, gameinfo_file): if hasattr(ET, 'indent'): ET.indent(gamelist_root, space=" ", level=0) - with open(gamelist_xml, 'w') as fh: + with open(gamelist_xml, 'w', encoding='utf-8') as fh: print("", file=fh) print("", file=fh) print(ET.tostring(gamelist_root, encoding='unicode'), file=fh) @@ -277,7 +277,8 @@ def gamelist_add_portmaster(self): temp_dir = None if self.hm.temp_dir is None else str(self.hm.temp_dir) with tempfile.TemporaryDirectory(dir=temp_dir) as work_dir: temp_gameinfo = Path(work_dir) / "gameinfo.xml" - gameinfo_tree.write(str(temp_gameinfo), encoding='unicode') + ## Bytes in UTF-8, as gamelist_add() reads it back (encoding='unicode' wrote the locale's codeset). + gameinfo_tree.write(str(temp_gameinfo), encoding='utf-8') self.gamelist_add(temp_gameinfo) def set_gcd_mode(self, mode=None): diff --git a/PortMaster/pylibs/harbourmaster/platform/muos.py b/PortMaster/pylibs/harbourmaster/platform/muos.py index fc59f06c..7b670e03 100644 --- a/PortMaster/pylibs/harbourmaster/platform/muos.py +++ b/PortMaster/pylibs/harbourmaster/platform/muos.py @@ -102,7 +102,7 @@ def gamelist_add(self, gameinfo_file): text = child.text.strip().split('\n', 1)[0].strip() logger.debug(f"creating {str(target_file)}") - with open(target_file, 'w') as fh: + with open(target_file, 'w', encoding='utf-8') as fh: print(text, file=fh) # HAHA THIS IS FUCKED diff --git a/PortMaster/pylibs/harbourmaster/platform/trimui.py b/PortMaster/pylibs/harbourmaster/platform/trimui.py index 314c4dc0..8b098365 100644 --- a/PortMaster/pylibs/harbourmaster/platform/trimui.py +++ b/PortMaster/pylibs/harbourmaster/platform/trimui.py @@ -117,7 +117,7 @@ def add_port_script(self, port_script): new_port_dir.mkdir(0o755, parents=True, exist_ok=True) logger.debug(f"Creating {str(new_port_dir / 'config.json')}") - with open(new_port_dir / "config.json", "w") as fh: + with open(new_port_dir / "config.json", "w", encoding="utf-8") as fh: fh.write( PORT_CONFIG_JSON .replace("{{PORTTITLE}}", port_script.stem) @@ -258,7 +258,7 @@ def gamelist_add(self, gameinfo_file): new_port_dir.mkdir(0o755, parents=True, exist_ok=True) logger.debug(f"Creating {str(new_port_dir / 'config.json')}") - with open(new_port_dir / "config.json", "w") as fh: + with open(new_port_dir / "config.json", "w", encoding="utf-8") as fh: fh.write( PORT_CONFIG_JSON .replace("{{PORTTITLE}}", port_title) diff --git a/PortMaster/pylibs/harbourmaster/util/files.py b/PortMaster/pylibs/harbourmaster/util/files.py index 4463ae9e..adcac53a 100644 --- a/PortMaster/pylibs/harbourmaster/util/files.py +++ b/PortMaster/pylibs/harbourmaster/util/files.py @@ -259,9 +259,13 @@ def get_path_fs(path): else: return None + ## df's own words in the C locale: PortMaster sets LANG for the language the user chose, + ## and a legacy codeset there (ISO-8859-5 for ru_RU on dArkOS) couldn't be decoded. + env = dict(os.environ, LC_ALL='C') try: - lines = subprocess.check_output(['df', '-PT', str(path)]).decode().split('\n') - except subprocess.CalledProcessError as err: + lines = subprocess.check_output(['df', '-PT', str(path)], env=env).decode('utf-8', 'replace').split('\n') + except (subprocess.CalledProcessError, OSError): + ## No df on this CFW, or it failed: we don't know the filesystem. return None if len(lines) < 2: diff --git a/PortMaster/pylibs/pugwash/lang.py b/PortMaster/pylibs/pugwash/lang.py index bc2a081e..ea02300a 100644 --- a/PortMaster/pylibs/pugwash/lang.py +++ b/PortMaster/pylibs/pugwash/lang.py @@ -119,18 +119,32 @@ def load_lang(): DEFAULT_LANG = check_lang(environ_lang) or 'en_US' if config_lang and check_lang(config_lang): - os.environ[environ_key] = check_lang(config_lang) + CURRENT_LANG = check_lang(config_lang) elif environ_lang and check_lang(environ_lang): - os.environ[environ_key] = check_lang(environ_lang) + CURRENT_LANG = check_lang(environ_lang) else: - os.environ[environ_key] = 'en_US' + CURRENT_LANG = 'en_US' - CURRENT_LANG = os.environ[environ_key] + os.environ[environ_key] = environ_value(environ_key, CURRENT_LANG) ENVIRON_KEY = environ_key +def environ_value(key, language): + """ + What the environment variable `key` gets for `language`: the locale with its UTF-8 + codeset (LANG, LC_ALL, LC_MESSAGES), or just the language for LANGUAGE (a gettext list). + + A bare "ru_RU" is a legacy codeset (ISO-8859-5) on systems that have the locale (dArkOS), + so the programs PortMaster starts (df, the theme server, ports) wrote in it: harbourmaster + couldn't read df and every install failed. + """ + if key == 'LANGUAGE': + return language + return language + '.UTF-8' + + def set_language(language): """ Use another language from now on, None for the system's: its translations, CURRENT_LANG @@ -141,7 +155,7 @@ def set_language(language): global CURRENT_LANG CURRENT_LANG = (language and check_lang(language)) or DEFAULT_LANG or 'en_US' - os.environ[ENVIRON_KEY] = CURRENT_LANG + os.environ[ENVIRON_KEY] = environ_value(ENVIRON_KEY, CURRENT_LANG) i18n.load_language(LANG_DIR, CURRENT_LANG) return CURRENT_LANG diff --git a/PortMaster/pylibs/pugwash/sdl/resources.py b/PortMaster/pylibs/pugwash/sdl/resources.py index 0489c1cb..ea08f484 100644 --- a/PortMaster/pylibs/pugwash/sdl/resources.py +++ b/PortMaster/pylibs/pugwash/sdl/resources.py @@ -42,6 +42,15 @@ def remove_path(self, path): self._paths.remove(path) self.find.cache_clear() + @staticmethod + def _is_file(path): + ## Theme values that are text ("Zurück" on a button bar) get looked up too; a name the + ## filesystem refuses (EPERM on exFAT for one that isn't UTF-8) is just not a file. + try: + return path.is_file() + except (OSError, ValueError): + return False + @functools.lru_cache(512) def find(self, file_name): if isinstance(file_name, pathlib.PurePath): @@ -59,13 +68,13 @@ def find(self, file_name): if file_name.name in ('.', '..'): return None - if file_name.is_file(): + if self._is_file(file_name): return file_name for path in reversed(self._paths): full_file_name = path / file_name - if full_file_name.is_file(): + if self._is_file(full_file_name): return full_file_name return None diff --git a/tests/test_gamelist.py b/tests/test_gamelist.py index 1cf1da63..7fcd7f61 100644 --- a/tests/test_gamelist.py +++ b/tests/test_gamelist.py @@ -72,6 +72,31 @@ def test_add_creates_gamelist(platform, gameinfo): assert platform.ports_changed() +def test_add_writes_utf8_whatever_the_locale(platform, gameinfo, monkeypatch): + ## PortMaster sets LANG to the user's language; a bare "en_US" is Latin-1 where the locale + ## exists (dArkOS), and open() without an encoding follows it: "’" couldn't be written (the + ## update failed and was rolled back), and "é" went out as Latin-1 in a file that says utf-8. + from harbourmaster.platform import base + + def latin1_open(file, mode='r', *args, **kwargs): + if 'b' not in mode and 'encoding' not in kwargs and not args: + kwargs['encoding'] = 'latin-1' + return open(file, mode, *args, **kwargs) + + monkeypatch.setattr(base, "open", latin1_open, raising=False) + platform.test_gamelist.write_bytes( + "./Other.shIt’s a café".encode("utf-8")) + gameinfo.write_bytes(GAMEINFO.replace("Test Port", "Test Port ’é").encode("utf-8")) + + platform.gamelist_add(gameinfo) + + raw = platform.test_gamelist.read_bytes() + raw.decode("utf-8") + result = games(platform.test_gamelist) + assert result["./Test Port.sh"]['name'] == "Test Port ’é" + assert result["./Other.sh"]['name'] == "It’s a café" + + def test_add_updates_existing_entry(platform, gameinfo): platform.test_gamelist.write_text( "./Test Port.shOld3" diff --git a/tests/test_lang.py b/tests/test_lang.py index 564df174..6ae55002 100644 --- a/tests/test_lang.py +++ b/tests/test_lang.py @@ -73,7 +73,7 @@ def test_check_lang_nothing(lang, value): ## load_lang: config.json's language, else the system's, else English def test_system_language(lang): assert load(lang, LANG="fr_FR.UTF-8") == ("fr_FR", "fr_FR") - assert os.environ['LANG'] == "fr_FR" + assert os.environ['LANG'] == "fr_FR.UTF-8" def test_chosen_language_wins(lang): @@ -93,7 +93,7 @@ def test_no_usable_system_language_is_english(lang, value): def test_no_language_variable_at_all(lang): assert load(lang) == ("en_US", "en_US") - assert lang.ENVIRON_KEY == "LANG" and os.environ["LANG"] == "en_US" + assert lang.ENVIRON_KEY == "LANG" and os.environ["LANG"] == "en_US.UTF-8" def test_first_language_variable_is_used(lang): @@ -102,9 +102,26 @@ def test_first_language_variable_is_used(lang): assert "LANGUAGE" not in os.environ and "LC_ALL" not in os.environ +@pytest.mark.parametrize("value", ["ru_RU", "ru_RU.KOI8-R", "ja_JP.eucJP", "zh_CN.GB2312"]) +def test_the_environment_gets_utf8(lang, value): + ## A bare or legacy locale is a legacy codeset where the locale exists (dArkOS has them + ## all): df, the theme server and ports PortMaster starts wrote in it, and installs failed. + load(lang, LANG=value) + assert os.environ["LANG"].endswith(".UTF-8") + assert lang.CURRENT_LANG == lang.check_lang(value) + + +def test_language_variable_stays_a_language_list(lang): + ## LANGUAGE is gettext's list of languages, not a locale: no codeset. + assert load(lang, LANGUAGE="de_DE") == ("de_DE", "de_DE") + assert os.environ["LANGUAGE"] == "de_DE" + lang.set_language("fr_FR") + assert os.environ["LANGUAGE"] == "fr_FR" + + def test_other_language_variable(lang): assert load(lang, LC_ALL="ru_RU.UTF-8") == ("ru_RU", "ru_RU") - assert lang.ENVIRON_KEY == "LC_ALL" and os.environ["LC_ALL"] == "ru_RU" + assert lang.ENVIRON_KEY == "LC_ALL" and os.environ["LC_ALL"] == "ru_RU.UTF-8" ################################################################################ @@ -113,7 +130,7 @@ def test_set_language(lang): load(lang, LANG="en_US") assert lang.set_language("de_DE") == "de_DE" - assert lang.CURRENT_LANG == "de_DE" and os.environ[lang.ENVIRON_KEY] == "de_DE" + assert lang.CURRENT_LANG == "de_DE" and os.environ[lang.ENVIRON_KEY] == "de_DE.UTF-8" assert i18n._("Options") == "Optionen" ## None, or one PortMaster hasn't: the system's. diff --git a/tests/test_sdl_helpers.py b/tests/test_sdl_helpers.py index 74d7b2e2..cea9307c 100644 --- a/tests/test_sdl_helpers.py +++ b/tests/test_sdl_helpers.py @@ -353,3 +353,26 @@ def test_animation_scene_change(pugwash, ticks): manager.change_scene() assert gui.data["animation.resets"] == "000" assert gui.data["animation.keeps"] == "002" + + +################################################################################ +## ResourceManager +def test_resource_find_survives_names_the_filesystem_refuses(pugwash, tmp_path, monkeypatch): + ## Button bar text ("Zurück") is looked up as a file too; on exFAT a name that isn't UTF-8 + ## gives EPERM, and Path.is_file() lets that through: the theme server's render crashed. + import pathlib + from pugwash.sdl.resources import ResourceManager + + (tmp_path / "icon.png").write_bytes(b"png") + resources = ResourceManager(None) + resources.add_path(tmp_path) + real_is_file = pathlib.Path.is_file + + def is_file(self): + if "Zurück" in str(self): + raise PermissionError(1, "Operation not permitted", str(self)) + return real_is_file(self) + + monkeypatch.setattr(pathlib.Path, "is_file", is_file) + assert resources.find("Zurück") is None + assert resources.find("icon.png") == tmp_path / "icon.png" diff --git a/tests/test_util_edges.py b/tests/test_util_edges.py index aaf0db12..0ac6eae9 100644 --- a/tests/test_util_edges.py +++ b/tests/test_util_edges.py @@ -302,6 +302,34 @@ def fail(*args, **kwargs): assert util.get_path_fs(tmp_path) is None +@pytest.mark.parametrize("error", [FileNotFoundError(2, "No such file or directory: 'df'"), PermissionError(13, "df")]) +def test_get_path_fs_without_df(tmp_path, monkeypatch, error): + ## A CFW without df (or one we can't run): the filesystem is unknown, installs go on. + monkeypatch.setattr(util_files, "HM_TESTING", False) + + def fail(*args, **kwargs): + raise error + + monkeypatch.setattr(util_files.subprocess, "check_output", fail) + assert util.get_path_fs(tmp_path) is None + + +def test_get_path_fs_runs_df_in_the_c_locale(tmp_path, monkeypatch): + ## PortMaster sets LANG to the user's language; df's header in a legacy codeset + ## (ru_RU is ISO-8859-5 on dArkOS: "Файловая система" starts with 0xC4) failed every install. + monkeypatch.setattr(util_files, "HM_TESTING", False) + monkeypatch.setenv("LANG", "ru_RU") + calls = [] + + def df(args, **kwargs): + calls.append(kwargs.get('env') or {}) + return "Файловая система Тип\n".encode("iso8859-5") + b"/dev/mmcblk2p1 exfat 100 50 50 50% /roms2\n" + + monkeypatch.setattr(util_files.subprocess, "check_output", df) + assert util.get_path_fs(tmp_path) == "exfat" + assert calls[0].get('LC_ALL') == "C" and calls[0].get('LANG') == "ru_RU" + + ################################################################################ ## Network class Response: