diff --git a/PortMaster/pylibs/harbourmaster/platform/base.py b/PortMaster/pylibs/harbourmaster/platform/base.py
index e89a36c7..cedf32a7 100644
--- a/PortMaster/pylibs/harbourmaster/platform/base.py
+++ b/PortMaster/pylibs/harbourmaster/platform/base.py
@@ -107,7 +107,7 @@ def gamelist_backup(self):
if broken:
logger.debug(f"Creating empty {gamelist_xml}")
- with open(gamelist_xml, 'w') as fh:
+ with open(gamelist_xml, 'w', encoding='utf-8') as fh:
print(self.BLANK_GAMELIST_XML, file=fh)
try:
@@ -193,7 +193,7 @@ def gamelist_add(self, gameinfo_file):
if hasattr(ET, 'indent'):
ET.indent(gamelist_root, space=" ", level=0)
- with open(gamelist_xml, 'w') as fh:
+ with open(gamelist_xml, 'w', encoding='utf-8') as fh:
print("", file=fh)
print("", file=fh)
print(ET.tostring(gamelist_root, encoding='unicode'), file=fh)
@@ -277,7 +277,8 @@ def gamelist_add_portmaster(self):
temp_dir = None if self.hm.temp_dir is None else str(self.hm.temp_dir)
with tempfile.TemporaryDirectory(dir=temp_dir) as work_dir:
temp_gameinfo = Path(work_dir) / "gameinfo.xml"
- gameinfo_tree.write(str(temp_gameinfo), encoding='unicode')
+ ## Bytes in UTF-8, as gamelist_add() reads it back (encoding='unicode' wrote the locale's codeset).
+ gameinfo_tree.write(str(temp_gameinfo), encoding='utf-8')
self.gamelist_add(temp_gameinfo)
def set_gcd_mode(self, mode=None):
diff --git a/PortMaster/pylibs/harbourmaster/platform/muos.py b/PortMaster/pylibs/harbourmaster/platform/muos.py
index fc59f06c..7b670e03 100644
--- a/PortMaster/pylibs/harbourmaster/platform/muos.py
+++ b/PortMaster/pylibs/harbourmaster/platform/muos.py
@@ -102,7 +102,7 @@ def gamelist_add(self, gameinfo_file):
text = child.text.strip().split('\n', 1)[0].strip()
logger.debug(f"creating {str(target_file)}")
- with open(target_file, 'w') as fh:
+ with open(target_file, 'w', encoding='utf-8') as fh:
print(text, file=fh)
# HAHA THIS IS FUCKED
diff --git a/PortMaster/pylibs/harbourmaster/platform/trimui.py b/PortMaster/pylibs/harbourmaster/platform/trimui.py
index 314c4dc0..8b098365 100644
--- a/PortMaster/pylibs/harbourmaster/platform/trimui.py
+++ b/PortMaster/pylibs/harbourmaster/platform/trimui.py
@@ -117,7 +117,7 @@ def add_port_script(self, port_script):
new_port_dir.mkdir(0o755, parents=True, exist_ok=True)
logger.debug(f"Creating {str(new_port_dir / 'config.json')}")
- with open(new_port_dir / "config.json", "w") as fh:
+ with open(new_port_dir / "config.json", "w", encoding="utf-8") as fh:
fh.write(
PORT_CONFIG_JSON
.replace("{{PORTTITLE}}", port_script.stem)
@@ -258,7 +258,7 @@ def gamelist_add(self, gameinfo_file):
new_port_dir.mkdir(0o755, parents=True, exist_ok=True)
logger.debug(f"Creating {str(new_port_dir / 'config.json')}")
- with open(new_port_dir / "config.json", "w") as fh:
+ with open(new_port_dir / "config.json", "w", encoding="utf-8") as fh:
fh.write(
PORT_CONFIG_JSON
.replace("{{PORTTITLE}}", port_title)
diff --git a/PortMaster/pylibs/harbourmaster/util/files.py b/PortMaster/pylibs/harbourmaster/util/files.py
index 4463ae9e..adcac53a 100644
--- a/PortMaster/pylibs/harbourmaster/util/files.py
+++ b/PortMaster/pylibs/harbourmaster/util/files.py
@@ -259,9 +259,13 @@ def get_path_fs(path):
else:
return None
+ ## df's own words in the C locale: PortMaster sets LANG for the language the user chose,
+ ## and a legacy codeset there (ISO-8859-5 for ru_RU on dArkOS) couldn't be decoded.
+ env = dict(os.environ, LC_ALL='C')
try:
- lines = subprocess.check_output(['df', '-PT', str(path)]).decode().split('\n')
- except subprocess.CalledProcessError as err:
+ lines = subprocess.check_output(['df', '-PT', str(path)], env=env).decode('utf-8', 'replace').split('\n')
+ except (subprocess.CalledProcessError, OSError):
+ ## No df on this CFW, or it failed: we don't know the filesystem.
return None
if len(lines) < 2:
diff --git a/PortMaster/pylibs/pugwash/lang.py b/PortMaster/pylibs/pugwash/lang.py
index bc2a081e..ea02300a 100644
--- a/PortMaster/pylibs/pugwash/lang.py
+++ b/PortMaster/pylibs/pugwash/lang.py
@@ -119,18 +119,32 @@ def load_lang():
DEFAULT_LANG = check_lang(environ_lang) or 'en_US'
if config_lang and check_lang(config_lang):
- os.environ[environ_key] = check_lang(config_lang)
+ CURRENT_LANG = check_lang(config_lang)
elif environ_lang and check_lang(environ_lang):
- os.environ[environ_key] = check_lang(environ_lang)
+ CURRENT_LANG = check_lang(environ_lang)
else:
- os.environ[environ_key] = 'en_US'
+ CURRENT_LANG = 'en_US'
- CURRENT_LANG = os.environ[environ_key]
+ os.environ[environ_key] = environ_value(environ_key, CURRENT_LANG)
ENVIRON_KEY = environ_key
+def environ_value(key, language):
+ """
+ What the environment variable `key` gets for `language`: the locale with its UTF-8
+ codeset (LANG, LC_ALL, LC_MESSAGES), or just the language for LANGUAGE (a gettext list).
+
+ A bare "ru_RU" is a legacy codeset (ISO-8859-5) on systems that have the locale (dArkOS),
+ so the programs PortMaster starts (df, the theme server, ports) wrote in it: harbourmaster
+ couldn't read df and every install failed.
+ """
+ if key == 'LANGUAGE':
+ return language
+ return language + '.UTF-8'
+
+
def set_language(language):
"""
Use another language from now on, None for the system's: its translations, CURRENT_LANG
@@ -141,7 +155,7 @@ def set_language(language):
global CURRENT_LANG
CURRENT_LANG = (language and check_lang(language)) or DEFAULT_LANG or 'en_US'
- os.environ[ENVIRON_KEY] = CURRENT_LANG
+ os.environ[ENVIRON_KEY] = environ_value(ENVIRON_KEY, CURRENT_LANG)
i18n.load_language(LANG_DIR, CURRENT_LANG)
return CURRENT_LANG
diff --git a/PortMaster/pylibs/pugwash/sdl/resources.py b/PortMaster/pylibs/pugwash/sdl/resources.py
index 0489c1cb..ea08f484 100644
--- a/PortMaster/pylibs/pugwash/sdl/resources.py
+++ b/PortMaster/pylibs/pugwash/sdl/resources.py
@@ -42,6 +42,15 @@ def remove_path(self, path):
self._paths.remove(path)
self.find.cache_clear()
+ @staticmethod
+ def _is_file(path):
+ ## Theme values that are text ("Zurück" on a button bar) get looked up too; a name the
+ ## filesystem refuses (EPERM on exFAT for one that isn't UTF-8) is just not a file.
+ try:
+ return path.is_file()
+ except (OSError, ValueError):
+ return False
+
@functools.lru_cache(512)
def find(self, file_name):
if isinstance(file_name, pathlib.PurePath):
@@ -59,13 +68,13 @@ def find(self, file_name):
if file_name.name in ('.', '..'):
return None
- if file_name.is_file():
+ if self._is_file(file_name):
return file_name
for path in reversed(self._paths):
full_file_name = path / file_name
- if full_file_name.is_file():
+ if self._is_file(full_file_name):
return full_file_name
return None
diff --git a/tests/test_gamelist.py b/tests/test_gamelist.py
index 1cf1da63..7fcd7f61 100644
--- a/tests/test_gamelist.py
+++ b/tests/test_gamelist.py
@@ -72,6 +72,31 @@ def test_add_creates_gamelist(platform, gameinfo):
assert platform.ports_changed()
+def test_add_writes_utf8_whatever_the_locale(platform, gameinfo, monkeypatch):
+ ## PortMaster sets LANG to the user's language; a bare "en_US" is Latin-1 where the locale
+ ## exists (dArkOS), and open() without an encoding follows it: "’" couldn't be written (the
+ ## update failed and was rolled back), and "é" went out as Latin-1 in a file that says utf-8.
+ from harbourmaster.platform import base
+
+ def latin1_open(file, mode='r', *args, **kwargs):
+ if 'b' not in mode and 'encoding' not in kwargs and not args:
+ kwargs['encoding'] = 'latin-1'
+ return open(file, mode, *args, **kwargs)
+
+ monkeypatch.setattr(base, "open", latin1_open, raising=False)
+ platform.test_gamelist.write_bytes(
+ "./Other.shIt’s a café".encode("utf-8"))
+ gameinfo.write_bytes(GAMEINFO.replace("Test Port", "Test Port ’é").encode("utf-8"))
+
+ platform.gamelist_add(gameinfo)
+
+ raw = platform.test_gamelist.read_bytes()
+ raw.decode("utf-8")
+ result = games(platform.test_gamelist)
+ assert result["./Test Port.sh"]['name'] == "Test Port ’é"
+ assert result["./Other.sh"]['name'] == "It’s a café"
+
+
def test_add_updates_existing_entry(platform, gameinfo):
platform.test_gamelist.write_text(
"./Test Port.shOld3"
diff --git a/tests/test_lang.py b/tests/test_lang.py
index 564df174..6ae55002 100644
--- a/tests/test_lang.py
+++ b/tests/test_lang.py
@@ -73,7 +73,7 @@ def test_check_lang_nothing(lang, value):
## load_lang: config.json's language, else the system's, else English
def test_system_language(lang):
assert load(lang, LANG="fr_FR.UTF-8") == ("fr_FR", "fr_FR")
- assert os.environ['LANG'] == "fr_FR"
+ assert os.environ['LANG'] == "fr_FR.UTF-8"
def test_chosen_language_wins(lang):
@@ -93,7 +93,7 @@ def test_no_usable_system_language_is_english(lang, value):
def test_no_language_variable_at_all(lang):
assert load(lang) == ("en_US", "en_US")
- assert lang.ENVIRON_KEY == "LANG" and os.environ["LANG"] == "en_US"
+ assert lang.ENVIRON_KEY == "LANG" and os.environ["LANG"] == "en_US.UTF-8"
def test_first_language_variable_is_used(lang):
@@ -102,9 +102,26 @@ def test_first_language_variable_is_used(lang):
assert "LANGUAGE" not in os.environ and "LC_ALL" not in os.environ
+@pytest.mark.parametrize("value", ["ru_RU", "ru_RU.KOI8-R", "ja_JP.eucJP", "zh_CN.GB2312"])
+def test_the_environment_gets_utf8(lang, value):
+ ## A bare or legacy locale is a legacy codeset where the locale exists (dArkOS has them
+ ## all): df, the theme server and ports PortMaster starts wrote in it, and installs failed.
+ load(lang, LANG=value)
+ assert os.environ["LANG"].endswith(".UTF-8")
+ assert lang.CURRENT_LANG == lang.check_lang(value)
+
+
+def test_language_variable_stays_a_language_list(lang):
+ ## LANGUAGE is gettext's list of languages, not a locale: no codeset.
+ assert load(lang, LANGUAGE="de_DE") == ("de_DE", "de_DE")
+ assert os.environ["LANGUAGE"] == "de_DE"
+ lang.set_language("fr_FR")
+ assert os.environ["LANGUAGE"] == "fr_FR"
+
+
def test_other_language_variable(lang):
assert load(lang, LC_ALL="ru_RU.UTF-8") == ("ru_RU", "ru_RU")
- assert lang.ENVIRON_KEY == "LC_ALL" and os.environ["LC_ALL"] == "ru_RU"
+ assert lang.ENVIRON_KEY == "LC_ALL" and os.environ["LC_ALL"] == "ru_RU.UTF-8"
################################################################################
@@ -113,7 +130,7 @@ def test_set_language(lang):
load(lang, LANG="en_US")
assert lang.set_language("de_DE") == "de_DE"
- assert lang.CURRENT_LANG == "de_DE" and os.environ[lang.ENVIRON_KEY] == "de_DE"
+ assert lang.CURRENT_LANG == "de_DE" and os.environ[lang.ENVIRON_KEY] == "de_DE.UTF-8"
assert i18n._("Options") == "Optionen"
## None, or one PortMaster hasn't: the system's.
diff --git a/tests/test_sdl_helpers.py b/tests/test_sdl_helpers.py
index 74d7b2e2..cea9307c 100644
--- a/tests/test_sdl_helpers.py
+++ b/tests/test_sdl_helpers.py
@@ -353,3 +353,26 @@ def test_animation_scene_change(pugwash, ticks):
manager.change_scene()
assert gui.data["animation.resets"] == "000"
assert gui.data["animation.keeps"] == "002"
+
+
+################################################################################
+## ResourceManager
+def test_resource_find_survives_names_the_filesystem_refuses(pugwash, tmp_path, monkeypatch):
+ ## Button bar text ("Zurück") is looked up as a file too; on exFAT a name that isn't UTF-8
+ ## gives EPERM, and Path.is_file() lets that through: the theme server's render crashed.
+ import pathlib
+ from pugwash.sdl.resources import ResourceManager
+
+ (tmp_path / "icon.png").write_bytes(b"png")
+ resources = ResourceManager(None)
+ resources.add_path(tmp_path)
+ real_is_file = pathlib.Path.is_file
+
+ def is_file(self):
+ if "Zurück" in str(self):
+ raise PermissionError(1, "Operation not permitted", str(self))
+ return real_is_file(self)
+
+ monkeypatch.setattr(pathlib.Path, "is_file", is_file)
+ assert resources.find("Zurück") is None
+ assert resources.find("icon.png") == tmp_path / "icon.png"
diff --git a/tests/test_util_edges.py b/tests/test_util_edges.py
index aaf0db12..0ac6eae9 100644
--- a/tests/test_util_edges.py
+++ b/tests/test_util_edges.py
@@ -302,6 +302,34 @@ def fail(*args, **kwargs):
assert util.get_path_fs(tmp_path) is None
+@pytest.mark.parametrize("error", [FileNotFoundError(2, "No such file or directory: 'df'"), PermissionError(13, "df")])
+def test_get_path_fs_without_df(tmp_path, monkeypatch, error):
+ ## A CFW without df (or one we can't run): the filesystem is unknown, installs go on.
+ monkeypatch.setattr(util_files, "HM_TESTING", False)
+
+ def fail(*args, **kwargs):
+ raise error
+
+ monkeypatch.setattr(util_files.subprocess, "check_output", fail)
+ assert util.get_path_fs(tmp_path) is None
+
+
+def test_get_path_fs_runs_df_in_the_c_locale(tmp_path, monkeypatch):
+ ## PortMaster sets LANG to the user's language; df's header in a legacy codeset
+ ## (ru_RU is ISO-8859-5 on dArkOS: "Файловая система" starts with 0xC4) failed every install.
+ monkeypatch.setattr(util_files, "HM_TESTING", False)
+ monkeypatch.setenv("LANG", "ru_RU")
+ calls = []
+
+ def df(args, **kwargs):
+ calls.append(kwargs.get('env') or {})
+ return "Файловая система Тип\n".encode("iso8859-5") + b"/dev/mmcblk2p1 exfat 100 50 50 50% /roms2\n"
+
+ monkeypatch.setattr(util_files.subprocess, "check_output", df)
+ assert util.get_path_fs(tmp_path) == "exfat"
+ assert calls[0].get('LC_ALL') == "C" and calls[0].get('LANG') == "ru_RU"
+
+
################################################################################
## Network
class Response: