Languages: LANG keeps its UTF-8 codeset; df, gamelist.xml and theme l…

PortMaster/pylibs/harbourmaster/platform/base.py

@@ -107,7 +107,7 @@ def gamelist_backup(self):
 
         if broken:
             logger.debug(f"Creating empty {gamelist_xml}")
-            with open(gamelist_xml, 'w') as fh:
+            with open(gamelist_xml, 'w', encoding='utf-8') as fh:
                 print(self.BLANK_GAMELIST_XML, file=fh)
 
         try:
@@ -193,7 +193,7 @@ def gamelist_add(self, gameinfo_file):
             if hasattr(ET, 'indent'):
                 ET.indent(gamelist_root, space="  ", level=0)
 
-            with open(gamelist_xml, 'w') as fh:
+            with open(gamelist_xml, 'w', encoding='utf-8') as fh:
                 print("<?xml version='1.0' encoding='utf-8'?>", file=fh)
                 print("", file=fh)
                 print(ET.tostring(gamelist_root, encoding='unicode'), file=fh)
@@ -277,7 +277,8 @@ def gamelist_add_portmaster(self):
         temp_dir = None if self.hm.temp_dir is None else str(self.hm.temp_dir)
         with tempfile.TemporaryDirectory(dir=temp_dir) as work_dir:
             temp_gameinfo = Path(work_dir) / "gameinfo.xml"
-            gameinfo_tree.write(str(temp_gameinfo), encoding='unicode')
+            ## Bytes in UTF-8, as gamelist_add() reads it back (encoding='unicode' wrote the locale's codeset).
+            gameinfo_tree.write(str(temp_gameinfo), encoding='utf-8')
             self.gamelist_add(temp_gameinfo)
 
     def set_gcd_mode(self, mode=None):

PortMaster/pylibs/harbourmaster/platform/muos.py

@@ -102,7 +102,7 @@ def gamelist_add(self, gameinfo_file):
                         text = child.text.strip().split('\n', 1)[0].strip()
 
                         logger.debug(f"creating {str(target_file)}")
-                        with open(target_file, 'w') as fh:
+                        with open(target_file, 'w', encoding='utf-8') as fh:
                             print(text, file=fh)
 
             # HAHA THIS IS FUCKED

PortMaster/pylibs/harbourmaster/platform/trimui.py

@@ -117,7 +117,7 @@ def add_port_script(self, port_script):
             new_port_dir.mkdir(0o755, parents=True, exist_ok=True)
 
             logger.debug(f"Creating {str(new_port_dir / 'config.json')}")
-            with open(new_port_dir / "config.json", "w") as fh:
+            with open(new_port_dir / "config.json", "w", encoding="utf-8") as fh:
                 fh.write(
                     PORT_CONFIG_JSON
                     .replace("{{PORTTITLE}}", port_script.stem)
@@ -258,7 +258,7 @@ def gamelist_add(self, gameinfo_file):
                     new_port_dir.mkdir(0o755, parents=True, exist_ok=True)
 
                     logger.debug(f"Creating {str(new_port_dir / 'config.json')}")
-                    with open(new_port_dir / "config.json", "w") as fh:
+                    with open(new_port_dir / "config.json", "w", encoding="utf-8") as fh:
                         fh.write(
                             PORT_CONFIG_JSON
                             .replace("{{PORTTITLE}}", port_title)

PortMaster/pylibs/harbourmaster/util/files.py

@@ -259,9 +259,13 @@ def get_path_fs(path):
     else:
         return None
 
+    ## df's own words in the C locale: PortMaster sets LANG for the language the user chose,
+    ## and a legacy codeset there (ISO-8859-5 for ru_RU on dArkOS) couldn't be decoded.
+    env = dict(os.environ, LC_ALL='C')
     try:
-        lines = subprocess.check_output(['df', '-PT', str(path)]).decode().split('\n')
-    except subprocess.CalledProcessError as err:
+        lines = subprocess.check_output(['df', '-PT', str(path)], env=env).decode('utf-8', 'replace').split('\n')
+    except (subprocess.CalledProcessError, OSError):
+        ## No df on this CFW, or it failed: we don't know the filesystem.
         return None
 
     if len(lines) < 2:

PortMaster/pylibs/pugwash/lang.py

@@ -119,18 +119,32 @@ def load_lang():
     DEFAULT_LANG = check_lang(environ_lang) or 'en_US'
 
     if config_lang and check_lang(config_lang):
-        os.environ[environ_key] = check_lang(config_lang)
+        CURRENT_LANG = check_lang(config_lang)
 
     elif environ_lang and check_lang(environ_lang):
-        os.environ[environ_key] = check_lang(environ_lang)
+        CURRENT_LANG = check_lang(environ_lang)
 
     else:
-        os.environ[environ_key] = 'en_US'
+        CURRENT_LANG = 'en_US'
 
-    CURRENT_LANG = os.environ[environ_key]
+    os.environ[environ_key] = environ_value(environ_key, CURRENT_LANG)
     ENVIRON_KEY = environ_key
 
 
+def environ_value(key, language):
+    """
+    What the environment variable `key` gets for `language`: the locale with its UTF-8
+    codeset (LANG, LC_ALL, LC_MESSAGES), or just the language for LANGUAGE (a gettext list).
+
+    A bare "ru_RU" is a legacy codeset (ISO-8859-5) on systems that have the locale (dArkOS),
+    so the programs PortMaster starts (df, the theme server, ports) wrote in it: harbourmaster
+    couldn't read df and every install failed.
+    """
+    if key == 'LANGUAGE':
+        return language
+    return language + '.UTF-8'
+
+
 def set_language(language):
     """
     Use another language from now on, None for the system's: its translations, CURRENT_LANG
@@ -141,7 +155,7 @@ def set_language(language):
     global CURRENT_LANG
 
     CURRENT_LANG = (language and check_lang(language)) or DEFAULT_LANG or 'en_US'
-    os.environ[ENVIRON_KEY] = CURRENT_LANG
+    os.environ[ENVIRON_KEY] = environ_value(ENVIRON_KEY, CURRENT_LANG)
     i18n.load_language(LANG_DIR, CURRENT_LANG)
     return CURRENT_LANG
 

PortMaster/pylibs/pugwash/sdl/resources.py

@@ -42,6 +42,15 @@ def remove_path(self, path):
             self._paths.remove(path)
             self.find.cache_clear()
 
+    @staticmethod
+    def _is_file(path):
+        ## Theme values that are text ("Zurück" on a button bar) get looked up too; a name the
+        ## filesystem refuses (EPERM on exFAT for one that isn't UTF-8) is just not a file.
+        try:
+            return path.is_file()
+        except (OSError, ValueError):
+            return False
+
     @functools.lru_cache(512)
     def find(self, file_name):
         if isinstance(file_name, pathlib.PurePath):
@@ -59,13 +68,13 @@ def find(self, file_name):
         if file_name.name in ('.', '..'):
             return None
 
-        if file_name.is_file():
+        if self._is_file(file_name):
             return file_name
 
         for path in reversed(self._paths):
             full_file_name = path / file_name
 
-            if full_file_name.is_file():
+            if self._is_file(full_file_name):
                 return full_file_name
 
         return None

tests/test_gamelist.py

@@ -72,6 +72,31 @@ def test_add_creates_gamelist(platform, gameinfo):
     assert platform.ports_changed()
 
 
+def test_add_writes_utf8_whatever_the_locale(platform, gameinfo, monkeypatch):
+    ## PortMaster sets LANG to the user's language; a bare "en_US" is Latin-1 where the locale
+    ## exists (dArkOS), and open() without an encoding follows it: "’" couldn't be written (the
+    ## update failed and was rolled back), and "é" went out as Latin-1 in a file that says utf-8.
+    from harbourmaster.platform import base
+
+    def latin1_open(file, mode='r', *args, **kwargs):
+        if 'b' not in mode and 'encoding' not in kwargs and not args:
+            kwargs['encoding'] = 'latin-1'
+        return open(file, mode, *args, **kwargs)
+
+    monkeypatch.setattr(base, "open", latin1_open, raising=False)
+    platform.test_gamelist.write_bytes(
+        "<gameList><game><path>./Other.sh</path><name>It’s a café</name></game></gameList>".encode("utf-8"))
+    gameinfo.write_bytes(GAMEINFO.replace("Test Port</name>", "Test Port ’é</name>").encode("utf-8"))
+
+    platform.gamelist_add(gameinfo)
+
+    raw = platform.test_gamelist.read_bytes()
+    raw.decode("utf-8")
+    result = games(platform.test_gamelist)
+    assert result["./Test Port.sh"]['name'] == "Test Port ’é"
+    assert result["./Other.sh"]['name'] == "It’s a café"
+
+
 def test_add_updates_existing_entry(platform, gameinfo):
     platform.test_gamelist.write_text(
         "<gameList><game><path>./Test Port.sh</path><name>Old</name><playcount>3</playcount></game>"

tests/test_lang.py

@@ -73,7 +73,7 @@ def test_check_lang_nothing(lang, value):
 ## load_lang: config.json's language, else the system's, else English
 def test_system_language(lang):
     assert load(lang, LANG="fr_FR.UTF-8") == ("fr_FR", "fr_FR")
-    assert os.environ['LANG'] == "fr_FR"
+    assert os.environ['LANG'] == "fr_FR.UTF-8"
 
 
 def test_chosen_language_wins(lang):
@@ -93,7 +93,7 @@ def test_no_usable_system_language_is_english(lang, value):
 
 def test_no_language_variable_at_all(lang):
     assert load(lang) == ("en_US", "en_US")
-    assert lang.ENVIRON_KEY == "LANG" and os.environ["LANG"] == "en_US"
+    assert lang.ENVIRON_KEY == "LANG" and os.environ["LANG"] == "en_US.UTF-8"
 
 
 def test_first_language_variable_is_used(lang):
@@ -102,9 +102,26 @@ def test_first_language_variable_is_used(lang):
     assert "LANGUAGE" not in os.environ and "LC_ALL" not in os.environ
 
 
+@pytest.mark.parametrize("value", ["ru_RU", "ru_RU.KOI8-R", "ja_JP.eucJP", "zh_CN.GB2312"])
+def test_the_environment_gets_utf8(lang, value):
+    ## A bare or legacy locale is a legacy codeset where the locale exists (dArkOS has them
+    ## all): df, the theme server and ports PortMaster starts wrote in it, and installs failed.
+    load(lang, LANG=value)
+    assert os.environ["LANG"].endswith(".UTF-8")
+    assert lang.CURRENT_LANG == lang.check_lang(value)
+
+
+def test_language_variable_stays_a_language_list(lang):
+    ## LANGUAGE is gettext's list of languages, not a locale: no codeset.
+    assert load(lang, LANGUAGE="de_DE") == ("de_DE", "de_DE")
+    assert os.environ["LANGUAGE"] == "de_DE"
+    lang.set_language("fr_FR")
+    assert os.environ["LANGUAGE"] == "fr_FR"
+
+
 def test_other_language_variable(lang):
     assert load(lang, LC_ALL="ru_RU.UTF-8") == ("ru_RU", "ru_RU")
-    assert lang.ENVIRON_KEY == "LC_ALL" and os.environ["LC_ALL"] == "ru_RU"
+    assert lang.ENVIRON_KEY == "LC_ALL" and os.environ["LC_ALL"] == "ru_RU.UTF-8"
 
 
 ################################################################################
@@ -113,7 +130,7 @@ def test_set_language(lang):
     load(lang, LANG="en_US")
 
     assert lang.set_language("de_DE") == "de_DE"
-    assert lang.CURRENT_LANG == "de_DE" and os.environ[lang.ENVIRON_KEY] == "de_DE"
+    assert lang.CURRENT_LANG == "de_DE" and os.environ[lang.ENVIRON_KEY] == "de_DE.UTF-8"
     assert i18n._("Options") == "Optionen"
 
     ## None, or one PortMaster hasn't: the system's.

tests/test_sdl_helpers.py

@@ -353,3 +353,26 @@ def test_animation_scene_change(pugwash, ticks):
     manager.change_scene()
     assert gui.data["animation.resets"] == "000"
     assert gui.data["animation.keeps"] == "002"
+
+
+################################################################################
+## ResourceManager
+def test_resource_find_survives_names_the_filesystem_refuses(pugwash, tmp_path, monkeypatch):
+    ## Button bar text ("Zurück") is looked up as a file too; on exFAT a name that isn't UTF-8
+    ## gives EPERM, and Path.is_file() lets that through: the theme server's render crashed.
+    import pathlib
+    from pugwash.sdl.resources import ResourceManager
+
+    (tmp_path / "icon.png").write_bytes(b"png")
+    resources = ResourceManager(None)
+    resources.add_path(tmp_path)
+    real_is_file = pathlib.Path.is_file
+
+    def is_file(self):
+        if "Zurück" in str(self):
+            raise PermissionError(1, "Operation not permitted", str(self))
+        return real_is_file(self)
+
+    monkeypatch.setattr(pathlib.Path, "is_file", is_file)
+    assert resources.find("Zurück") is None
+    assert resources.find("icon.png") == tmp_path / "icon.png"

tests/test_util_edges.py

@@ -302,6 +302,34 @@ def fail(*args, **kwargs):
     assert util.get_path_fs(tmp_path) is None
 
 
+@pytest.mark.parametrize("error", [FileNotFoundError(2, "No such file or directory: 'df'"), PermissionError(13, "df")])
+def test_get_path_fs_without_df(tmp_path, monkeypatch, error):
+    ## A CFW without df (or one we can't run): the filesystem is unknown, installs go on.
+    monkeypatch.setattr(util_files, "HM_TESTING", False)
+
+    def fail(*args, **kwargs):
+        raise error
+
+    monkeypatch.setattr(util_files.subprocess, "check_output", fail)
+    assert util.get_path_fs(tmp_path) is None
+
+
+def test_get_path_fs_runs_df_in_the_c_locale(tmp_path, monkeypatch):
+    ## PortMaster sets LANG to the user's language; df's header in a legacy codeset
+    ## (ru_RU is ISO-8859-5 on dArkOS: "Файловая система" starts with 0xC4) failed every install.
+    monkeypatch.setattr(util_files, "HM_TESTING", False)
+    monkeypatch.setenv("LANG", "ru_RU")
+    calls = []
+
+    def df(args, **kwargs):
+        calls.append(kwargs.get('env') or {})
+        return "Файловая система Тип\n".encode("iso8859-5") + b"/dev/mmcblk2p1 exfat 100 50 50 50% /roms2\n"
+
+    monkeypatch.setattr(util_files.subprocess, "check_output", df)
+    assert util.get_path_fs(tmp_path) == "exfat"
+    assert calls[0].get('LC_ALL') == "C" and calls[0].get('LANG') == "ru_RU"
+
+
 ################################################################################
 ## Network
 class Response: