fix(loaders): say "unsupported format" for .xls, .ods and other binary files (#1018)

faircode/loaders.py

@@ -4,6 +4,9 @@
 extra: `pip install faircode[excel]`). Files with an unrecognized or missing
 extension fall back to sniffing the delimiter from their content, so a
 tab-separated export saved with a `.csv` extension still reads correctly.
+A few well-known formats that are not supported (.xls, .ods and so on) are
+rejected by name instead, because reading them as text only produces an
+encoding error (#1018).
 """
 
 from __future__ import annotations
@@ -12,6 +15,10 @@
 
 import pandas as pd
 
+# Formats people commonly try that are binary and not supported. Reading one as text fails with a
+# "not valid utf-8" error that points at --encoding, which cannot help, so name the format instead (#1018).
+UNSUPPORTED_SUFFIXES = (".xls", ".ods", ".feather", ".pkl", ".zip", ".7z")
+
 SNIFF_DELIMITERS = ",\t;|"
 SNIFF_SAMPLE_BYTES = 8192
 
@@ -67,6 +74,10 @@ def read_table(path: str, encoding: str | None = None) -> pd.DataFrame:
     if suffix == ".csv":
         return _read_delimited(path, default=",", encoding=encoding)
 
+    # A missing file keeps its "file not found" error: only an existing file is reported as the wrong format.
+    if suffix in UNSUPPORTED_SUFFIXES and Path(path).exists():
+        raise ValueError(f"unsupported format {suffix}: save it as .xlsx or .csv")
+
     return _read_delimited(path, default=",", encoding=encoding)
 
 

tests/test_cli.py

@@ -1689,3 +1689,16 @@ def test_compare_rename_across_kinds_reports_the_kind_mismatch(tmp_path, capsys)
     b.write_text("race\n" + "x\ny\n" * 40)
     assert main(["compare", str(a), str(b), "--rename", "sex=race"]) == 0
     assert "detected as sex vs race on the two datasets" in capsys.readouterr().out
+
+
+def test_unsupported_spreadsheet_format_is_named_not_blamed_on_encoding(tmp_path, capsys):
+    # #1018
+    path = tmp_path / "old.xls"
+    path.write_bytes(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 200)
+
+    with pytest.raises(SystemExit) as exc:
+        main(["profile", str(path)])
+    assert exc.value.code == 2
+    err = capsys.readouterr().err
+    assert "unsupported format .xls" in err and ".xlsx" in err
+    assert "--encoding" not in err and "not valid utf-8" not in err

tests/test_loaders.py

@@ -407,3 +407,27 @@ def test_an_empty_file_says_so_in_plain_words(tmp_path, capsys, monkeypatch):
     header_only = tmp_path / "header.csv"
     header_only.write_text("sex,race\n", encoding="utf-8")
     assert list(read_table(str(header_only)).columns) == ["sex", "race"]    # a header row is not empty
+
+
+@pytest.mark.parametrize("suffix", [".xls", ".ods", ".feather", ".pkl", ".zip", ".7z", ".XLS"])
+def test_read_table_names_an_unsupported_binary_format(tmp_path, suffix):
+    # #1018: the old file read as text and failed with a "not valid utf-8, pass --encoding" hint, which cannot help.
+    path = tmp_path / f"data{suffix}"
+    path.write_bytes(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 200)
+
+    with pytest.raises(ValueError, match=r"unsupported format .*: save it as \.xlsx or \.csv") as exc:
+        read_table(str(path))
+    assert suffix.lower() in str(exc.value)
+    assert "--encoding" not in str(exc.value)
+
+
+def test_read_table_reports_a_missing_unsupported_file_as_missing(tmp_path):
+    with pytest.raises(FileNotFoundError):
+        read_table(str(tmp_path / "gone.xls"))
+
+
+def test_read_table_still_sniffs_an_unknown_text_extension(tmp_path):
+    path = tmp_path / "data.dat"
+    path.write_text("sex,age\nM,30\nF,41\n", encoding="utf-8")
+
+    assert list(read_table(str(path)).columns) == ["sex", "age"]