diff --git a/faircode/loaders.py b/faircode/loaders.py index 4f0a1cde..98d90823 100644 --- a/faircode/loaders.py +++ b/faircode/loaders.py @@ -4,6 +4,9 @@ extra: `pip install faircode[excel]`). Files with an unrecognized or missing extension fall back to sniffing the delimiter from their content, so a tab-separated export saved with a `.csv` extension still reads correctly. +A few well-known formats that are not supported (.xls, .ods and so on) are +rejected by name instead, because reading them as text only produces an +encoding error (#1018). """ from __future__ import annotations @@ -12,6 +15,10 @@ import pandas as pd +# Formats people commonly try that are binary and not supported. Reading one as text fails with a +# "not valid utf-8" error that points at --encoding, which cannot help, so name the format instead (#1018). +UNSUPPORTED_SUFFIXES = (".xls", ".ods", ".feather", ".pkl", ".zip", ".7z") + SNIFF_DELIMITERS = ",\t;|" SNIFF_SAMPLE_BYTES = 8192 @@ -67,6 +74,10 @@ def read_table(path: str, encoding: str | None = None) -> pd.DataFrame: if suffix == ".csv": return _read_delimited(path, default=",", encoding=encoding) + # A missing file keeps its "file not found" error: only an existing file is reported as the wrong format. + if suffix in UNSUPPORTED_SUFFIXES and Path(path).exists(): + raise ValueError(f"unsupported format {suffix}: save it as .xlsx or .csv") + return _read_delimited(path, default=",", encoding=encoding) diff --git a/tests/test_cli.py b/tests/test_cli.py index 47e037bb..c31fd63c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1689,3 +1689,16 @@ def test_compare_rename_across_kinds_reports_the_kind_mismatch(tmp_path, capsys) b.write_text("race\n" + "x\ny\n" * 40) assert main(["compare", str(a), str(b), "--rename", "sex=race"]) == 0 assert "detected as sex vs race on the two datasets" in capsys.readouterr().out + + +def test_unsupported_spreadsheet_format_is_named_not_blamed_on_encoding(tmp_path, capsys): + # #1018 + path = tmp_path / "old.xls" + path.write_bytes(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 200) + + with pytest.raises(SystemExit) as exc: + main(["profile", str(path)]) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert "unsupported format .xls" in err and ".xlsx" in err + assert "--encoding" not in err and "not valid utf-8" not in err diff --git a/tests/test_loaders.py b/tests/test_loaders.py index b58ecd35..5a5e7fef 100644 --- a/tests/test_loaders.py +++ b/tests/test_loaders.py @@ -407,3 +407,27 @@ def test_an_empty_file_says_so_in_plain_words(tmp_path, capsys, monkeypatch): header_only = tmp_path / "header.csv" header_only.write_text("sex,race\n", encoding="utf-8") assert list(read_table(str(header_only)).columns) == ["sex", "race"] # a header row is not empty + + +@pytest.mark.parametrize("suffix", [".xls", ".ods", ".feather", ".pkl", ".zip", ".7z", ".XLS"]) +def test_read_table_names_an_unsupported_binary_format(tmp_path, suffix): + # #1018: the old file read as text and failed with a "not valid utf-8, pass --encoding" hint, which cannot help. + path = tmp_path / f"data{suffix}" + path.write_bytes(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 200) + + with pytest.raises(ValueError, match=r"unsupported format .*: save it as \.xlsx or \.csv") as exc: + read_table(str(path)) + assert suffix.lower() in str(exc.value) + assert "--encoding" not in str(exc.value) + + +def test_read_table_reports_a_missing_unsupported_file_as_missing(tmp_path): + with pytest.raises(FileNotFoundError): + read_table(str(tmp_path / "gone.xls")) + + +def test_read_table_still_sniffs_an_unknown_text_extension(tmp_path): + path = tmp_path / "data.dat" + path.write_text("sex,age\nM,30\nF,41\n", encoding="utf-8") + + assert list(read_table(str(path)).columns) == ["sex", "age"]