diff --git a/docs/readers.md b/docs/readers.md index fefb4e5..bdb23e8 100644 --- a/docs/readers.md +++ b/docs/readers.md @@ -32,6 +32,18 @@ Example: {{ read_csv('assets/tables/basic_table.csv') | add_indentation(spaces=4) }} +!!! info "Multiline cells" + + A markdown table row has to fit on a single line, so newlines inside a cell are + replaced with `
`. Note that such a cell needs to be quoted to be valid CSV: + + ```csv + id,description + 23456,"Some description. + + With a line break." + ``` + ### `read_fwf` diff --git a/src/mkdocs_table_reader_plugin/markdown.py b/src/mkdocs_table_reader_plugin/markdown.py index 856a3c6..2ec24e8 100644 --- a/src/mkdocs_table_reader_plugin/markdown.py +++ b/src/mkdocs_table_reader_plugin/markdown.py @@ -3,6 +3,10 @@ import pandas as pd +# Table formats that render each row on a single line, and therefore cannot +# contain literal newlines. See https://github.com/timvink/mkdocs-table-reader-plugin/issues/83 +SINGLE_LINE_TABLE_FORMATS = ("pipe", "github") + def replace_unescaped_pipes(text: str) -> str: """ @@ -19,29 +23,53 @@ def replace_unescaped_pipes(text: str) -> str: return re.sub(r"(? str: + """ + Replace newlines with
. + + A markdown table row must fit on a single line, so a cell that contains a + newline (as multiline CSV cells do) would otherwise break up the table. + + Args: + text (str): input string + + Returns: + str: output string + """ + return re.sub(r"\r\n|\r|\n", "
", text) + + def convert_to_md_table(df: pd.DataFrame, **markdown_kwargs: dict) -> str: """ Convert dataframe to markdown table using tabulate. """ + if "index" not in markdown_kwargs: + markdown_kwargs["index"] = False + if "tablefmt" not in markdown_kwargs: + markdown_kwargs["tablefmt"] = "pipe" + # Escape any pipe characters, | to \| # See https://github.com/astanin/python-tabulate/issues/241 - df.columns = [ - replace_unescaped_pipes(c) if isinstance(c, str) else c for c in df.columns - ] + # And replace newlines with
, but only for table formats that need it: + # formats like 'grid' display multiline cells just fine. + escape_newlines = markdown_kwargs["tablefmt"] in SINGLE_LINE_TABLE_FORMATS + + def escape(value): + if not isinstance(value, str): + return value + value = replace_unescaped_pipes(value) + if escape_newlines: + value = replace_newlines(value) + return value + + df.columns = [escape(c) for c in df.columns] # Avoid deprecated applymap warning on pandas>=2.0 # See https://github.com/timvink/mkdocs-table-reader-plugin/issues/55 if pd.__version__ >= "2.1.0": - df = df.map(lambda s: replace_unescaped_pipes(s) if isinstance(s, str) else s) + df = df.map(escape) else: - df = df.applymap( - lambda s: replace_unescaped_pipes(s) if isinstance(s, str) else s - ) - - if "index" not in markdown_kwargs: - markdown_kwargs["index"] = False - if "tablefmt" not in markdown_kwargs: - markdown_kwargs["tablefmt"] = "pipe" + df = df.applymap(escape) return df.to_markdown(**markdown_kwargs) diff --git a/tests/fixtures/csv_multiline/docs/example_multiline.csv b/tests/fixtures/csv_multiline/docs/example_multiline.csv new file mode 100644 index 0000000..61afa22 --- /dev/null +++ b/tests/fixtures/csv_multiline/docs/example_multiline.csv @@ -0,0 +1,5 @@ +id,severity,description +23456,low,"Sometimes the cell text is quoted. + +But not always" +34567,high,Single line description. diff --git a/tests/fixtures/csv_multiline/docs/index.md b/tests/fixtures/csv_multiline/docs/index.md new file mode 100644 index 0000000..a470812 --- /dev/null +++ b/tests/fixtures/csv_multiline/docs/index.md @@ -0,0 +1,9 @@ +# Test page + +This test is related to this issue: https://github.com/timvink/mkdocs-table-reader-plugin/issues/83 + +## Table with a multiline cell + +A cell containing newlines, which are rendered as `
`. + +{{ read_csv("example_multiline.csv") }} diff --git a/tests/fixtures/csv_multiline/mkdocs.yml b/tests/fixtures/csv_multiline/mkdocs.yml new file mode 100644 index 0000000..9ba92b9 --- /dev/null +++ b/tests/fixtures/csv_multiline/mkdocs.yml @@ -0,0 +1,7 @@ +site_name: test git_table_reader site +use_directory_urls: true + +plugins: + - search + - table-reader: + data_path: "docs" diff --git a/tests/test_build.py b/tests/test_build.py index 004250b..77a5ffb 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -347,8 +347,12 @@ def test_wrong_path(tmp_path): result = build_docs_setup(tmp_proj) assert result.exit_code == 1, "'mkdocs build' command succeeded but should have failed" - assert "[table-reader-plugin]: Cannot find table file" in result.output - assert "non_existing_table.csv" in result.output + # Assert on the raised exception rather than on the captured log output. + # mkdocs logs the error and then re-raises it, and whether that log record + # makes it into result.output turns out to be flaky on windows. + assert isinstance(result.exception, FileNotFoundError) + assert "[table-reader-plugin]: Cannot find table file" in str(result.exception) + assert "non_existing_table.csv" in str(result.exception) def test_mixed_quotation_marks(tmp_path): @@ -408,3 +412,30 @@ def test_non_utf8_encoding(tmp_path): # read_raw() inserted the cp1251 encoded markdown file assert re.search(r"539956", contents) assert re.search(r"Сыр", contents) + + +def test_csv_with_multiline_cells(tmp_path): + """ + A CSV with a quoted, multiline cell should render as a single table row. + + See https://github.com/timvink/mkdocs-table-reader-plugin/issues/83 + """ + + tmp_proj = setup_clean_mkdocs_folder( + "tests/fixtures/csv_multiline/mkdocs.yml", tmp_path + ) + + result = build_docs_setup(tmp_proj) + assert result.exit_code == 0, "'mkdocs build' command failed" + + page_with_tag = tmp_proj / "site/index.html" + contents = page_with_tag.read_text() + + # The multiline cell is kept together on one row, separated by
+ assert re.search( + r"Sometimes the cell text is quoted\.

But not always", contents + ) + # Both CSV records became a table row, and no newline split them up + table = re.search(r".*?
", contents, flags=re.DOTALL) + assert table is not None, "no table was inserted" + assert len(re.findall(r"", table.group())) == 3 diff --git a/tests/test_markdown.py b/tests/test_markdown.py index 9425820..238ab11 100644 --- a/tests/test_markdown.py +++ b/tests/test_markdown.py @@ -1,11 +1,24 @@ import pandas as pd -from mkdocs_table_reader_plugin.markdown import convert_to_md_table, replace_unescaped_pipes + +from mkdocs_table_reader_plugin.markdown import ( + convert_to_md_table, + replace_newlines, + replace_unescaped_pipes, +) def test_unescaped_pipes(): assert replace_unescaped_pipes("hi|there\\|you|there") == "hi\\|there\\|you\\|there" +def test_replace_newlines(): + assert replace_newlines("one\ntwo") == "one
two" + assert replace_newlines("one\r\ntwo") == "one
two" + assert replace_newlines("one\rtwo") == "one
two" + assert replace_newlines("one\n\ntwo") == "one

two" + assert replace_newlines("no newlines here") == "no newlines here" + + def test_convert_to_md_table(): df_bad = pd.read_csv("tests/fixtures/csv_with_pipes/docs/example_unescaped.csv") @@ -16,4 +29,31 @@ def test_convert_to_md_table(): # Because we escape pipes, the 'bad' df md_bad = convert_to_md_table(df_bad, **{}) md_good = convert_to_md_table(df_good, **{}) - assert md_bad == md_good \ No newline at end of file + assert md_bad == md_good + + +def test_convert_to_md_table_multiline(): + """ + A multiline cell should not break up a markdown table. + + See https://github.com/timvink/mkdocs-table-reader-plugin/issues/83 + """ + df = pd.read_csv("tests/fixtures/csv_multiline/docs/example_multiline.csv") + assert df.shape == (2, 3) + + md = convert_to_md_table(df, **{}) + assert "Sometimes the cell text is quoted.

But not always" in md + # A header, a separator and one line per record + assert len(md.split("\n")) == 4 + + +def test_convert_to_md_table_multiline_other_tablefmt(): + """ + Table formats that display multiline cells themselves are left alone. + """ + df = pd.read_csv("tests/fixtures/csv_multiline/docs/example_multiline.csv") + + md = convert_to_md_table(df, tablefmt="grid") + assert "
" not in md + assert "Sometimes the cell text is quoted." in md + assert "But not always" in md