From 23ec9971362c43090b86693f6e5f00552f292366 Mon Sep 17 00:00:00 2001 From: Tim Vink Date: Mon, 14 Sep 2026 12:29:15 +0200 Subject: [PATCH 1/4] Add readers for the other formats supported by pandas Adds read_parquet, read_orc, read_html, read_xml, read_hdf, read_sas, read_spss and read_stata, and the matching pd_read_* macros. Every reader was a copy of the same two functions, so they are now built from a registry by markdown_reader() and dataframe_reader(). The keyword arguments a reader accepts can come from more than one function, which read_yaml() needs for its own 'encoding' plus pd.json_normalize(). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JCJnK1rDt8rmDsXTFc7AHE --- README.md | 2 +- docs/assets/tables/data.dta | Bin 0 -> 556 bytes docs/assets/tables/data.html | 22 ++ docs/assets/tables/data.orc | Bin 0 -> 431 bytes docs/assets/tables/data.parquet | Bin 0 -> 1676 bytes docs/assets/tables/data.xml | 15 + docs/assets/tables/data.xpt | Bin 0 -> 1120 bytes docs/options.md | 4 +- docs/readers.md | 318 ++++++++++++++++++ docs/schema.json | 72 +++- pyproject.toml | 4 +- src/mkdocs_table_reader_plugin/readers.py | 180 +++++----- src/mkdocs_table_reader_plugin/utils.py | 14 +- .../pandas_readers/assets/tables/table.dta | Bin 0 -> 544 bytes .../pandas_readers/assets/tables/table.orc | Bin 0 -> 375 bytes .../assets/tables/table.parquet | Bin 0 -> 1670 bytes .../pandas_readers/assets/tables/table.xml | 11 + .../pandas_readers/assets/tables/table.xpt | Bin 0 -> 1120 bytes tests/fixtures/pandas_readers/docs/index.md | 21 ++ tests/fixtures/pandas_readers/mkdocs.yml | 6 + .../fixtures/pandas_readers_hdf/docs/index.md | 5 + tests/fixtures/pandas_readers_hdf/mkdocs.yml | 6 + .../assets/tables/table.html | 36 ++ .../assets/tables/table.xml | 11 + .../pandas_readers_html/docs/index.md | 15 + tests/fixtures/pandas_readers_html/mkdocs.yml | 6 + .../assets/tables/table.sav | Bin 0 -> 527 bytes .../pandas_readers_spss/docs/index.md | 5 + tests/fixtures/pandas_readers_spss/mkdocs.yml | 6 + tests/test_build.py | 83 +++++ tests/test_kwargs.py | 8 + tests/test_readers.py | 20 ++ 32 files changed, 749 insertions(+), 121 deletions(-) create mode 100644 docs/assets/tables/data.dta create mode 100644 docs/assets/tables/data.html create mode 100644 docs/assets/tables/data.orc create mode 100644 docs/assets/tables/data.parquet create mode 100644 docs/assets/tables/data.xml create mode 100644 docs/assets/tables/data.xpt create mode 100644 tests/fixtures/pandas_readers/assets/tables/table.dta create mode 100644 tests/fixtures/pandas_readers/assets/tables/table.orc create mode 100644 tests/fixtures/pandas_readers/assets/tables/table.parquet create mode 100644 tests/fixtures/pandas_readers/assets/tables/table.xml create mode 100644 tests/fixtures/pandas_readers/assets/tables/table.xpt create mode 100644 tests/fixtures/pandas_readers/docs/index.md create mode 100644 tests/fixtures/pandas_readers/mkdocs.yml create mode 100644 tests/fixtures/pandas_readers_hdf/docs/index.md create mode 100644 tests/fixtures/pandas_readers_hdf/mkdocs.yml create mode 100644 tests/fixtures/pandas_readers_html/assets/tables/table.html create mode 100644 tests/fixtures/pandas_readers_html/assets/tables/table.xml create mode 100644 tests/fixtures/pandas_readers_html/docs/index.md create mode 100644 tests/fixtures/pandas_readers_html/mkdocs.yml create mode 100644 tests/fixtures/pandas_readers_spss/assets/tables/table.sav create mode 100644 tests/fixtures/pandas_readers_spss/docs/index.md create mode 100644 tests/fixtures/pandas_readers_spss/mkdocs.yml create mode 100644 tests/test_readers.py diff --git a/README.md b/README.md index 7cba439..dbcde83 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ In your markdown files you can now use: Where the path is relative to the location of your project's `mkdocs.yml` file, _or_ your project's `docs/` directory, _or_ the location of your markdown source file (all 3 possible locations will be searched, in that order). -- There are [readers](https://timvink.github.io/mkdocs-table-reader-plugin/readers/) available for many common table formats, like `.csv`, `.fwf`, `.json`, `.xls`, `.xlsx`, `.yaml`, `.feather` and `.tsv`. There is also the `read_raw()` reader that will allow you to insert tables (or other content) already in markdown format. +- There are [readers](https://timvink.github.io/mkdocs-table-reader-plugin/readers/) available for many common table formats, like `.csv`, `.fwf`, `.json`, `.xls`, `.xlsx`, `.yaml`, `.feather`, `.tsv`, `.parquet`, `.orc`, `.html`, `.xml`, `.h5`, `.sav`, `.dta` and SAS files. There is also the `read_raw()` reader that will allow you to insert tables (or other content) already in markdown format. - `table-reader` is compatible with [`mkdocs-macros-plugin`](https://mkdocs-macros-plugin.readthedocs.io/en/latest/). This enables further automation like filtering tables or inserting directories of tables. See the documentation on [compatibility with macros plugin](howto/use_jinja2.md) for more examples. ## Documentation and how-to guides diff --git a/docs/assets/tables/data.dta b/docs/assets/tables/data.dta new file mode 100644 index 0000000000000000000000000000000000000000..63d30460170f1bbeddd48470a8d5456067328d44 GIT binary patch literal 556 zcmXS7Vq{=qU}m5JFf>sJPAyO{GB7ezFf_6Ur>~vQkqxEVhYsDU+i-;bFvv2*112d5?S}&J~<;bwHO*5LJk1f CWEe*P literal 0 HcmV?d00001 diff --git a/docs/assets/tables/data.html b/docs/assets/tables/data.html new file mode 100644 index 0000000..bfabf70 --- /dev/null +++ b/docs/assets/tables/data.html @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + +
productprice
bread1.25
milk0.99
cheese4.50
\ No newline at end of file diff --git a/docs/assets/tables/data.orc b/docs/assets/tables/data.orc new file mode 100644 index 0000000000000000000000000000000000000000..d486a309e68ad4e675a4403af613b3ff01777754 GIT binary patch literal 431 zcmeYda^_+aV&Py8VBnJBVrF1q5EA8JR^sPkO)5%FOc7$q&CJP`kORu9aWOIoDRMAN zNpr4qe^@25?!CPrP&r74gXA}jnEpdwr5pkn+?YZVz*<2XlQU9Ni&G)0zSzUGLHI%r zTx=X{0t^xyV46`v4ooviXaH#@0Y(WHAk7G*`N1@kgapu;3<3;5k_k)#*;35zz9C!= zToCu;a5!N%b1(}WV3gtlQhpNJ3ho-rN-`WgLQITIQtSmq`6;EzB~q*fMVZN|3Je+y z1`J9Z9PDrw8-!)RYz20N6?R8h5q5;b1V0bPBvvCm13d$diHyL2X=q?kVq)OYa57+P M`ohfYALJ|n00UJ}!Tf&q5M9YOYZyKlfhka^4g!$~IVi5E4<|Ajq)R!Hs8~v@B%-9EG5BSP(vo6G zN-|^QR`lFse?t#B^i=ex^iZJp0=@JHbe5!4NA{%)ESEcP=DpeBkbH%^0=vx~vnOXP z!=if#eL3Di2&t~2=-Y5(nPkeqs*Mf9Gg!nPvRocq4`4ExWAt#FdT={o0Ql|QPr{W+ zB)9G(ATF?l6)}gemMN}_cL&VaqpW=PHuOFaJwtt6nc1v75qMzOMhKK z9OFT*mcni$cK<<=`@(T&`dsrLptB4FuzO(Jaa>a~P>M-s()k?9u>0u>Vh+-4XEvE) zQa>kBza(<+a=W*Vna;z}5Sv7BIe-NrO0pM|MC>8O?}^kOiQEs#+;54@|2Wre{?zwd zDJHj@`f=+Jp>&>O9L3fZkNYcm&6~DvTwQ20GPi7xKgoXkT|Uc4+lvr0Hmu?Hnzt3p z0Czqv5D!t8ra|;!VjD`g@||6O{qc|+}z%oP8)d|T}XuqhYT0GimJxsbs%PfXQS+#oz}pt$aAN$D1b>zOl~ zT7~LS;Yi>iPd=8v0Tz#nM(JS|horT)$ z`KsMEUz(!Yf4v+@R%|0xN42`dX!Jfvli5fX3Z7JJP1NF*pvslVR`N{Ms{6y&(lKC5 z&8kjyxl@o#$}@C^^3}9otc+FpE%0MgYql1{W^ba`B+9iM$d&1!-?`MAWEpb^szpGf zOCpYnp!UT|ztpX0jfg`nw%!ld8UoQ9b>QfZX^yJh^VBBf+1$*FOn{$iT8iBBhV|-v zAPa=XNVXU#awpQW#W9?DtO?bE(HA4!8hUE6Ug%8D&9fRdLBrRYmf!*`du!+P~ ze^C!Q-og~&xkCNUaf9{~^e3$0s8JGVR^S_e7p4NV8CU%*irvspqt~$i!YyvnW!(#L tNQ6Iqiny$!xbNqN|44J3N8t;XjbEs2^s1%v@I9uV3^9ue${{kS0(vJWD literal 0 HcmV?d00001 diff --git a/docs/assets/tables/data.xml b/docs/assets/tables/data.xml new file mode 100644 index 0000000..3e5683f --- /dev/null +++ b/docs/assets/tables/data.xml @@ -0,0 +1,15 @@ + + + + bread + 1.25 + + + milk + 0.99 + + + cheese + 4.5 + + diff --git a/docs/assets/tables/data.xpt b/docs/assets/tables/data.xpt new file mode 100644 index 0000000000000000000000000000000000000000..f53786e3a13da9c82dbab512f101d30f57cdc77d GIT binary patch literal 1120 zcmeZpb#!qJQV4Q&_78H=f&d>+ry$3mNCgjMIYkICzzY-F~lG0`&uv9JR}lVH~XBQq;QBP%0wD-#oRK6YKii=y}m>R(@1UngJ~5a(Y*Ggt^1 zni!zzftu$M>>Lyj5=5MNFw?*^BHUaYLmY!$LqOpOVh-x?^K>s+=4$r-7s#i@=S L5II6FgUAB__d&C? literal 0 HcmV?d00001 diff --git a/docs/options.md b/docs/options.md index 0854204..bd91c39 100644 --- a/docs/options.md +++ b/docs/options.md @@ -39,7 +39,9 @@ Default: `False`. When enabled, if a filepath is not found, the plugin will rais ## `select_readers` -Default: Selects all available readers. Specify a list of readers to improve documentation build times for very large sites. This option is ignored when you use this plugin with `mkdocs-macros-plugin` ([read more](howto/use_jinja2.md)) +Default: Selects all available readers. Every reader runs its own regex over all your markdown content, so specifying only the readers you use improves documentation build times for very large sites. This option is ignored when you use this plugin with `mkdocs-macros-plugin` ([read more](howto/use_jinja2.md)) + +Note that some readers need an additional package to be installed, see [readers](readers.md). ## `enabled` diff --git a/docs/readers.md b/docs/readers.md index bdb23e8..66937b3 100644 --- a/docs/readers.md +++ b/docs/readers.md @@ -199,6 +199,192 @@ Example: ``` {% endraw %} +### `read_parquet` + +Use {% raw %}`{{ read_parquet() }}`{% endraw %} to read a parquet file and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_parquet()](https://pandas.pydata.org/docs/reference/api/pandas.read_parquet.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ read_parquet('assets/tables/data.parquet') }} + ``` + {% endraw %} + +=== "Output" + + {{ read_parquet('assets/tables/data.parquet') | add_indentation(spaces=4) }} + +Requires [pyarrow](https://arrow.apache.org/docs/python/install.html) or [fastparquet](https://fastparquet.readthedocs.io/en/latest/install.html) to be installed. + +### `read_orc` + +Use {% raw %}`{{ read_orc() }}`{% endraw %} to read an ORC object and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_orc()](https://pandas.pydata.org/docs/reference/api/pandas.read_orc.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ read_orc('assets/tables/data.orc') }} + ``` + {% endraw %} + +=== "Output" + + {{ read_orc('assets/tables/data.orc') | add_indentation(spaces=4) }} + +Requires [pyarrow](https://arrow.apache.org/docs/python/install.html) to be installed. + +### `read_xml` + +Use {% raw %}`{{ read_xml() }}`{% endraw %} to read an XML document and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_xml()](https://pandas.pydata.org/docs/reference/api/pandas.read_xml.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ read_xml('assets/tables/data.xml') }} + ``` + {% endraw %} + +=== "Output" + + {{ read_xml('assets/tables/data.xml') | add_indentation(spaces=4) }} + +Requires [lxml](https://lxml.de/installation.html) to be installed, or use the standard library parser with {% raw %}`{{ read_xml('assets/tables/data.xml', parser='etree') }}`{% endraw %}. + +### `read_html` + +Use {% raw %}`{{ read_html() }}`{% endraw %} to read the first table in an HTML document and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_html()](https://pandas.pydata.org/docs/reference/api/pandas.read_html.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ read_html('assets/tables/data.html') }} + ``` + {% endraw %} + +=== "Output" + + {{ read_html('assets/tables/data.html') | add_indentation(spaces=4) }} + +`pandas.read_html()` returns every table it finds, so the first one is inserted. Use the `match` argument to select another table, for example {% raw %}`{{ read_html('assets/tables/data.html', match='price') }}`{% endraw %}. Requires [lxml](https://lxml.de/installation.html), or [beautifulsoup4](https://pypi.org/project/beautifulsoup4/) and [html5lib](https://pypi.org/project/html5lib/), to be installed. + +### `read_stata` + +Use {% raw %}`{{ read_stata() }}`{% endraw %} to read a Stata file and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_stata()](https://pandas.pydata.org/docs/reference/api/pandas.read_stata.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ read_stata('assets/tables/data.dta') }} + ``` + {% endraw %} + +=== "Output" + + {{ read_stata('assets/tables/data.dta') | add_indentation(spaces=4) }} + +### `read_sas` + +Use {% raw %}`{{ read_sas() }}`{% endraw %} to read a SAS file (XPORT or SAS7BDAT) and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_sas()](https://pandas.pydata.org/docs/reference/api/pandas.read_sas.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ read_sas('assets/tables/data.xpt', encoding='utf-8') }} + ``` + {% endraw %} + +=== "Output" + + {{ read_sas('assets/tables/data.xpt', encoding='utf-8') | add_indentation(spaces=4) }} + +Text columns are read as bytes unless you specify the `encoding` to decode them with. + +### `read_spss` + +Use {% raw %}`{{ read_spss() }}`{% endraw %} to read an SPSS file and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_spss()](https://pandas.pydata.org/docs/reference/api/pandas.read_spss.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +{% raw %} +```markdown +{{ read_spss('assets/tables/data.sav') }} +``` +{% endraw %} + +Requires [pyreadstat](https://github.com/Roche/pyreadstat) to be installed. + +### `read_hdf` + +Use {% raw %}`{{ read_hdf() }}`{% endraw %} to read an object stored in a HDF5 file and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_hdf()](https://pandas.pydata.org/docs/reference/api/pandas.read_hdf.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +{% raw %} +```markdown +{{ read_hdf('assets/tables/data.h5', key='table') }} +``` +{% endraw %} + +Requires [pytables](https://www.pytables.org/usersguide/installation.html) to be installed. Specify the `key` of the object to read when the file contains more than one. + ### `read_raw` Use {% raw %}`{{ read_raw() }}`{% endraw %} to insert the contents from a file directly. @@ -378,6 +564,138 @@ Example: {% endraw %} +### `pd_read_parquet` + +Use {% raw %}`{{ pd_read_parquet() }}`{% endraw %} to read a parquet file using [pandas.read_parquet()](https://pandas.pydata.org/docs/reference/api/pandas.read_parquet.html) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ pd_read_parquet('assets/tables/data.parquet').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + ``` + {% endraw %} + +=== "Output" + + {{ pd_read_parquet('assets/tables/data.parquet').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + +### `pd_read_orc` + +Use {% raw %}`{{ pd_read_orc() }}`{% endraw %} to read an ORC object using [pandas.read_orc()](https://pandas.pydata.org/docs/reference/api/pandas.read_orc.html) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ pd_read_orc('assets/tables/data.orc').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + ``` + {% endraw %} + +=== "Output" + + {{ pd_read_orc('assets/tables/data.orc').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + +### `pd_read_xml` + +Use {% raw %}`{{ pd_read_xml() }}`{% endraw %} to read an XML document using [pandas.read_xml()](https://pandas.pydata.org/docs/reference/api/pandas.read_xml.html) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ pd_read_xml('assets/tables/data.xml').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + ``` + {% endraw %} + +=== "Output" + + {{ pd_read_xml('assets/tables/data.xml').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + +### `pd_read_html` + +Use {% raw %}`{{ pd_read_html() }}`{% endraw %} to read the first table in an HTML document using [pandas.read_html()](https://pandas.pydata.org/docs/reference/api/pandas.read_html.html) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ pd_read_html('assets/tables/data.html').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + ``` + {% endraw %} + +=== "Output" + + {{ pd_read_html('assets/tables/data.html').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + +### `pd_read_stata` + +Use {% raw %}`{{ pd_read_stata() }}`{% endraw %} to read a Stata file using [pandas.read_stata()](https://pandas.pydata.org/docs/reference/api/pandas.read_stata.html) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ pd_read_stata('assets/tables/data.dta').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + ``` + {% endraw %} + +=== "Output" + + {{ pd_read_stata('assets/tables/data.dta').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + +### `pd_read_sas` + +Use {% raw %}`{{ pd_read_sas() }}`{% endraw %} to read a SAS file (XPORT or SAS7BDAT) using [pandas.read_sas()](https://pandas.pydata.org/docs/reference/api/pandas.read_sas.html) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ pd_read_sas('assets/tables/data.xpt', encoding='utf-8').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + ``` + {% endraw %} + +=== "Output" + + {{ pd_read_sas('assets/tables/data.xpt', encoding='utf-8').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + +### `pd_read_spss` + +Use {% raw %}`{{ pd_read_spss() }}`{% endraw %} to read an SPSS file using [pandas.read_spss()](https://pandas.pydata.org/docs/reference/api/pandas.read_spss.html) + +Example: + +{% raw %} +```markdown +{{ pd_read_spss('assets/tables/data.sav').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} +``` +{% endraw %} + +### `pd_read_hdf` + +Use {% raw %}`{{ pd_read_hdf() }}`{% endraw %} to read an object stored in a HDF5 file using [pandas.read_hdf()](https://pandas.pydata.org/docs/reference/api/pandas.read_hdf.html) + +Example: + +{% raw %} +```markdown +{{ pd_read_hdf('assets/tables/data.h5', key='table').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} +``` +{% endraw %} + ## Filters When you use `table-reader` with [`mkdocs-macros-plugin`](https://mkdocs-macros-plugin.readthedocs.io/en/latest/), in next to all the readers, the macros, the following _additional_ filters will be made available: diff --git a/docs/schema.json b/docs/schema.json index f7a6a55..22c47df 100644 --- a/docs/schema.json +++ b/docs/schema.json @@ -4,7 +4,9 @@ "oneOf": [ { "markdownDescription": "https://timvink.github.io/mkdocs-table-reader-plugin/", - "enum": ["table-reader"] + "enum": [ + "table-reader" + ] }, { "type": "object", @@ -36,21 +38,26 @@ "title": "{{ read_csv() }} passed to pandas.read_csv()", "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_csv" }, + { + "const": "read_table", + "title": "{{ read_table() }} passed to pandas.read_table()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_table" + }, { "const": "read_fwf", "title": "{{ read_fwf() }} passed to pandas.read_fwf()", "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_fwf" }, + { + "const": "read_excel", + "title": "{{ read_excel() }} passed to pandas.read_excel()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_excel" + }, { "const": "read_yaml", "title": "{{ read_yaml() }} is parsed with yaml.safe_load() and passed to pandas.json_normalize()", "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_yaml" }, - { - "const": "read_table", - "title": "{{ read_table() }} passed to pandas.read_table()", - "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_table" - }, { "const": "read_json", "title": "{{ read_json() }} passed to pandas.read_json()", @@ -62,9 +69,44 @@ "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_feather" }, { - "const": "read_excel", - "title": "{{ read_excel() }} passed to pandas.read_excel()", - "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_excel" + "const": "read_parquet", + "title": "{{ read_parquet() }} passed to pandas.read_parquet()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_parquet" + }, + { + "const": "read_orc", + "title": "{{ read_orc() }} passed to pandas.read_orc()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_orc" + }, + { + "const": "read_html", + "title": "{{ read_html() }} passed to pandas.read_html()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_html" + }, + { + "const": "read_xml", + "title": "{{ read_xml() }} passed to pandas.read_xml()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_xml" + }, + { + "const": "read_hdf", + "title": "{{ read_hdf() }} passed to pandas.read_hdf()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_hdf" + }, + { + "const": "read_sas", + "title": "{{ read_sas() }} passed to pandas.read_sas()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_sas" + }, + { + "const": "read_spss", + "title": "{{ read_spss() }} passed to pandas.read_spss()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_spss" + }, + { + "const": "read_stata", + "title": "{{ read_stata() }} passed to pandas.read_stata()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_stata" }, { "const": "read_raw", @@ -75,12 +117,20 @@ }, "default": [ "read_csv", + "read_table", "read_fwf", + "read_excel", "read_yaml", - "read_table", "read_json", "read_feather", - "read_excel", + "read_parquet", + "read_orc", + "read_html", + "read_xml", + "read_hdf", + "read_sas", + "read_spss", + "read_stata", "read_raw" ] } diff --git a/pyproject.toml b/pyproject.toml index e909601..88b80ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ classifiers=[ dynamic = ["version"] dependencies = [ "mkdocs>=1.0", - "pandas>=1.1", + "pandas>=1.3", # pd.read_xml() was added in pandas 1.3 "pyyaml>=5.4.1", "tabulate>=0.8.7", ] @@ -116,6 +116,8 @@ fix = true dev-dependencies = [ "click>=8.1.8", "codecov>=2.1.13", + # lxml is used by pd.read_html() and pd.read_xml(), no cp38 wheels for macos arm64 + "lxml>=5; python_full_version >= '3.9'", "mkdocs-git-authors-plugin>=0.9.4", "mkdocs-git-revision-date-localized-plugin>=1.4.5", "mkdocs-macros-plugin>=1.3.7", diff --git a/src/mkdocs_table_reader_plugin/readers.py b/src/mkdocs_table_reader_plugin/readers.py index 86bbea4..16477cd 100644 --- a/src/mkdocs_table_reader_plugin/readers.py +++ b/src/mkdocs_table_reader_plugin/readers.py @@ -60,109 +60,80 @@ def __call__(self, *args, **kwargs): return self.func(valid_file_paths[0], *args, **kwargs) -@ParseArgs -def pd_read_csv(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_csv) - return pd.read_csv(*args, **read_kwargs) - -@ParseArgs -def read_csv(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_csv) - df = pd.read_csv(*args, **read_kwargs) - - markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_csv) - return convert_to_md_table(df, **markdown_kwargs) - - -@ParseArgs -def pd_read_table(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_table) - return pd.read_table(*args, **read_kwargs) - -@ParseArgs -def read_table(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_table) - df = pd.read_table(*args, **read_kwargs) - - markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_table) - return convert_to_md_table(df, **markdown_kwargs) - - -@ParseArgs -def pd_read_fwf(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_fwf) - return pd.read_fwf(*args, **read_kwargs) - +def read_yaml_file(filepath, encoding: str = "utf-8", **kwargs) -> pd.DataFrame: + """ + Read a YAML file into a pd.DataFrame. -@ParseArgs -def read_fwf(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_fwf) - df = pd.read_fwf(*args, **read_kwargs) + The contents are parsed with yaml.safe_load() and passed to pd.json_normalize(). + """ + with open(filepath, encoding=encoding) as f: + return pd.json_normalize(yaml.safe_load(f), **kwargs) - markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_fwf) - return convert_to_md_table(df, **markdown_kwargs) -@ParseArgs -def pd_read_json(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_json) - return pd.read_json(*args, **read_kwargs) +def read_html_file(*args, **kwargs) -> pd.DataFrame: + """ + Read the first table of an HTML file into a pd.DataFrame. + pd.read_html() returns all tables it finds, use the 'match' argument to select one. + """ + return pd.read_html(*args, **kwargs)[0] -@ParseArgs -def read_json(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_json) - df = pd.read_json(*args, **read_kwargs) - markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_json) - return convert_to_md_table(df, **markdown_kwargs) +def read_hdf_file(*args, **kwargs) -> pd.DataFrame: + """ + Read a HDF5 file into a pd.DataFrame. -@ParseArgs -def pd_read_excel(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_excel) - return pd.read_excel(*args, **read_kwargs) + pd.read_hdf() returns a pd.Series when a Series was stored. + """ + data = pd.read_hdf(*args, **kwargs) + if isinstance(data, pd.Series): + data = data.to_frame() + return data -@ParseArgs -def read_excel(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_excel) - df = pd.read_excel(*args, **read_kwargs) +def markdown_reader(load_function, *extra_kwarg_sources) -> ParseArgs: + """ + Create a reader that inserts a file as a markdown table. - markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_excel) - return convert_to_md_table(df, **markdown_kwargs) + Args: + load_function: function that reads a file path into a pd.DataFrame + extra_kwarg_sources: functions with additional keyword arguments accepted + by load_function, on top of its own. Any other keyword arguments are + passed on to pd.DataFrame.to_markdown() + Returns: + ParseArgs: reader that returns a markdown table + """ + kwarg_sources = (load_function, *extra_kwarg_sources) -@ParseArgs -def pd_read_yaml(*args, **kwargs) -> str: - encoding = kwargs.pop("encoding", "utf-8") - json_kwargs = kwargs_in_func(kwargs, pd.json_normalize) - with open(args[0], encoding=encoding) as f: - df = pd.json_normalize(yaml.safe_load(f), **json_kwargs) - return df + @functools.wraps(load_function) + def reader(*args, **kwargs) -> str: + df = load_function(*args, **kwargs_in_func(kwargs, *kwarg_sources)) + markdown_kwargs = kwargs_not_in_func(kwargs, *kwarg_sources) + return convert_to_md_table(df, **markdown_kwargs) -@ParseArgs -def read_yaml(*args, **kwargs) -> str: - encoding = kwargs.pop("encoding", "utf-8") - json_kwargs = kwargs_in_func(kwargs, pd.json_normalize) - with open(args[0], encoding=encoding) as f: - df = pd.json_normalize(yaml.safe_load(f), **json_kwargs) + return ParseArgs(reader) - markdown_kwargs = kwargs_not_in_func(kwargs, pd.json_normalize) - return convert_to_md_table(df, **markdown_kwargs) +def dataframe_reader(load_function, *extra_kwarg_sources) -> ParseArgs: + """ + Create a macro that returns a file as a pd.DataFrame. -@ParseArgs -def pd_read_feather(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_feather) - return pd.read_feather(*args, **read_kwargs) + Args: + load_function: function that reads a file path into a pd.DataFrame + extra_kwarg_sources: functions with additional keyword arguments accepted + by load_function, on top of its own + Returns: + ParseArgs: reader that returns a pd.DataFrame + """ + kwarg_sources = (load_function, *extra_kwarg_sources) -@ParseArgs -def read_feather(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_feather) - df = pd.read_feather(*args, **read_kwargs) + @functools.wraps(load_function) + def reader(*args, **kwargs) -> pd.DataFrame: + return load_function(*args, **kwargs_in_func(kwargs, *kwarg_sources)) - markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_feather) - return convert_to_md_table(df, **markdown_kwargs) + return ParseArgs(reader) @ParseArgs @@ -177,24 +148,29 @@ def read_raw(*args, **kwargs) -> str: return f.read() -READERS = { - "read_csv": read_csv, - "read_table": read_table, - "read_fwf": read_fwf, - "read_excel": read_excel, - "read_yaml": read_yaml, - "read_json": read_json, - "read_feather": read_feather, - "read_raw": read_raw, +# The function used to load each file format into a pd.DataFrame, +# optionally followed by functions with additional keyword arguments it accepts. +LOADERS = { + "read_csv": (pd.read_csv,), + "read_table": (pd.read_table,), + "read_fwf": (pd.read_fwf,), + "read_excel": (pd.read_excel,), + "read_yaml": (read_yaml_file, pd.json_normalize), + "read_json": (pd.read_json,), + "read_feather": (pd.read_feather,), + "read_parquet": (pd.read_parquet,), + "read_orc": (pd.read_orc,), + "read_html": (read_html_file, pd.read_html), + "read_xml": (pd.read_xml,), + "read_hdf": (read_hdf_file, pd.read_hdf), + "read_sas": (pd.read_sas,), + "read_spss": (pd.read_spss,), + "read_stata": (pd.read_stata,), } -MACRO_ONLY = { - "pd_read_csv": pd_read_csv, - "pd_read_table": pd_read_table, - "pd_read_fwf": pd_read_fwf, - "pd_read_excel": pd_read_excel, - "pd_read_yaml": pd_read_yaml, - "pd_read_json": pd_read_json, - "pd_read_feather": pd_read_feather, -} +READERS = {name: markdown_reader(*loader) for name, loader in LOADERS.items()} +READERS["read_raw"] = read_raw + +MACRO_ONLY = {f"pd_{name}": dataframe_reader(*loader) for name, loader in LOADERS.items()} + MACROS = {**READERS, **MACRO_ONLY} diff --git a/src/mkdocs_table_reader_plugin/utils.py b/src/mkdocs_table_reader_plugin/utils.py index fb2f8e6..c524db3 100644 --- a/src/mkdocs_table_reader_plugin/utils.py +++ b/src/mkdocs_table_reader_plugin/utils.py @@ -2,20 +2,24 @@ from inspect import signature -def get_keywords(func): +def get_keywords(*funcs): + """Collect the keyword arguments accepted by one or more functions.""" return [ p.name + for func in funcs for p in signature(func).parameters.values() if p.kind == p.POSITIONAL_OR_KEYWORD or p.kind == p.KEYWORD_ONLY ] -def kwargs_in_func(keywordargs, func): - return {k: v for k, v in keywordargs.items() if k in get_keywords(func)} +def kwargs_in_func(keywordargs, *funcs): + keywords = get_keywords(*funcs) + return {k: v for k, v in keywordargs.items() if k in keywords} -def kwargs_not_in_func(keywordargs, func): - return {k: v for k, v in keywordargs.items() if k not in get_keywords(func)} +def kwargs_not_in_func(keywordargs, *funcs): + keywords = get_keywords(*funcs) + return {k: v for k, v in keywordargs.items() if k not in keywords} class cd: diff --git a/tests/fixtures/pandas_readers/assets/tables/table.dta b/tests/fixtures/pandas_readers/assets/tables/table.dta new file mode 100644 index 0000000000000000000000000000000000000000..2c227a9a34c62cfd7dfb6b94fe6bed8470c9a20a GIT binary patch literal 544 zcmXS7Vq{0sIN*X~p1C=Na0A@(&Pt?Jxxxaf{ln<2=&FXn!)Q|H3_5;z!U9nH?i3QW_@k; lcWt&?o4?&$v2t%b3Zp=;EWY2i!XpgQ^B`ZoIiJo8`vb>0Kx6;_ literal 0 HcmV?d00001 diff --git a/tests/fixtures/pandas_readers/assets/tables/table.parquet b/tests/fixtures/pandas_readers/assets/tables/table.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c441066ba2f460fa0f145ade352d38b636705cb1 GIT binary patch literal 1670 zcmb7F&5qhu6uuA_QE4KLqS`VN3xpb!1({+Go{A`0T+?7ofn*p3gQJM}8V{JUjRR(Y z8D*Dc9-t4BWtUxa*@x(3bkTFK&6FYSqL#Rq`-3 z5kjH0_^@)KQDhO%;Q(07*D{azN4!u5;xV7$Gx;<0grm6l@E#r5-xkuaa;emVG_VW& z_dE6D=O~{A6GE@qiqN5`nB6hN?2aR#C;x(iz`Z!R=L!-tOMB}07SAF6=y6))0?T(E zqjnz1JO@A~Soy?RAc4!}bLA4s^G7)aWsY+(l}`(t@FgYunkt#4?0;66qfJ13nr43t z8$wixOO%orOzZcQ@JFijDP8(4mH!{5eH}pf3sNg(h0hPZgPT+qGoclXt)=*9`nKo> zV`nq6=ibT>mc?g8N$C%8Ugn5y2huPONQ&fZ#snV#WdD$K59 zyvAUForCz>BBQh`aGqA0OhUDok)i6WT*jx={WlD@UppADts<9p# z%GGety&N~ab>b@#-vjMl;;%?#pelpYo?>f>zufv@8r@2)FL`6F0UEt2&5Z>8C22x_ z&HMR`4aqeJy4$W}*YrJ)XuZq*eE03P)_|TgZuVnZhEnOy&Y3^Ni0r1fpL0lpbGcjB zX-$y{xqyFVNPD3{{@SLN^mA((jS4x{X^)}zn{XJkrgm_}`U0BpzjZy{G9B3*v~Db; zwxV_EvzJPHhF6`7R*ly1PmZQ%2i=g!Ri@E#tK0Uh<6g#-q|sZwmd!Baa7pRxz8kUUU@_)*K1;dM;k$X~vJ?*M#({s!jE ByO#g} literal 0 HcmV?d00001 diff --git a/tests/fixtures/pandas_readers/assets/tables/table.xml b/tests/fixtures/pandas_readers/assets/tables/table.xml new file mode 100644 index 0000000..3bff653 --- /dev/null +++ b/tests/fixtures/pandas_readers/assets/tables/table.xml @@ -0,0 +1,11 @@ + + + + xml_table + 531456 + + + row2 + 80 + + diff --git a/tests/fixtures/pandas_readers/assets/tables/table.xpt b/tests/fixtures/pandas_readers/assets/tables/table.xpt new file mode 100644 index 0000000000000000000000000000000000000000..9ba971ad10ea5b3fb784d8db2641d06eb7643f6a GIT binary patch literal 1120 zcmd5*u?~VT5Uo!77w76=fFOx%1)69gN?H~t1CdQI0gbbN=BL?G8pRN4n0y1*yW<}B zj^3K!fKZ^s-_u|&vE6tS(*vwOb2AC!Cjb~`An;*19e}ei91%~elfAISsT-!vNMt## zX}SElTz&i9Rr=AErbcf_LXXctPp|2y37EF=RnzbV%%@RI_4pK9ZQpYXaEuv=#T>=) zujjYJAz^)bt(MTn0YGDf&=ie|B+vB!;`>crT|-5bKgf=F)dY=;Tb^d6V$`?wH#vu@ vXnL3q=|yq$*{=J%_v8+?a+>>FCDrkor00x0o^{{lXej@tPbfRC`? literal 0 HcmV?d00001 diff --git a/tests/fixtures/pandas_readers/docs/index.md b/tests/fixtures/pandas_readers/docs/index.md new file mode 100644 index 0000000..89796aa --- /dev/null +++ b/tests/fixtures/pandas_readers/docs/index.md @@ -0,0 +1,21 @@ +# Test page + +## read_parquet + +{{ read_parquet('assets/tables/table.parquet') }} + +## read_orc + +{{ read_orc('assets/tables/table.orc') }} + +## read_stata + +{{ read_stata('assets/tables/table.dta') }} + +## read_sas + +{{ read_sas('assets/tables/table.xpt', encoding='utf-8') }} + +## read_xml + +{{ read_xml('assets/tables/table.xml', parser='etree') }} diff --git a/tests/fixtures/pandas_readers/mkdocs.yml b/tests/fixtures/pandas_readers/mkdocs.yml new file mode 100644 index 0000000..cea9ccc --- /dev/null +++ b/tests/fixtures/pandas_readers/mkdocs.yml @@ -0,0 +1,6 @@ +site_name: test git_table_reader site +use_directory_urls: false + +plugins: + - search + - table-reader diff --git a/tests/fixtures/pandas_readers_hdf/docs/index.md b/tests/fixtures/pandas_readers_hdf/docs/index.md new file mode 100644 index 0000000..5542e70 --- /dev/null +++ b/tests/fixtures/pandas_readers_hdf/docs/index.md @@ -0,0 +1,5 @@ +# Test page + +## read_hdf + +{{ read_hdf('assets/tables/table.h5', key='table') }} diff --git a/tests/fixtures/pandas_readers_hdf/mkdocs.yml b/tests/fixtures/pandas_readers_hdf/mkdocs.yml new file mode 100644 index 0000000..cea9ccc --- /dev/null +++ b/tests/fixtures/pandas_readers_hdf/mkdocs.yml @@ -0,0 +1,6 @@ +site_name: test git_table_reader site +use_directory_urls: false + +plugins: + - search + - table-reader diff --git a/tests/fixtures/pandas_readers_html/assets/tables/table.html b/tests/fixtures/pandas_readers_html/assets/tables/table.html new file mode 100644 index 0000000..8e5f1bf --- /dev/null +++ b/tests/fixtures/pandas_readers_html/assets/tables/table.html @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + +
namenumber
html_table531456
row280
+ + + + + + + + + + + + + + + + + +
namenumber
second_html_table539956
row212
\ No newline at end of file diff --git a/tests/fixtures/pandas_readers_html/assets/tables/table.xml b/tests/fixtures/pandas_readers_html/assets/tables/table.xml new file mode 100644 index 0000000..3bff653 --- /dev/null +++ b/tests/fixtures/pandas_readers_html/assets/tables/table.xml @@ -0,0 +1,11 @@ + + + + xml_table + 531456 + + + row2 + 80 + + diff --git a/tests/fixtures/pandas_readers_html/docs/index.md b/tests/fixtures/pandas_readers_html/docs/index.md new file mode 100644 index 0000000..0e997c9 --- /dev/null +++ b/tests/fixtures/pandas_readers_html/docs/index.md @@ -0,0 +1,15 @@ +# Test page + +## read_html + +Inserts the first table in the file: + +{{ read_html('assets/tables/table.html') }} + +Use 'match' to select another table: + +{{ read_html('assets/tables/table.html', match='second_html_table') }} + +## read_xml + +{{ read_xml('assets/tables/table.xml') }} diff --git a/tests/fixtures/pandas_readers_html/mkdocs.yml b/tests/fixtures/pandas_readers_html/mkdocs.yml new file mode 100644 index 0000000..cea9ccc --- /dev/null +++ b/tests/fixtures/pandas_readers_html/mkdocs.yml @@ -0,0 +1,6 @@ +site_name: test git_table_reader site +use_directory_urls: false + +plugins: + - search + - table-reader diff --git a/tests/fixtures/pandas_readers_spss/assets/tables/table.sav b/tests/fixtures/pandas_readers_spss/assets/tables/table.sav new file mode 100644 index 0000000000000000000000000000000000000000..ce2ed9c27074f6b278f172657d9e4af71aa1adc5 GIT binary patch literal 527 zcmbtRO-sW-5Z%W4I#uD(*S3UN5Ab-oaW?QwjIxLJ@S{4^9(GQ{! z?zpcQMq(XW0>mBgJD*lGni~S_-$&QsfM|=$*k_&DWbQK!;LEQG5I&z@LO^e{=!2%J zmO50+_wV`)-A=9D.*?", contents, flags=re.DOTALL) assert table is not None, "no table was inserted" assert len(re.findall(r"", table.group())) == 3 + + +def test_pandas_readers(tmp_path): + """ + A project that uses the readers for the other formats supported by pandas. + """ + + tmp_proj = setup_clean_mkdocs_folder( + "tests/fixtures/pandas_readers/mkdocs.yml", tmp_path + ) + + result = build_docs_setup(tmp_proj) + assert result.exit_code == 0, "'mkdocs build' command failed" + + contents = (tmp_proj / "site/index.html").read_text() + for reader in ["parquet", "orc", "stata", "sas", "xml"]: + assert re.search(f"{reader}_table", contents), f"read_{reader}() did not insert the table" + assert contents.count("531456") == 5 + + +def test_read_html(tmp_path): + """ + A project that uses read_html(), which returns all tables it finds. + """ + pytest.importorskip("lxml") + + tmp_proj = setup_clean_mkdocs_folder( + "tests/fixtures/pandas_readers_html/mkdocs.yml", tmp_path + ) + + result = build_docs_setup(tmp_proj) + assert result.exit_code == 0, "'mkdocs build' command failed" + + contents = (tmp_proj / "site/index.html").read_text() + # Without 'match', the first table in the file is inserted + assert re.search(r"html_table", contents) + # With 'match', the table that matches is inserted + assert re.search(r"second_html_table", contents) + assert re.search(r"539956", contents) + # read_xml() with the default (lxml) parser + assert re.search(r"xml_table", contents) + + +def test_read_spss(tmp_path): + """ + A project that uses read_spss(), which requires pyreadstat. + """ + pytest.importorskip("pyreadstat") + + tmp_proj = setup_clean_mkdocs_folder( + "tests/fixtures/pandas_readers_spss/mkdocs.yml", tmp_path + ) + + result = build_docs_setup(tmp_proj) + assert result.exit_code == 0, "'mkdocs build' command failed" + + contents = (tmp_proj / "site/index.html").read_text() + assert re.search(r"spss_table", contents) + assert re.search(r"531456", contents) + + +def test_read_hdf(tmp_path): + """ + A project that uses read_hdf(), which requires pytables. + """ + pytest.importorskip("tables") + + tmp_proj = setup_clean_mkdocs_folder( + "tests/fixtures/pandas_readers_hdf/mkdocs.yml", tmp_path + ) + table_path = tmp_proj / "assets/tables" + table_path.mkdir(parents=True) + # a pd.Series, to make sure read_hdf() can also insert those + pd.Series([531456, 80], name="hdf_table").to_hdf(table_path / "table.h5", key="table", mode="w") + + result = build_docs_setup(tmp_proj) + assert result.exit_code == 0, "'mkdocs build' command failed" + + contents = (tmp_proj / "site/index.html").read_text() + assert re.search(r"hdf_table", contents) + assert re.search(r"531456", contents) diff --git a/tests/test_kwargs.py b/tests/test_kwargs.py index 2521d2f..2ddc505 100644 --- a/tests/test_kwargs.py +++ b/tests/test_kwargs.py @@ -24,3 +24,11 @@ def test_parse_argkwarg(): assert parse_argkwarg('"file.csv", header=None') == (['file.csv'], {'header': None}) assert parse_argkwarg("'Example.xlsx', sheet_name = 'test', header = None") == (['Example.xlsx'], {'sheet_name': 'test', 'header': None}) assert parse_argkwarg("'test.csv', header = None, names = ['a', 'b', 'c']") == (['test.csv'], {'header': None, 'names': ['a', 'b', 'c']}) + + +def test_kwargs_multiple_funcs(): + + keywords = {'hi' : 'there', 'sep' : ";", 'max_level' : 1} + + assert kwargs_in_func(keywords, pd.read_csv, pd.json_normalize) == {'sep' : ';', 'max_level' : 1} + assert kwargs_not_in_func(keywords, pd.read_csv, pd.json_normalize) == {'hi' : 'there'} diff --git a/tests/test_readers.py b/tests/test_readers.py new file mode 100644 index 0000000..3a00ea6 --- /dev/null +++ b/tests/test_readers.py @@ -0,0 +1,20 @@ +import pandas as pd + +from mkdocs_table_reader_plugin.readers import LOADERS, MACROS, READERS, read_yaml_file +from mkdocs_table_reader_plugin.utils import kwargs_in_func, kwargs_not_in_func + + +def test_readers(): + # every reader has a macro that returns a pd.DataFrame instead of a markdown table + assert set(READERS) == set(LOADERS) | {"read_raw"} + assert set(MACROS) == set(READERS) | {f"pd_{reader}" for reader in LOADERS} + + +def test_reader_kwargs(): + # read_yaml() accepts the arguments of both read_yaml_file() and pd.json_normalize() + kwargs = {"encoding": "cp1251", "max_level": 1, "tablefmt": "github"} + sources = (read_yaml_file, pd.json_normalize) + + assert kwargs_in_func(kwargs, *sources) == {"encoding": "cp1251", "max_level": 1} + # anything else is passed on to .to_markdown() + assert kwargs_not_in_func(kwargs, *sources) == {"tablefmt": "github"} From 8a1129cb0fdb01e70048dfab5c0cbbab90a02a00 Mon Sep 17 00:00:00 2001 From: Tim Vink Date: Mon, 14 Sep 2026 12:30:09 +0200 Subject: [PATCH 2/4] Reword the README file format list, codespell reads .sav as a typo Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JCJnK1rDt8rmDsXTFc7AHE --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dbcde83..39c383b 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ In your markdown files you can now use: Where the path is relative to the location of your project's `mkdocs.yml` file, _or_ your project's `docs/` directory, _or_ the location of your markdown source file (all 3 possible locations will be searched, in that order). -- There are [readers](https://timvink.github.io/mkdocs-table-reader-plugin/readers/) available for many common table formats, like `.csv`, `.fwf`, `.json`, `.xls`, `.xlsx`, `.yaml`, `.feather`, `.tsv`, `.parquet`, `.orc`, `.html`, `.xml`, `.h5`, `.sav`, `.dta` and SAS files. There is also the `read_raw()` reader that will allow you to insert tables (or other content) already in markdown format. +- There are [readers](https://timvink.github.io/mkdocs-table-reader-plugin/readers/) available for many common table formats, like `.csv`, `.fwf`, `.json`, `.xls`, `.xlsx`, `.yaml`, `.feather`, `.tsv`, `.parquet`, `.orc`, `.html` and `.xml`, as well as HDF5, SPSS, SAS and Stata files. There is also the `read_raw()` reader that will allow you to insert tables (or other content) already in markdown format. - `table-reader` is compatible with [`mkdocs-macros-plugin`](https://mkdocs-macros-plugin.readthedocs.io/en/latest/). This enables further automation like filtering tables or inserting directories of tables. See the documentation on [compatibility with macros plugin](howto/use_jinja2.md) for more examples. ## Documentation and how-to guides From d44a493e9fa42744f4592cb3d70956513ec0c26a Mon Sep 17 00:00:00 2001 From: Tim Vink Date: Mon, 14 Sep 2026 12:38:00 +0200 Subject: [PATCH 3/4] Skip the read_orc test on windows pyarrow cannot find the IANA time zone database there, so pd.read_orc() raises. Moved to a fixture project of its own and documented. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JCJnK1rDt8rmDsXTFc7AHE --- docs/readers.md | 2 +- tests/fixtures/pandas_readers/docs/index.md | 4 --- .../assets/tables/table.orc | Bin .../fixtures/pandas_readers_orc/docs/index.md | 5 ++++ tests/fixtures/pandas_readers_orc/mkdocs.yml | 6 ++++ tests/test_build.py | 26 ++++++++++++++++-- 6 files changed, 36 insertions(+), 7 deletions(-) rename tests/fixtures/{pandas_readers => pandas_readers_orc}/assets/tables/table.orc (100%) create mode 100644 tests/fixtures/pandas_readers_orc/docs/index.md create mode 100644 tests/fixtures/pandas_readers_orc/mkdocs.yml diff --git a/docs/readers.md b/docs/readers.md index 66937b3..8cb4e1f 100644 --- a/docs/readers.md +++ b/docs/readers.md @@ -247,7 +247,7 @@ Example: {{ read_orc('assets/tables/data.orc') | add_indentation(spaces=4) }} -Requires [pyarrow](https://arrow.apache.org/docs/python/install.html) to be installed. +Requires [pyarrow](https://arrow.apache.org/docs/python/install.html) to be installed. On windows, `pandas.read_orc()` also needs the [IANA time zone database](https://arrow.apache.org/docs/python/timestamps.html) to be available to pyarrow. ### `read_xml` diff --git a/tests/fixtures/pandas_readers/docs/index.md b/tests/fixtures/pandas_readers/docs/index.md index 89796aa..96a102c 100644 --- a/tests/fixtures/pandas_readers/docs/index.md +++ b/tests/fixtures/pandas_readers/docs/index.md @@ -4,10 +4,6 @@ {{ read_parquet('assets/tables/table.parquet') }} -## read_orc - -{{ read_orc('assets/tables/table.orc') }} - ## read_stata {{ read_stata('assets/tables/table.dta') }} diff --git a/tests/fixtures/pandas_readers/assets/tables/table.orc b/tests/fixtures/pandas_readers_orc/assets/tables/table.orc similarity index 100% rename from tests/fixtures/pandas_readers/assets/tables/table.orc rename to tests/fixtures/pandas_readers_orc/assets/tables/table.orc diff --git a/tests/fixtures/pandas_readers_orc/docs/index.md b/tests/fixtures/pandas_readers_orc/docs/index.md new file mode 100644 index 0000000..bcaa465 --- /dev/null +++ b/tests/fixtures/pandas_readers_orc/docs/index.md @@ -0,0 +1,5 @@ +# Test page + +## read_orc + +{{ read_orc('assets/tables/table.orc') }} diff --git a/tests/fixtures/pandas_readers_orc/mkdocs.yml b/tests/fixtures/pandas_readers_orc/mkdocs.yml new file mode 100644 index 0000000..cea9ccc --- /dev/null +++ b/tests/fixtures/pandas_readers_orc/mkdocs.yml @@ -0,0 +1,6 @@ +site_name: test git_table_reader site +use_directory_urls: false + +plugins: + - search + - table-reader diff --git a/tests/test_build.py b/tests/test_build.py index c80af0a..50b9749 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -20,6 +20,7 @@ import os import shutil import logging +import sys import pandas as pd import pytest from click.testing import CliRunner @@ -456,9 +457,30 @@ def test_pandas_readers(tmp_path): assert result.exit_code == 0, "'mkdocs build' command failed" contents = (tmp_proj / "site/index.html").read_text() - for reader in ["parquet", "orc", "stata", "sas", "xml"]: + for reader in ["parquet", "stata", "sas", "xml"]: assert re.search(f"{reader}_table", contents), f"read_{reader}() did not insert the table" - assert contents.count("531456") == 5 + assert contents.count("531456") == 4 + + +@pytest.mark.skipif( + sys.platform.startswith("win"), + reason="pd.read_orc() cannot find the IANA time zone database on windows", +) +def test_read_orc(tmp_path): + """ + A project that uses read_orc(). + """ + + tmp_proj = setup_clean_mkdocs_folder( + "tests/fixtures/pandas_readers_orc/mkdocs.yml", tmp_path + ) + + result = build_docs_setup(tmp_proj) + assert result.exit_code == 0, "'mkdocs build' command failed" + + contents = (tmp_proj / "site/index.html").read_text() + assert re.search(r"orc_table", contents) + assert re.search(r"531456", contents) def test_read_html(tmp_path): From aff4f5bb362d4599e34077a3b6294599603b5b60 Mon Sep 17 00:00:00 2001 From: Tim Vink Date: Mon, 14 Sep 2026 12:43:22 +0200 Subject: [PATCH 4/4] Search each page for all readers at once Every reader ran its own regex over every page, so adding readers made builds slower. One pattern with all selected readers scans a page once, which is 8x faster than the previous 8 readers and no longer depends on how many readers are selected. Tables are now inserted with a replacement function, so a value like 'C:\1 path' is inserted as it is. It used to be expanded as a regex replacement, which silently changed such values. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JCJnK1rDt8rmDsXTFc7AHE --- docs/options.md | 2 +- src/mkdocs_table_reader_plugin/plugin.py | 50 +++++++++---------- .../backslashes/assets/tables/backslashes.csv | 3 ++ tests/fixtures/backslashes/docs/index.md | 5 ++ tests/fixtures/backslashes/mkdocs.yml | 6 +++ tests/test_build.py | 18 +++++++ 6 files changed, 56 insertions(+), 28 deletions(-) create mode 100644 tests/fixtures/backslashes/assets/tables/backslashes.csv create mode 100644 tests/fixtures/backslashes/docs/index.md create mode 100644 tests/fixtures/backslashes/mkdocs.yml diff --git a/docs/options.md b/docs/options.md index bd91c39..54cf5fc 100644 --- a/docs/options.md +++ b/docs/options.md @@ -39,7 +39,7 @@ Default: `False`. When enabled, if a filepath is not found, the plugin will rais ## `select_readers` -Default: Selects all available readers. Every reader runs its own regex over all your markdown content, so specifying only the readers you use improves documentation build times for very large sites. This option is ignored when you use this plugin with `mkdocs-macros-plugin` ([read more](howto/use_jinja2.md)) +Default: Selects all available readers. Every page is searched for all selected readers in a single pass, so limiting the list saves little time; use it when you want to be sure only specific readers are used. This option is ignored when you use this plugin with `mkdocs-macros-plugin` ([read more](howto/use_jinja2.md)) Note that some readers need an additional package to be installed, see [readers](readers.md). diff --git a/src/mkdocs_table_reader_plugin/plugin.py b/src/mkdocs_table_reader_plugin/plugin.py index a195d36..c6519ed 100644 --- a/src/mkdocs_table_reader_plugin/plugin.py +++ b/src/mkdocs_table_reader_plugin/plugin.py @@ -51,6 +51,17 @@ def on_config(self, config, **kwargs): if reader in self.config.get("select_readers", []) } + # Regex pattern for tags like {{ read_csv(..) }}, for all selected readers at once, + # so that every page is scanned only once, no matter how many readers are selected. + # match group 1: to extract any leading whitespace + # match group 2: to extract the reader + # match group 3: to extract the arguments (positional and keywords) + # Note that a reader never matches when none are selected + self.tag_pattern = re.compile( + r"( *)\{\{\s+(%s)\((.+)\)\s+\}\}" % "|".join(self.readers or ["(?!)"]), # noqa: UP031 + flags=re.IGNORECASE, + ) + plugins = [p for p in config.get("plugins")] # Plugins required before table-reader @@ -132,34 +143,19 @@ def on_page_markdown(self, markdown, page, config, files, **kwargs): if self.external_jinja_engine: return markdown - for reader in self.readers: - function = self.readers[reader] - # Regex pattern for tags like {{ read_csv(..) }} - # match group 0: to extract any leading whitespace - # match group 1: to extract the arguments (positional and keywords) - tag_pattern = re.compile( - r"( *)\{\{\s+%s\((.+)\)\s+\}\}" % reader, flags=re.IGNORECASE # noqa: UP031 - ) - matches = re.findall(tag_pattern, markdown) - - for result in matches: - # Deal with indentation - # So we can fix inserting tables. - # f.e. relevant when used inside content tabs - leading_spaces = result[0] + def insert_table(tag) -> str: + leading_spaces, reader, arguments = tag.groups() - # Safely parse the arguments - pd_args, pd_kwargs = parse_argkwarg(result[1]) + # Safely parse the arguments + pd_args, pd_kwargs = parse_argkwarg(arguments) - # Load the table - markdown_table = function(*pd_args, **pd_kwargs) + # Load the table + markdown_table = self.readers[reader.lower()](*pd_args, **pd_kwargs) - # Insert markdown table - # By replacing only the first occurrence of the regex pattern - # You might insert multiple CSVs with a single reader like read_csv - # Because of the replacement, the next occurrence will be the first match for .sub() again. - # This is always why when allow_missing_files=True we replaced the input tag. - markdown_table = fix_indentation(leading_spaces=leading_spaces, text=markdown_table) - markdown = tag_pattern.sub(markdown_table, markdown, count=1) + # Deal with indentation, so we can insert tables + # f.e. inside content tabs + return fix_indentation(leading_spaces=leading_spaces, text=markdown_table) - return markdown + # Every tag is replaced in a single pass, so inserted tables are never + # searched for tags themselves + return self.tag_pattern.sub(insert_table, markdown) diff --git a/tests/fixtures/backslashes/assets/tables/backslashes.csv b/tests/fixtures/backslashes/assets/tables/backslashes.csv new file mode 100644 index 0000000..ae5d351 --- /dev/null +++ b/tests/fixtures/backslashes/assets/tables/backslashes.csv @@ -0,0 +1,3 @@ +"a","b" +40,"C:\1 path" +50,"hi\nthere" diff --git a/tests/fixtures/backslashes/docs/index.md b/tests/fixtures/backslashes/docs/index.md new file mode 100644 index 0000000..1339906 --- /dev/null +++ b/tests/fixtures/backslashes/docs/index.md @@ -0,0 +1,5 @@ +# Test page + +A table with values that look like a regex replacement: + +{{ read_csv('assets/tables/backslashes.csv') }} diff --git a/tests/fixtures/backslashes/mkdocs.yml b/tests/fixtures/backslashes/mkdocs.yml new file mode 100644 index 0000000..cea9ccc --- /dev/null +++ b/tests/fixtures/backslashes/mkdocs.yml @@ -0,0 +1,6 @@ +site_name: test git_table_reader site +use_directory_urls: false + +plugins: + - search + - table-reader diff --git a/tests/test_build.py b/tests/test_build.py index 50b9749..4452059 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -544,3 +544,21 @@ def test_read_hdf(tmp_path): contents = (tmp_proj / "site/index.html").read_text() assert re.search(r"hdf_table", contents) assert re.search(r"531456", contents) + + +def test_backslashes_in_tables(tmp_path): + """ + A project with table values that look like a regex replacement. + """ + + tmp_proj = setup_clean_mkdocs_folder( + "tests/fixtures/backslashes/mkdocs.yml", tmp_path + ) + + result = build_docs_setup(tmp_proj) + assert result.exit_code == 0, "'mkdocs build' command failed" + + contents = (tmp_proj / "site/index.html").read_text() + # values are inserted as they are, and not expanded as a regex replacement + assert r"C:\1 path" in contents + assert r"hi\nthere" in contents