diff --git a/README.md b/README.md index 7cba439..39c383b 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ In your markdown files you can now use: Where the path is relative to the location of your project's `mkdocs.yml` file, _or_ your project's `docs/` directory, _or_ the location of your markdown source file (all 3 possible locations will be searched, in that order). -- There are [readers](https://timvink.github.io/mkdocs-table-reader-plugin/readers/) available for many common table formats, like `.csv`, `.fwf`, `.json`, `.xls`, `.xlsx`, `.yaml`, `.feather` and `.tsv`. There is also the `read_raw()` reader that will allow you to insert tables (or other content) already in markdown format. +- There are [readers](https://timvink.github.io/mkdocs-table-reader-plugin/readers/) available for many common table formats, like `.csv`, `.fwf`, `.json`, `.xls`, `.xlsx`, `.yaml`, `.feather`, `.tsv`, `.parquet`, `.orc`, `.html` and `.xml`, as well as HDF5, SPSS, SAS and Stata files. There is also the `read_raw()` reader that will allow you to insert tables (or other content) already in markdown format. - `table-reader` is compatible with [`mkdocs-macros-plugin`](https://mkdocs-macros-plugin.readthedocs.io/en/latest/). This enables further automation like filtering tables or inserting directories of tables. See the documentation on [compatibility with macros plugin](howto/use_jinja2.md) for more examples. ## Documentation and how-to guides diff --git a/docs/assets/tables/data.dta b/docs/assets/tables/data.dta new file mode 100644 index 0000000..63d3046 Binary files /dev/null and b/docs/assets/tables/data.dta differ diff --git a/docs/assets/tables/data.html b/docs/assets/tables/data.html new file mode 100644 index 0000000..bfabf70 --- /dev/null +++ b/docs/assets/tables/data.html @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + +
productprice
bread1.25
milk0.99
cheese4.50
\ No newline at end of file diff --git a/docs/assets/tables/data.orc b/docs/assets/tables/data.orc new file mode 100644 index 0000000..d486a30 Binary files /dev/null and b/docs/assets/tables/data.orc differ diff --git a/docs/assets/tables/data.parquet b/docs/assets/tables/data.parquet new file mode 100644 index 0000000..fccd765 Binary files /dev/null and b/docs/assets/tables/data.parquet differ diff --git a/docs/assets/tables/data.xml b/docs/assets/tables/data.xml new file mode 100644 index 0000000..3e5683f --- /dev/null +++ b/docs/assets/tables/data.xml @@ -0,0 +1,15 @@ + + + + bread + 1.25 + + + milk + 0.99 + + + cheese + 4.5 + + diff --git a/docs/assets/tables/data.xpt b/docs/assets/tables/data.xpt new file mode 100644 index 0000000..f53786e Binary files /dev/null and b/docs/assets/tables/data.xpt differ diff --git a/docs/options.md b/docs/options.md index 0854204..54cf5fc 100644 --- a/docs/options.md +++ b/docs/options.md @@ -39,7 +39,9 @@ Default: `False`. When enabled, if a filepath is not found, the plugin will rais ## `select_readers` -Default: Selects all available readers. Specify a list of readers to improve documentation build times for very large sites. This option is ignored when you use this plugin with `mkdocs-macros-plugin` ([read more](howto/use_jinja2.md)) +Default: Selects all available readers. Every page is searched for all selected readers in a single pass, so limiting the list saves little time; use it when you want to be sure only specific readers are used. This option is ignored when you use this plugin with `mkdocs-macros-plugin` ([read more](howto/use_jinja2.md)) + +Note that some readers need an additional package to be installed, see [readers](readers.md). ## `enabled` diff --git a/docs/readers.md b/docs/readers.md index bdb23e8..8cb4e1f 100644 --- a/docs/readers.md +++ b/docs/readers.md @@ -199,6 +199,192 @@ Example: ``` {% endraw %} +### `read_parquet` + +Use {% raw %}`{{ read_parquet() }}`{% endraw %} to read a parquet file and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_parquet()](https://pandas.pydata.org/docs/reference/api/pandas.read_parquet.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ read_parquet('assets/tables/data.parquet') }} + ``` + {% endraw %} + +=== "Output" + + {{ read_parquet('assets/tables/data.parquet') | add_indentation(spaces=4) }} + +Requires [pyarrow](https://arrow.apache.org/docs/python/install.html) or [fastparquet](https://fastparquet.readthedocs.io/en/latest/install.html) to be installed. + +### `read_orc` + +Use {% raw %}`{{ read_orc() }}`{% endraw %} to read an ORC object and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_orc()](https://pandas.pydata.org/docs/reference/api/pandas.read_orc.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ read_orc('assets/tables/data.orc') }} + ``` + {% endraw %} + +=== "Output" + + {{ read_orc('assets/tables/data.orc') | add_indentation(spaces=4) }} + +Requires [pyarrow](https://arrow.apache.org/docs/python/install.html) to be installed. On windows, `pandas.read_orc()` also needs the [IANA time zone database](https://arrow.apache.org/docs/python/timestamps.html) to be available to pyarrow. + +### `read_xml` + +Use {% raw %}`{{ read_xml() }}`{% endraw %} to read an XML document and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_xml()](https://pandas.pydata.org/docs/reference/api/pandas.read_xml.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ read_xml('assets/tables/data.xml') }} + ``` + {% endraw %} + +=== "Output" + + {{ read_xml('assets/tables/data.xml') | add_indentation(spaces=4) }} + +Requires [lxml](https://lxml.de/installation.html) to be installed, or use the standard library parser with {% raw %}`{{ read_xml('assets/tables/data.xml', parser='etree') }}`{% endraw %}. + +### `read_html` + +Use {% raw %}`{{ read_html() }}`{% endraw %} to read the first table in an HTML document and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_html()](https://pandas.pydata.org/docs/reference/api/pandas.read_html.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ read_html('assets/tables/data.html') }} + ``` + {% endraw %} + +=== "Output" + + {{ read_html('assets/tables/data.html') | add_indentation(spaces=4) }} + +`pandas.read_html()` returns every table it finds, so the first one is inserted. Use the `match` argument to select another table, for example {% raw %}`{{ read_html('assets/tables/data.html', match='price') }}`{% endraw %}. Requires [lxml](https://lxml.de/installation.html), or [beautifulsoup4](https://pypi.org/project/beautifulsoup4/) and [html5lib](https://pypi.org/project/html5lib/), to be installed. + +### `read_stata` + +Use {% raw %}`{{ read_stata() }}`{% endraw %} to read a Stata file and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_stata()](https://pandas.pydata.org/docs/reference/api/pandas.read_stata.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ read_stata('assets/tables/data.dta') }} + ``` + {% endraw %} + +=== "Output" + + {{ read_stata('assets/tables/data.dta') | add_indentation(spaces=4) }} + +### `read_sas` + +Use {% raw %}`{{ read_sas() }}`{% endraw %} to read a SAS file (XPORT or SAS7BDAT) and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_sas()](https://pandas.pydata.org/docs/reference/api/pandas.read_sas.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ read_sas('assets/tables/data.xpt', encoding='utf-8') }} + ``` + {% endraw %} + +=== "Output" + + {{ read_sas('assets/tables/data.xpt', encoding='utf-8') | add_indentation(spaces=4) }} + +Text columns are read as bytes unless you specify the `encoding` to decode them with. + +### `read_spss` + +Use {% raw %}`{{ read_spss() }}`{% endraw %} to read an SPSS file and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_spss()](https://pandas.pydata.org/docs/reference/api/pandas.read_spss.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +{% raw %} +```markdown +{{ read_spss('assets/tables/data.sav') }} +``` +{% endraw %} + +Requires [pyreadstat](https://github.com/Roche/pyreadstat) to be installed. + +### `read_hdf` + +Use {% raw %}`{{ read_hdf() }}`{% endraw %} to read an object stored in a HDF5 file and output as a markdown table. + +1. Arguments are parsed safely and then passed to corresponding functions below +2. File is read using [pandas.read_hdf()](https://pandas.pydata.org/docs/reference/api/pandas.read_hdf.html) +3. The `pd.DataFrame` is then converted to a markdown table using [`.to_markdown()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_markdown.html) +4. The markdown table is fixed to match the indentation used by the tag in the markdown document (only when _not_ used with `mkdocs-macros-plugin`. See [compatibility with macros plugin](howto/use_jinja2.md)) + +Example: + +{% raw %} +```markdown +{{ read_hdf('assets/tables/data.h5', key='table') }} +``` +{% endraw %} + +Requires [pytables](https://www.pytables.org/usersguide/installation.html) to be installed. Specify the `key` of the object to read when the file contains more than one. + ### `read_raw` Use {% raw %}`{{ read_raw() }}`{% endraw %} to insert the contents from a file directly. @@ -378,6 +564,138 @@ Example: {% endraw %} +### `pd_read_parquet` + +Use {% raw %}`{{ pd_read_parquet() }}`{% endraw %} to read a parquet file using [pandas.read_parquet()](https://pandas.pydata.org/docs/reference/api/pandas.read_parquet.html) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ pd_read_parquet('assets/tables/data.parquet').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + ``` + {% endraw %} + +=== "Output" + + {{ pd_read_parquet('assets/tables/data.parquet').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + +### `pd_read_orc` + +Use {% raw %}`{{ pd_read_orc() }}`{% endraw %} to read an ORC object using [pandas.read_orc()](https://pandas.pydata.org/docs/reference/api/pandas.read_orc.html) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ pd_read_orc('assets/tables/data.orc').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + ``` + {% endraw %} + +=== "Output" + + {{ pd_read_orc('assets/tables/data.orc').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + +### `pd_read_xml` + +Use {% raw %}`{{ pd_read_xml() }}`{% endraw %} to read an XML document using [pandas.read_xml()](https://pandas.pydata.org/docs/reference/api/pandas.read_xml.html) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ pd_read_xml('assets/tables/data.xml').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + ``` + {% endraw %} + +=== "Output" + + {{ pd_read_xml('assets/tables/data.xml').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + +### `pd_read_html` + +Use {% raw %}`{{ pd_read_html() }}`{% endraw %} to read the first table in an HTML document using [pandas.read_html()](https://pandas.pydata.org/docs/reference/api/pandas.read_html.html) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ pd_read_html('assets/tables/data.html').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + ``` + {% endraw %} + +=== "Output" + + {{ pd_read_html('assets/tables/data.html').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + +### `pd_read_stata` + +Use {% raw %}`{{ pd_read_stata() }}`{% endraw %} to read a Stata file using [pandas.read_stata()](https://pandas.pydata.org/docs/reference/api/pandas.read_stata.html) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ pd_read_stata('assets/tables/data.dta').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + ``` + {% endraw %} + +=== "Output" + + {{ pd_read_stata('assets/tables/data.dta').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + +### `pd_read_sas` + +Use {% raw %}`{{ pd_read_sas() }}`{% endraw %} to read a SAS file (XPORT or SAS7BDAT) using [pandas.read_sas()](https://pandas.pydata.org/docs/reference/api/pandas.read_sas.html) + +Example: + +=== "Input" + + {% raw %} + ```markdown + {{ pd_read_sas('assets/tables/data.xpt', encoding='utf-8').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + ``` + {% endraw %} + +=== "Output" + + {{ pd_read_sas('assets/tables/data.xpt', encoding='utf-8').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} + +### `pd_read_spss` + +Use {% raw %}`{{ pd_read_spss() }}`{% endraw %} to read an SPSS file using [pandas.read_spss()](https://pandas.pydata.org/docs/reference/api/pandas.read_spss.html) + +Example: + +{% raw %} +```markdown +{{ pd_read_spss('assets/tables/data.sav').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} +``` +{% endraw %} + +### `pd_read_hdf` + +Use {% raw %}`{{ pd_read_hdf() }}`{% endraw %} to read an object stored in a HDF5 file using [pandas.read_hdf()](https://pandas.pydata.org/docs/reference/api/pandas.read_hdf.html) + +Example: + +{% raw %} +```markdown +{{ pd_read_hdf('assets/tables/data.h5', key='table').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} +``` +{% endraw %} + ## Filters When you use `table-reader` with [`mkdocs-macros-plugin`](https://mkdocs-macros-plugin.readthedocs.io/en/latest/), in next to all the readers, the macros, the following _additional_ filters will be made available: diff --git a/docs/schema.json b/docs/schema.json index f7a6a55..22c47df 100644 --- a/docs/schema.json +++ b/docs/schema.json @@ -4,7 +4,9 @@ "oneOf": [ { "markdownDescription": "https://timvink.github.io/mkdocs-table-reader-plugin/", - "enum": ["table-reader"] + "enum": [ + "table-reader" + ] }, { "type": "object", @@ -36,21 +38,26 @@ "title": "{{ read_csv() }} passed to pandas.read_csv()", "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_csv" }, + { + "const": "read_table", + "title": "{{ read_table() }} passed to pandas.read_table()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_table" + }, { "const": "read_fwf", "title": "{{ read_fwf() }} passed to pandas.read_fwf()", "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_fwf" }, + { + "const": "read_excel", + "title": "{{ read_excel() }} passed to pandas.read_excel()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_excel" + }, { "const": "read_yaml", "title": "{{ read_yaml() }} is parsed with yaml.safe_load() and passed to pandas.json_normalize()", "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_yaml" }, - { - "const": "read_table", - "title": "{{ read_table() }} passed to pandas.read_table()", - "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_table" - }, { "const": "read_json", "title": "{{ read_json() }} passed to pandas.read_json()", @@ -62,9 +69,44 @@ "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_feather" }, { - "const": "read_excel", - "title": "{{ read_excel() }} passed to pandas.read_excel()", - "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_excel" + "const": "read_parquet", + "title": "{{ read_parquet() }} passed to pandas.read_parquet()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_parquet" + }, + { + "const": "read_orc", + "title": "{{ read_orc() }} passed to pandas.read_orc()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_orc" + }, + { + "const": "read_html", + "title": "{{ read_html() }} passed to pandas.read_html()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_html" + }, + { + "const": "read_xml", + "title": "{{ read_xml() }} passed to pandas.read_xml()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_xml" + }, + { + "const": "read_hdf", + "title": "{{ read_hdf() }} passed to pandas.read_hdf()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_hdf" + }, + { + "const": "read_sas", + "title": "{{ read_sas() }} passed to pandas.read_sas()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_sas" + }, + { + "const": "read_spss", + "title": "{{ read_spss() }} passed to pandas.read_spss()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_spss" + }, + { + "const": "read_stata", + "title": "{{ read_stata() }} passed to pandas.read_stata()", + "description": "https://timvink.github.io/mkdocs-table-reader-plugin/readers/#read_stata" }, { "const": "read_raw", @@ -75,12 +117,20 @@ }, "default": [ "read_csv", + "read_table", "read_fwf", + "read_excel", "read_yaml", - "read_table", "read_json", "read_feather", - "read_excel", + "read_parquet", + "read_orc", + "read_html", + "read_xml", + "read_hdf", + "read_sas", + "read_spss", + "read_stata", "read_raw" ] } diff --git a/pyproject.toml b/pyproject.toml index e909601..88b80ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ classifiers=[ dynamic = ["version"] dependencies = [ "mkdocs>=1.0", - "pandas>=1.1", + "pandas>=1.3", # pd.read_xml() was added in pandas 1.3 "pyyaml>=5.4.1", "tabulate>=0.8.7", ] @@ -116,6 +116,8 @@ fix = true dev-dependencies = [ "click>=8.1.8", "codecov>=2.1.13", + # lxml is used by pd.read_html() and pd.read_xml(), no cp38 wheels for macos arm64 + "lxml>=5; python_full_version >= '3.9'", "mkdocs-git-authors-plugin>=0.9.4", "mkdocs-git-revision-date-localized-plugin>=1.4.5", "mkdocs-macros-plugin>=1.3.7", diff --git a/src/mkdocs_table_reader_plugin/plugin.py b/src/mkdocs_table_reader_plugin/plugin.py index a195d36..c6519ed 100644 --- a/src/mkdocs_table_reader_plugin/plugin.py +++ b/src/mkdocs_table_reader_plugin/plugin.py @@ -51,6 +51,17 @@ def on_config(self, config, **kwargs): if reader in self.config.get("select_readers", []) } + # Regex pattern for tags like {{ read_csv(..) }}, for all selected readers at once, + # so that every page is scanned only once, no matter how many readers are selected. + # match group 1: to extract any leading whitespace + # match group 2: to extract the reader + # match group 3: to extract the arguments (positional and keywords) + # Note that a reader never matches when none are selected + self.tag_pattern = re.compile( + r"( *)\{\{\s+(%s)\((.+)\)\s+\}\}" % "|".join(self.readers or ["(?!)"]), # noqa: UP031 + flags=re.IGNORECASE, + ) + plugins = [p for p in config.get("plugins")] # Plugins required before table-reader @@ -132,34 +143,19 @@ def on_page_markdown(self, markdown, page, config, files, **kwargs): if self.external_jinja_engine: return markdown - for reader in self.readers: - function = self.readers[reader] - # Regex pattern for tags like {{ read_csv(..) }} - # match group 0: to extract any leading whitespace - # match group 1: to extract the arguments (positional and keywords) - tag_pattern = re.compile( - r"( *)\{\{\s+%s\((.+)\)\s+\}\}" % reader, flags=re.IGNORECASE # noqa: UP031 - ) - matches = re.findall(tag_pattern, markdown) - - for result in matches: - # Deal with indentation - # So we can fix inserting tables. - # f.e. relevant when used inside content tabs - leading_spaces = result[0] + def insert_table(tag) -> str: + leading_spaces, reader, arguments = tag.groups() - # Safely parse the arguments - pd_args, pd_kwargs = parse_argkwarg(result[1]) + # Safely parse the arguments + pd_args, pd_kwargs = parse_argkwarg(arguments) - # Load the table - markdown_table = function(*pd_args, **pd_kwargs) + # Load the table + markdown_table = self.readers[reader.lower()](*pd_args, **pd_kwargs) - # Insert markdown table - # By replacing only the first occurrence of the regex pattern - # You might insert multiple CSVs with a single reader like read_csv - # Because of the replacement, the next occurrence will be the first match for .sub() again. - # This is always why when allow_missing_files=True we replaced the input tag. - markdown_table = fix_indentation(leading_spaces=leading_spaces, text=markdown_table) - markdown = tag_pattern.sub(markdown_table, markdown, count=1) + # Deal with indentation, so we can insert tables + # f.e. inside content tabs + return fix_indentation(leading_spaces=leading_spaces, text=markdown_table) - return markdown + # Every tag is replaced in a single pass, so inserted tables are never + # searched for tags themselves + return self.tag_pattern.sub(insert_table, markdown) diff --git a/src/mkdocs_table_reader_plugin/readers.py b/src/mkdocs_table_reader_plugin/readers.py index 86bbea4..16477cd 100644 --- a/src/mkdocs_table_reader_plugin/readers.py +++ b/src/mkdocs_table_reader_plugin/readers.py @@ -60,109 +60,80 @@ def __call__(self, *args, **kwargs): return self.func(valid_file_paths[0], *args, **kwargs) -@ParseArgs -def pd_read_csv(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_csv) - return pd.read_csv(*args, **read_kwargs) - -@ParseArgs -def read_csv(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_csv) - df = pd.read_csv(*args, **read_kwargs) - - markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_csv) - return convert_to_md_table(df, **markdown_kwargs) - - -@ParseArgs -def pd_read_table(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_table) - return pd.read_table(*args, **read_kwargs) - -@ParseArgs -def read_table(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_table) - df = pd.read_table(*args, **read_kwargs) - - markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_table) - return convert_to_md_table(df, **markdown_kwargs) - - -@ParseArgs -def pd_read_fwf(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_fwf) - return pd.read_fwf(*args, **read_kwargs) - +def read_yaml_file(filepath, encoding: str = "utf-8", **kwargs) -> pd.DataFrame: + """ + Read a YAML file into a pd.DataFrame. -@ParseArgs -def read_fwf(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_fwf) - df = pd.read_fwf(*args, **read_kwargs) + The contents are parsed with yaml.safe_load() and passed to pd.json_normalize(). + """ + with open(filepath, encoding=encoding) as f: + return pd.json_normalize(yaml.safe_load(f), **kwargs) - markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_fwf) - return convert_to_md_table(df, **markdown_kwargs) -@ParseArgs -def pd_read_json(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_json) - return pd.read_json(*args, **read_kwargs) +def read_html_file(*args, **kwargs) -> pd.DataFrame: + """ + Read the first table of an HTML file into a pd.DataFrame. + pd.read_html() returns all tables it finds, use the 'match' argument to select one. + """ + return pd.read_html(*args, **kwargs)[0] -@ParseArgs -def read_json(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_json) - df = pd.read_json(*args, **read_kwargs) - markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_json) - return convert_to_md_table(df, **markdown_kwargs) +def read_hdf_file(*args, **kwargs) -> pd.DataFrame: + """ + Read a HDF5 file into a pd.DataFrame. -@ParseArgs -def pd_read_excel(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_excel) - return pd.read_excel(*args, **read_kwargs) + pd.read_hdf() returns a pd.Series when a Series was stored. + """ + data = pd.read_hdf(*args, **kwargs) + if isinstance(data, pd.Series): + data = data.to_frame() + return data -@ParseArgs -def read_excel(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_excel) - df = pd.read_excel(*args, **read_kwargs) +def markdown_reader(load_function, *extra_kwarg_sources) -> ParseArgs: + """ + Create a reader that inserts a file as a markdown table. - markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_excel) - return convert_to_md_table(df, **markdown_kwargs) + Args: + load_function: function that reads a file path into a pd.DataFrame + extra_kwarg_sources: functions with additional keyword arguments accepted + by load_function, on top of its own. Any other keyword arguments are + passed on to pd.DataFrame.to_markdown() + Returns: + ParseArgs: reader that returns a markdown table + """ + kwarg_sources = (load_function, *extra_kwarg_sources) -@ParseArgs -def pd_read_yaml(*args, **kwargs) -> str: - encoding = kwargs.pop("encoding", "utf-8") - json_kwargs = kwargs_in_func(kwargs, pd.json_normalize) - with open(args[0], encoding=encoding) as f: - df = pd.json_normalize(yaml.safe_load(f), **json_kwargs) - return df + @functools.wraps(load_function) + def reader(*args, **kwargs) -> str: + df = load_function(*args, **kwargs_in_func(kwargs, *kwarg_sources)) + markdown_kwargs = kwargs_not_in_func(kwargs, *kwarg_sources) + return convert_to_md_table(df, **markdown_kwargs) -@ParseArgs -def read_yaml(*args, **kwargs) -> str: - encoding = kwargs.pop("encoding", "utf-8") - json_kwargs = kwargs_in_func(kwargs, pd.json_normalize) - with open(args[0], encoding=encoding) as f: - df = pd.json_normalize(yaml.safe_load(f), **json_kwargs) + return ParseArgs(reader) - markdown_kwargs = kwargs_not_in_func(kwargs, pd.json_normalize) - return convert_to_md_table(df, **markdown_kwargs) +def dataframe_reader(load_function, *extra_kwarg_sources) -> ParseArgs: + """ + Create a macro that returns a file as a pd.DataFrame. -@ParseArgs -def pd_read_feather(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_feather) - return pd.read_feather(*args, **read_kwargs) + Args: + load_function: function that reads a file path into a pd.DataFrame + extra_kwarg_sources: functions with additional keyword arguments accepted + by load_function, on top of its own + Returns: + ParseArgs: reader that returns a pd.DataFrame + """ + kwarg_sources = (load_function, *extra_kwarg_sources) -@ParseArgs -def read_feather(*args, **kwargs) -> str: - read_kwargs = kwargs_in_func(kwargs, pd.read_feather) - df = pd.read_feather(*args, **read_kwargs) + @functools.wraps(load_function) + def reader(*args, **kwargs) -> pd.DataFrame: + return load_function(*args, **kwargs_in_func(kwargs, *kwarg_sources)) - markdown_kwargs = kwargs_not_in_func(kwargs, pd.read_feather) - return convert_to_md_table(df, **markdown_kwargs) + return ParseArgs(reader) @ParseArgs @@ -177,24 +148,29 @@ def read_raw(*args, **kwargs) -> str: return f.read() -READERS = { - "read_csv": read_csv, - "read_table": read_table, - "read_fwf": read_fwf, - "read_excel": read_excel, - "read_yaml": read_yaml, - "read_json": read_json, - "read_feather": read_feather, - "read_raw": read_raw, +# The function used to load each file format into a pd.DataFrame, +# optionally followed by functions with additional keyword arguments it accepts. +LOADERS = { + "read_csv": (pd.read_csv,), + "read_table": (pd.read_table,), + "read_fwf": (pd.read_fwf,), + "read_excel": (pd.read_excel,), + "read_yaml": (read_yaml_file, pd.json_normalize), + "read_json": (pd.read_json,), + "read_feather": (pd.read_feather,), + "read_parquet": (pd.read_parquet,), + "read_orc": (pd.read_orc,), + "read_html": (read_html_file, pd.read_html), + "read_xml": (pd.read_xml,), + "read_hdf": (read_hdf_file, pd.read_hdf), + "read_sas": (pd.read_sas,), + "read_spss": (pd.read_spss,), + "read_stata": (pd.read_stata,), } -MACRO_ONLY = { - "pd_read_csv": pd_read_csv, - "pd_read_table": pd_read_table, - "pd_read_fwf": pd_read_fwf, - "pd_read_excel": pd_read_excel, - "pd_read_yaml": pd_read_yaml, - "pd_read_json": pd_read_json, - "pd_read_feather": pd_read_feather, -} +READERS = {name: markdown_reader(*loader) for name, loader in LOADERS.items()} +READERS["read_raw"] = read_raw + +MACRO_ONLY = {f"pd_{name}": dataframe_reader(*loader) for name, loader in LOADERS.items()} + MACROS = {**READERS, **MACRO_ONLY} diff --git a/src/mkdocs_table_reader_plugin/utils.py b/src/mkdocs_table_reader_plugin/utils.py index fb2f8e6..c524db3 100644 --- a/src/mkdocs_table_reader_plugin/utils.py +++ b/src/mkdocs_table_reader_plugin/utils.py @@ -2,20 +2,24 @@ from inspect import signature -def get_keywords(func): +def get_keywords(*funcs): + """Collect the keyword arguments accepted by one or more functions.""" return [ p.name + for func in funcs for p in signature(func).parameters.values() if p.kind == p.POSITIONAL_OR_KEYWORD or p.kind == p.KEYWORD_ONLY ] -def kwargs_in_func(keywordargs, func): - return {k: v for k, v in keywordargs.items() if k in get_keywords(func)} +def kwargs_in_func(keywordargs, *funcs): + keywords = get_keywords(*funcs) + return {k: v for k, v in keywordargs.items() if k in keywords} -def kwargs_not_in_func(keywordargs, func): - return {k: v for k, v in keywordargs.items() if k not in get_keywords(func)} +def kwargs_not_in_func(keywordargs, *funcs): + keywords = get_keywords(*funcs) + return {k: v for k, v in keywordargs.items() if k not in keywords} class cd: diff --git a/tests/fixtures/backslashes/assets/tables/backslashes.csv b/tests/fixtures/backslashes/assets/tables/backslashes.csv new file mode 100644 index 0000000..ae5d351 --- /dev/null +++ b/tests/fixtures/backslashes/assets/tables/backslashes.csv @@ -0,0 +1,3 @@ +"a","b" +40,"C:\1 path" +50,"hi\nthere" diff --git a/tests/fixtures/backslashes/docs/index.md b/tests/fixtures/backslashes/docs/index.md new file mode 100644 index 0000000..1339906 --- /dev/null +++ b/tests/fixtures/backslashes/docs/index.md @@ -0,0 +1,5 @@ +# Test page + +A table with values that look like a regex replacement: + +{{ read_csv('assets/tables/backslashes.csv') }} diff --git a/tests/fixtures/backslashes/mkdocs.yml b/tests/fixtures/backslashes/mkdocs.yml new file mode 100644 index 0000000..cea9ccc --- /dev/null +++ b/tests/fixtures/backslashes/mkdocs.yml @@ -0,0 +1,6 @@ +site_name: test git_table_reader site +use_directory_urls: false + +plugins: + - search + - table-reader diff --git a/tests/fixtures/pandas_readers/assets/tables/table.dta b/tests/fixtures/pandas_readers/assets/tables/table.dta new file mode 100644 index 0000000..2c227a9 Binary files /dev/null and b/tests/fixtures/pandas_readers/assets/tables/table.dta differ diff --git a/tests/fixtures/pandas_readers/assets/tables/table.parquet b/tests/fixtures/pandas_readers/assets/tables/table.parquet new file mode 100644 index 0000000..c441066 Binary files /dev/null and b/tests/fixtures/pandas_readers/assets/tables/table.parquet differ diff --git a/tests/fixtures/pandas_readers/assets/tables/table.xml b/tests/fixtures/pandas_readers/assets/tables/table.xml new file mode 100644 index 0000000..3bff653 --- /dev/null +++ b/tests/fixtures/pandas_readers/assets/tables/table.xml @@ -0,0 +1,11 @@ + + + + xml_table + 531456 + + + row2 + 80 + + diff --git a/tests/fixtures/pandas_readers/assets/tables/table.xpt b/tests/fixtures/pandas_readers/assets/tables/table.xpt new file mode 100644 index 0000000..9ba971a Binary files /dev/null and b/tests/fixtures/pandas_readers/assets/tables/table.xpt differ diff --git a/tests/fixtures/pandas_readers/docs/index.md b/tests/fixtures/pandas_readers/docs/index.md new file mode 100644 index 0000000..96a102c --- /dev/null +++ b/tests/fixtures/pandas_readers/docs/index.md @@ -0,0 +1,17 @@ +# Test page + +## read_parquet + +{{ read_parquet('assets/tables/table.parquet') }} + +## read_stata + +{{ read_stata('assets/tables/table.dta') }} + +## read_sas + +{{ read_sas('assets/tables/table.xpt', encoding='utf-8') }} + +## read_xml + +{{ read_xml('assets/tables/table.xml', parser='etree') }} diff --git a/tests/fixtures/pandas_readers/mkdocs.yml b/tests/fixtures/pandas_readers/mkdocs.yml new file mode 100644 index 0000000..cea9ccc --- /dev/null +++ b/tests/fixtures/pandas_readers/mkdocs.yml @@ -0,0 +1,6 @@ +site_name: test git_table_reader site +use_directory_urls: false + +plugins: + - search + - table-reader diff --git a/tests/fixtures/pandas_readers_hdf/docs/index.md b/tests/fixtures/pandas_readers_hdf/docs/index.md new file mode 100644 index 0000000..5542e70 --- /dev/null +++ b/tests/fixtures/pandas_readers_hdf/docs/index.md @@ -0,0 +1,5 @@ +# Test page + +## read_hdf + +{{ read_hdf('assets/tables/table.h5', key='table') }} diff --git a/tests/fixtures/pandas_readers_hdf/mkdocs.yml b/tests/fixtures/pandas_readers_hdf/mkdocs.yml new file mode 100644 index 0000000..cea9ccc --- /dev/null +++ b/tests/fixtures/pandas_readers_hdf/mkdocs.yml @@ -0,0 +1,6 @@ +site_name: test git_table_reader site +use_directory_urls: false + +plugins: + - search + - table-reader diff --git a/tests/fixtures/pandas_readers_html/assets/tables/table.html b/tests/fixtures/pandas_readers_html/assets/tables/table.html new file mode 100644 index 0000000..8e5f1bf --- /dev/null +++ b/tests/fixtures/pandas_readers_html/assets/tables/table.html @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + +
namenumber
html_table531456
row280
+ + + + + + + + + + + + + + + + + +
namenumber
second_html_table539956
row212
\ No newline at end of file diff --git a/tests/fixtures/pandas_readers_html/assets/tables/table.xml b/tests/fixtures/pandas_readers_html/assets/tables/table.xml new file mode 100644 index 0000000..3bff653 --- /dev/null +++ b/tests/fixtures/pandas_readers_html/assets/tables/table.xml @@ -0,0 +1,11 @@ + + + + xml_table + 531456 + + + row2 + 80 + + diff --git a/tests/fixtures/pandas_readers_html/docs/index.md b/tests/fixtures/pandas_readers_html/docs/index.md new file mode 100644 index 0000000..0e997c9 --- /dev/null +++ b/tests/fixtures/pandas_readers_html/docs/index.md @@ -0,0 +1,15 @@ +# Test page + +## read_html + +Inserts the first table in the file: + +{{ read_html('assets/tables/table.html') }} + +Use 'match' to select another table: + +{{ read_html('assets/tables/table.html', match='second_html_table') }} + +## read_xml + +{{ read_xml('assets/tables/table.xml') }} diff --git a/tests/fixtures/pandas_readers_html/mkdocs.yml b/tests/fixtures/pandas_readers_html/mkdocs.yml new file mode 100644 index 0000000..cea9ccc --- /dev/null +++ b/tests/fixtures/pandas_readers_html/mkdocs.yml @@ -0,0 +1,6 @@ +site_name: test git_table_reader site +use_directory_urls: false + +plugins: + - search + - table-reader diff --git a/tests/fixtures/pandas_readers_orc/assets/tables/table.orc b/tests/fixtures/pandas_readers_orc/assets/tables/table.orc new file mode 100644 index 0000000..e8e240e Binary files /dev/null and b/tests/fixtures/pandas_readers_orc/assets/tables/table.orc differ diff --git a/tests/fixtures/pandas_readers_orc/docs/index.md b/tests/fixtures/pandas_readers_orc/docs/index.md new file mode 100644 index 0000000..bcaa465 --- /dev/null +++ b/tests/fixtures/pandas_readers_orc/docs/index.md @@ -0,0 +1,5 @@ +# Test page + +## read_orc + +{{ read_orc('assets/tables/table.orc') }} diff --git a/tests/fixtures/pandas_readers_orc/mkdocs.yml b/tests/fixtures/pandas_readers_orc/mkdocs.yml new file mode 100644 index 0000000..cea9ccc --- /dev/null +++ b/tests/fixtures/pandas_readers_orc/mkdocs.yml @@ -0,0 +1,6 @@ +site_name: test git_table_reader site +use_directory_urls: false + +plugins: + - search + - table-reader diff --git a/tests/fixtures/pandas_readers_spss/assets/tables/table.sav b/tests/fixtures/pandas_readers_spss/assets/tables/table.sav new file mode 100644 index 0000000..ce2ed9c Binary files /dev/null and b/tests/fixtures/pandas_readers_spss/assets/tables/table.sav differ diff --git a/tests/fixtures/pandas_readers_spss/docs/index.md b/tests/fixtures/pandas_readers_spss/docs/index.md new file mode 100644 index 0000000..e98ccfa --- /dev/null +++ b/tests/fixtures/pandas_readers_spss/docs/index.md @@ -0,0 +1,5 @@ +# Test page + +## read_spss + +{{ read_spss('assets/tables/table.sav') }} diff --git a/tests/fixtures/pandas_readers_spss/mkdocs.yml b/tests/fixtures/pandas_readers_spss/mkdocs.yml new file mode 100644 index 0000000..cea9ccc --- /dev/null +++ b/tests/fixtures/pandas_readers_spss/mkdocs.yml @@ -0,0 +1,6 @@ +site_name: test git_table_reader site +use_directory_urls: false + +plugins: + - search + - table-reader diff --git a/tests/test_build.py b/tests/test_build.py index 77a5ffb..4452059 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -20,6 +20,9 @@ import os import shutil import logging +import sys +import pandas as pd +import pytest from click.testing import CliRunner from mkdocs.__main__ import build_command @@ -439,3 +442,123 @@ def test_csv_with_multiline_cells(tmp_path): table = re.search(r".*?
", contents, flags=re.DOTALL) assert table is not None, "no table was inserted" assert len(re.findall(r"", table.group())) == 3 + + +def test_pandas_readers(tmp_path): + """ + A project that uses the readers for the other formats supported by pandas. + """ + + tmp_proj = setup_clean_mkdocs_folder( + "tests/fixtures/pandas_readers/mkdocs.yml", tmp_path + ) + + result = build_docs_setup(tmp_proj) + assert result.exit_code == 0, "'mkdocs build' command failed" + + contents = (tmp_proj / "site/index.html").read_text() + for reader in ["parquet", "stata", "sas", "xml"]: + assert re.search(f"{reader}_table", contents), f"read_{reader}() did not insert the table" + assert contents.count("531456") == 4 + + +@pytest.mark.skipif( + sys.platform.startswith("win"), + reason="pd.read_orc() cannot find the IANA time zone database on windows", +) +def test_read_orc(tmp_path): + """ + A project that uses read_orc(). + """ + + tmp_proj = setup_clean_mkdocs_folder( + "tests/fixtures/pandas_readers_orc/mkdocs.yml", tmp_path + ) + + result = build_docs_setup(tmp_proj) + assert result.exit_code == 0, "'mkdocs build' command failed" + + contents = (tmp_proj / "site/index.html").read_text() + assert re.search(r"orc_table", contents) + assert re.search(r"531456", contents) + + +def test_read_html(tmp_path): + """ + A project that uses read_html(), which returns all tables it finds. + """ + pytest.importorskip("lxml") + + tmp_proj = setup_clean_mkdocs_folder( + "tests/fixtures/pandas_readers_html/mkdocs.yml", tmp_path + ) + + result = build_docs_setup(tmp_proj) + assert result.exit_code == 0, "'mkdocs build' command failed" + + contents = (tmp_proj / "site/index.html").read_text() + # Without 'match', the first table in the file is inserted + assert re.search(r"html_table", contents) + # With 'match', the table that matches is inserted + assert re.search(r"second_html_table", contents) + assert re.search(r"539956", contents) + # read_xml() with the default (lxml) parser + assert re.search(r"xml_table", contents) + + +def test_read_spss(tmp_path): + """ + A project that uses read_spss(), which requires pyreadstat. + """ + pytest.importorskip("pyreadstat") + + tmp_proj = setup_clean_mkdocs_folder( + "tests/fixtures/pandas_readers_spss/mkdocs.yml", tmp_path + ) + + result = build_docs_setup(tmp_proj) + assert result.exit_code == 0, "'mkdocs build' command failed" + + contents = (tmp_proj / "site/index.html").read_text() + assert re.search(r"spss_table", contents) + assert re.search(r"531456", contents) + + +def test_read_hdf(tmp_path): + """ + A project that uses read_hdf(), which requires pytables. + """ + pytest.importorskip("tables") + + tmp_proj = setup_clean_mkdocs_folder( + "tests/fixtures/pandas_readers_hdf/mkdocs.yml", tmp_path + ) + table_path = tmp_proj / "assets/tables" + table_path.mkdir(parents=True) + # a pd.Series, to make sure read_hdf() can also insert those + pd.Series([531456, 80], name="hdf_table").to_hdf(table_path / "table.h5", key="table", mode="w") + + result = build_docs_setup(tmp_proj) + assert result.exit_code == 0, "'mkdocs build' command failed" + + contents = (tmp_proj / "site/index.html").read_text() + assert re.search(r"hdf_table", contents) + assert re.search(r"531456", contents) + + +def test_backslashes_in_tables(tmp_path): + """ + A project with table values that look like a regex replacement. + """ + + tmp_proj = setup_clean_mkdocs_folder( + "tests/fixtures/backslashes/mkdocs.yml", tmp_path + ) + + result = build_docs_setup(tmp_proj) + assert result.exit_code == 0, "'mkdocs build' command failed" + + contents = (tmp_proj / "site/index.html").read_text() + # values are inserted as they are, and not expanded as a regex replacement + assert r"C:\1 path" in contents + assert r"hi\nthere" in contents diff --git a/tests/test_kwargs.py b/tests/test_kwargs.py index 2521d2f..2ddc505 100644 --- a/tests/test_kwargs.py +++ b/tests/test_kwargs.py @@ -24,3 +24,11 @@ def test_parse_argkwarg(): assert parse_argkwarg('"file.csv", header=None') == (['file.csv'], {'header': None}) assert parse_argkwarg("'Example.xlsx', sheet_name = 'test', header = None") == (['Example.xlsx'], {'sheet_name': 'test', 'header': None}) assert parse_argkwarg("'test.csv', header = None, names = ['a', 'b', 'c']") == (['test.csv'], {'header': None, 'names': ['a', 'b', 'c']}) + + +def test_kwargs_multiple_funcs(): + + keywords = {'hi' : 'there', 'sep' : ";", 'max_level' : 1} + + assert kwargs_in_func(keywords, pd.read_csv, pd.json_normalize) == {'sep' : ';', 'max_level' : 1} + assert kwargs_not_in_func(keywords, pd.read_csv, pd.json_normalize) == {'hi' : 'there'} diff --git a/tests/test_readers.py b/tests/test_readers.py new file mode 100644 index 0000000..3a00ea6 --- /dev/null +++ b/tests/test_readers.py @@ -0,0 +1,20 @@ +import pandas as pd + +from mkdocs_table_reader_plugin.readers import LOADERS, MACROS, READERS, read_yaml_file +from mkdocs_table_reader_plugin.utils import kwargs_in_func, kwargs_not_in_func + + +def test_readers(): + # every reader has a macro that returns a pd.DataFrame instead of a markdown table + assert set(READERS) == set(LOADERS) | {"read_raw"} + assert set(MACROS) == set(READERS) | {f"pd_{reader}" for reader in LOADERS} + + +def test_reader_kwargs(): + # read_yaml() accepts the arguments of both read_yaml_file() and pd.json_normalize() + kwargs = {"encoding": "cp1251", "max_level": 1, "tablefmt": "github"} + sources = (read_yaml_file, pd.json_normalize) + + assert kwargs_in_func(kwargs, *sources) == {"encoding": "cp1251", "max_level": 1} + # anything else is passed on to .to_markdown() + assert kwargs_not_in_func(kwargs, *sources) == {"tablefmt": "github"}