diff --git a/.github/workflows/tools.yml b/.github/workflows/tools.yml index 79e33d2..d70a01f 100644 --- a/.github/workflows/tools.yml +++ b/.github/workflows/tools.yml @@ -1,8 +1,9 @@ name: Tool image # Builds the tool container and draws with it, which the main CI cannot: its tests run the tools on the -# host. Runs when the image definition or the diagram tools change, once a week (a tool can break without -# a change here, through a Debian package), and by hand. The image takes 5 to 30 minutes to build. +# host. Runs when the image definition, the diagram tools or the pandoc finishing change, once a week (a +# tool can break without a change here, through a Debian package), and by hand. The image takes 5 to 30 +# minutes to build. on: pull_request: paths: @@ -12,6 +13,8 @@ on: - "src/py_doc/deck.py" - "src/py_doc/diagram_api.py" - "src/py_doc/icons.py" + - "src/py_doc/listtable.py" + - "src/py_doc/pandoc/**" - ".github/workflows/tools.yml" schedule: - cron: "17 5 * * 1" # Mondays @@ -42,6 +45,22 @@ jobs: - name: Every diagram type, through the container (the guide has one of each) working-directory: examples run: py-doc build guide -f html -f docx --keep-going + - name: The guide's [[LISTTABLE]] in Word, with the pandoc of the image + working-directory: examples + run: | + python - <<'PY' + from docx import Document + from docx.oxml.ns import qn + document = Document("build/guide-v1.0.docx") + table = next(t for t in document.tables if t.cell(0, 0).text == "Step")._tbl + rows = table.findall(qn("w:tr")) + assert table.tblPr.find(qn("w:tblLayout")).get(qn("w:type")) == "fixed", "a fixed layout" + assert all(r.find(f"{qn('w:trPr')}/{qn('w:cantSplit')}") is None for r in rows[1:]), "body rows may split" + assert table.xpath(".//w:vMerge") and table.xpath(".//w:gridSpan[@w:val='2']"), "merged cells" + assert table.xpath(".//wp:inline"), "the diagram in its cell" + assert table.xpath(".//w:rStyle[@w:val='HighlightYellow']"), "the highlight style" + print(f"{len(rows)} rows, fixed layout, merges, a diagram and the highlight style") + PY - name: The posters and their PDFs, through the container working-directory: examples run: py-doc poster --pdf diff --git a/README.md b/README.md index 900f4f0..349eb6c 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ While writing, `py-doc docx guide.md --watch` (or `slides`, or `poster`) builds py-doc documents itself with py-doc. The sources are in [`docs/`](docs), and the [`py-doc.yml`](py-doc.yml) at the root of this repository builds them: `py-doc build` for Word, HTML and PDF, `py-doc site build` for the web site. -- **User guide**: [introduction](docs/guide/00_introduction.md), [installation](docs/guide/01_install.md), [projects and `py-doc.yml`](docs/guide/02_projects.md), [writing](docs/guide/03_writing.md), [outputs and templates](docs/guide/04_outputs.md), [importing](docs/guide/05_import.md), [C4 architecture](docs/guide/06_c4.md), [linting](docs/guide/07_lint.md), [web sites](docs/guide/08_sites.md), [reference](docs/guide/09_reference.md), [legacy build files and document numbers](docs/guide/10_legacy.md), [generators, the build pipeline and analysis](docs/guide/11_pipeline.md), [posters](docs/guide/12_posters.md), [writing a poster](docs/guide/13_writing_posters.md), [writing a document](docs/guide/14_writing_documents.md), [writing a slide deck](docs/guide/15_writing_slides.md). +- **User guide**: [introduction](docs/guide/00_introduction.md), [installation](docs/guide/01_install.md), [projects and `py-doc.yml`](docs/guide/02_projects.md), [writing](docs/guide/03_writing.md) (with [rich table cells](docs/guide/03_writing.md#rich-table-cells) of paragraphs, lists, merged cells and diagrams), [outputs and templates](docs/guide/04_outputs.md), [importing](docs/guide/05_import.md), [C4 architecture](docs/guide/06_c4.md), [linting](docs/guide/07_lint.md), [web sites](docs/guide/08_sites.md), [reference](docs/guide/09_reference.md), [legacy build files and document numbers](docs/guide/10_legacy.md), [generators, the build pipeline and analysis](docs/guide/11_pipeline.md), [posters](docs/guide/12_posters.md), [writing a poster](docs/guide/13_writing_posters.md), [writing a document](docs/guide/14_writing_documents.md), [writing a slide deck](docs/guide/15_writing_slides.md). - **[Requirements](docs/requirements/00_introduction.md)**: what py-doc must do and how well, with the status of every requirement and the open issues. - **[Architecture](docs/architecture/00_introduction.md)**: how py-doc is built, with C4 chapters generated from [its Structurizr model](docs/architecture/workspace.dsl), and the decisions behind it. - **[Examples](examples/README.md)**: a small project that uses every feature, with a command for each output. diff --git a/docs/architecture/02_pipeline.md b/docs/architecture/02_pipeline.md index 3b85459..b1d473a 100644 --- a/docs/architecture/02_pipeline.md +++ b/docs/architecture/02_pipeline.md @@ -26,7 +26,7 @@ A source with `structurizr:` is replaced by Markdown files before anything else ## 3. Joining -`pandoc/source.py` reads each included source and expands its directives (`directives.py`: includes, conditions, variables). It splits off the front matter, applies the title and heading offset, and puts a page break block before sources that ask for one. The metadata of the document is merged from the configuration, the first source's front matter and its first heading (`build.py`). Headings are found line by line outside code fences, so a `#` in a code block is never mistaken for one. +`pandoc/source.py` reads each included source and expands its directives (`directives.py`: includes, conditions, variables). The same pass turns each `[[LISTTABLE]]` into a pandoc grid table in a Div, and each `[[BLOCK:]]` into a Div of its own (`listtable.py`). Done here, their warnings name the file and line, even inside an included file. It splits off the front matter, applies the title and heading offset, and puts a page break block before sources that ask for one. The metadata of the document is merged from the configuration, the first source's front matter and its first heading (`build.py`). Headings are found line by line outside code fences, so a `#` in a code block is never mistaken for one. ## 4. Diagrams @@ -38,13 +38,14 @@ A source with `structurizr:` is replaced by Markdown files before anything else | Filter | Job | Formats | |---|---|---| +| `listtable.lua` | The blocks of `[[LISTTABLE]]` cells move into their cells, without captions. In Word a hidden bookmark marks the table for the finishing | all, and site pages | | `admonitions.lua` | Callouts in both spellings become a box of their kind | all | | `pagebreak.lua` | Page break blocks become a page break, or a new slide | all | | `markers.lua` | `[[CAPTION:]]`, `[[STYLE:]]` and orientation markers. On slides the caption also becomes the alt text pandoc prints under a picture | PowerPoint, HTML, PDF | | `center-div.lua` | Centring divs become LaTeX centring | PDF | | `site.lua` | Admonitions or asides, figures as picture and caption, no repeated title | site pages | -Word handles the markers itself after pandoc (`pandoc/docx_post.py`), because they become Word fields, sections and styles. `pandoc/docx_finish.py` merges the result into the template, fills the placeholders and applies the layout. `pandoc/pptx_finish.py` keeps embedded fonts, fills the version token and the footer, and warns about crowded slides. The finished file is moved to its output path, and the build report records it with its checksum. +Word handles the markers itself after pandoc (`pandoc/docx_post.py`), because they become Word fields, sections and styles. `pandoc/docx_finish.py` merges the result into the template, fills the placeholders and applies the layout. It merges with docxcompose one list at a time, because docxcompose gives every list in one table the same numbering. Tables get the template's table style, centred at full width. An ordinary table with a header row is autofit, its header repeated and its rows kept whole. A `[[LISTTABLE]]` (found by its bookmark) or a table without a header keeps pandoc's column widths in a fixed layout. Its body rows may break across pages, and its pictures are made no wider than their cells. `pandoc/pptx_finish.py` keeps embedded fonts, fills the version token and the footer, and warns about crowded slides. The finished file is moved to its output path, and the build report records it with its checksum. ## Around the documents diff --git a/docs/architecture/04_sites_and_imports.md b/docs/architecture/04_sites_and_imports.md index 95de6e3..958f251 100644 --- a/docs/architecture/04_sites_and_imports.md +++ b/docs/architecture/04_sites_and_imports.md @@ -17,7 +17,7 @@ The site package keeps what all site generators share in `site/pages.py`, and wh ## Imports -`docx_import.py` walks the Word body in order: paragraphs, tables and content controls. It keeps tracked insertions, drops deletions, and renders blocks to Markdown. `pptx_import.py` walks each slide's shapes in reading order, including those pandoc wraps in `mc:AlternateContent`. It writes the conventions the slides converter reads, so a deck makes a round trip to the same slides. +`docx_import.py` walks the Word body in order: paragraphs, tables and content controls. It keeps tracked insertions, drops deletions, and renders blocks to Markdown. A table that a pipe table cannot hold (paragraphs or lists in a cell, merged cells, no header row) becomes a `[[LISTTABLE]]`. `pptx_import.py` walks each slide's shapes in reading order, including those pandoc wraps in `mc:AlternateContent`. It writes the conventions the slides converter reads, so a deck makes a round trip to the same slides. Both importers share picture conversion (Pillow for bitmaps, Inkscape in the container for EMF, WMF and SVG), escaping and the reading of document properties. Both count what they cannot carry over. diff --git a/docs/guide/02_projects.md b/docs/guide/02_projects.md index 0c0fda7..43ec69a 100644 --- a/docs/guide/02_projects.md +++ b/docs/guide/02_projects.md @@ -193,7 +193,7 @@ The old commands `py-doc2docx`, `py-doc2slides` and `py-doclint` still work and `py-doc build --report build.json`, or `reports: {path: build.json}` in `py-doc.yml`, writes a JSON report of the build: - the py-doc version, the image tag and id, and the version of each tool used; -- each document with its sources (words, sentences, paragraphs) and its outputs (size and SHA-256); +- each document with its sources (words, sentences, paragraphs and `[[LISTTABLE]]` tables) and its outputs (size and SHA-256); - the icons each poster uses, with their pack and licence; - every diagram: type, file, line, and whether the cache served it; - template problems, and every warning with file and line. diff --git a/docs/guide/03_writing.md b/docs/guide/03_writing.md index 509a8b9..9b48d32 100644 --- a/docs/guide/03_writing.md +++ b/docs/guide/03_writing.md @@ -17,6 +17,7 @@ Pandoc reads the Markdown: its own flavour, with fenced divs, raw HTML, table ca | `[[CAPTION:Figure]] text` | A numbered caption for the picture, diagram or table that follows | | `[[LANDSCAPE]]`, `[[PORTRAIT]]` | A section of that orientation (Word), and picture widths in the other formats | | `[[STYLE:Name]] text` | The paragraph in that Word style | +| `[[LISTTABLE]]` and `\| ... \|` rows | A table whose cells hold paragraphs, lists, merged cells and diagrams: see [Rich table cells](#rich-table-cells) | | `[[PAGEBREAK]]`, or `::: {.pagebreak}` and `:::` | A page break, or a new slide | | Fenced diagram blocks | A picture: see [Diagrams](#diagrams) | | Raw HTML | Kept in HTML and sites, dropped elsewhere by pandoc | @@ -188,6 +189,42 @@ Word uses the `Div.note` to `Div.caution` paragraph styles. The built-in referen - **Styles**: `[[STYLE:Name]] text` applies a named Word paragraph style, and warns once if the template has none of that name. Other formats keep the text. - **Page breaks**: `[[PAGEBREAK]]` on its own line, `::: {.pagebreak}` followed by `:::`, or `page_break: true` on a source. In a deck it starts a new slide, and on a site it is left out. +## Rich table cells + +A pipe table holds one line of text in each cell. For cells of several paragraphs, lists, merged cells or a diagram, put `[[LISTTABLE]]` on the line before the rows, and leave out the `|---|` line: + +````markdown +[[LISTTABLE: header, widths=8,52,40]] +| Step | What to do | Check | +| 1 | [Prepare the node:]{.underline}
Fit the batteries.
Before you go to site:
- The firmware is version {{version}}
- The node is registered
- with its own certificate | The light flashes green. | +| 2 | Pair the node:
1. Hold the button for five seconds.
2. Choose the node in the app. | ^ | +| 3 | The first reading goes this way:
[[BLOCK: reading]] | < | + +[[BLOCK: reading]] +```dot +digraph reading { rankdir=LR; Sensor -> Gateway -> Broker; } +``` +[[ENDBLOCK]] +```` + +- **Rows**: each row is one line that starts and ends with `|`, straight after the marker. `\|` is a pipe in the text. +- **Header**: `header` makes the first row the header row. Without it every row is body text. +- **Widths**: `widths=8,52,40` gives the relative widths of the columns. Without it the columns are the same width. +- **Paragraphs and lists**: `
` starts a new paragraph in a cell. A paragraph that starts with `- ` is a bullet and one that starts with `1. ` a numbered item, and items next to each other make one list. Two spaces before the `- ` or `1. ` put the item inside the one above. +- **Merged cells**: a cell of only `<` joins the cell on its left, and a cell of only `^` the cell above. Merged cells must make a rectangle, and a body cell cannot join the header row. `\<` and `\^` are the characters themselves. +- **Blocks**: a cell paragraph `[[BLOCK: name]]` is replaced by the lines between `[[BLOCK: name]]` and `[[ENDBLOCK]]` elsewhere in the same file. A block can hold a diagram, a picture, code or a callout. A picture or diagram in a cell has no caption of its own: caption the table. Plain pictures also work inline, as `![alt](picture.png)`. +- **Formatting**: bold, italic, links, `[text]{.underline}` and `[text]{custom-style="Name"}` work as anywhere else. A `custom-style` is a character style of the Word template. Pandoc makes one with no formatting if the template has none of that name, so add the styles you use to the template. + +py-doc turns the table into a pandoc grid table after includes, conditions and variables, so rich tables work in included files and take `{{variables}}`. A marker in a code block is left alone. In Word the columns keep the widths given, the rows may break across pages, and a picture is made no wider than its cell. Only a header row is repeated on each page and printed as a header. HTML, PDF and sites show the same table. On a slide pandoc shows merged cells as separate cells, bullets as plain lines and no pictures. + +The build warns about a row with too many or too few cells, which is padded or cut. It warns about a merge that cannot be made, which is left empty, and a marker with no rows, which is left as written. It also warns about a block that is used but not defined, or defined but not used. A `[[BLOCK:]]` that is never closed with `[[ENDBLOCK]]` stops the build. `py-doc lint` and `py-doc check` report all of these with their line. + +Known limits: + +- A word wider than its column breaks the table, and the build says so. Widen the column or shorten the word. +- Includes and conditions cannot go inside a cell, only around the whole table or inside a block. +- A block belongs to its file: a table cannot use a block defined in another file. + ## Includes, conditions and variables These lines are applied to each source before it is parsed. They are only recognised outside code, so a document can show the syntax in a fenced block or in backticks. diff --git a/docs/guide/05_import.md b/docs/guide/05_import.md index 57dcea8..3b959e7 100644 --- a/docs/guide/05_import.md +++ b/docs/guide/05_import.md @@ -25,7 +25,8 @@ A file that is not a Word document or deck exits with status 2. The file's prope ## Word documents - Headings are moved up so that the first one is `#`. Title and Subtitle paragraphs become front matter. -- Lists keep their nesting, tables become pipe tables (merged cells are flattened to empty cells), consecutive `Code` paragraphs become one fenced block, and `Quote` becomes a block quote. +- Tables become pipe tables, or [rich tables](03_writing.md#rich-table-cells) (`[[LISTTABLE]]`) when a cell has several paragraphs or a list, cells are merged, or the table has no header row. +- Lists keep their nesting, consecutive `Code` paragraphs become one fenced block, and `Quote` becomes a block quote. - `Note`, `Tip`, `Important`, `Warning` and `Caution` styles become `> [!NOTE]` callouts, and captions become `[[CAPTION:Figure]]` lines before their picture or table. - Tracked changes are accepted: insertions are kept and deletions dropped. The contents table is left out. - Pictures are saved as PNG in the order the document uses them. EMF, WMF and SVG go through Inkscape in the tool container. A picture that cannot be converted is named in a warning and shows as `(image not imported)`. diff --git a/docs/guide/07_lint.md b/docs/guide/07_lint.md index 50ab90f..063a8c6 100644 --- a/docs/guide/07_lint.md +++ b/docs/guide/07_lint.md @@ -73,7 +73,7 @@ lint: - **Structure rules** are project conventions as data. The rule types are `manifest-file-exists`, `regex-line`, `blank-line-between`, `path-exists`, `balanced-markers`, `requires-caption` and `cross-file-anchors`. Code blocks and inline code are never searched. Each rule has an `id`, a `severity` and a `message`, and `enabled: false` switches it off. A mistake in a rule is a configuration error that names the rule. `py-doc lint --print-example-rules` prints an example set. - **Cross-file anchors** work on the documents of `py-doc.yml`. All the files of a document share one set of heading anchors, as they do in pandoc. A repeated heading gets `-1`, `-2` and so on, and a `[x](#anchor)` link is checked against that set. Files pulled in with `[[INCLUDE:]]` count. A broken link and a repeated heading are reported with file and line. -- **markdownlint** runs in the tool container (`markdownlint-cli2`), only when `lint.markdownlint` is set, and its findings appear with rules `MD001` and so on. +- **markdownlint** runs in the tool container (`markdownlint-cli2`), only when `lint.markdownlint` is set, and its findings appear with rules `MD001` and so on. Its rule against raw HTML (`MD033`) reports the `
` of [rich tables](03_writing.md#rich-table-cells): allow it with `MD033: {allowed_elements: [br]}`. - **Business Process Model and Notation (BPMN) fences**: bpmn-text checks every ```` ```bpmnt ```` fence in the tool container. Its findings appear at their line of the Markdown file, with bpmn-text's own codes, such as `BT202` for a name that matches no element. `lint.rules` takes those codes like any other rule, such as `BT301: off`. The check runs only when the tool image is already built: without it, one warning says the fences were not checked, and `py-doc tools build` builds it. A broken fence still stops the build, with the same message. Without markdownlint or a `bpmnt` fence, lint needs no container. diff --git a/docs/guide/14_writing_documents.md b/docs/guide/14_writing_documents.md index c85b59a..e3937ad 100644 --- a/docs/guide/14_writing_documents.md +++ b/docs/guide/14_writing_documents.md @@ -96,6 +96,17 @@ Use a table to compare things or to look them up, with a caption on the line bef Keep tables narrow enough for the page: four or five columns in portrait. A wider table belongs in a landscape section (below) or split in two. +When a cell needs more than one line, such as the steps of a procedure with the details of each, use a [rich table](03_writing.md#rich-table-cells). `
` starts a paragraph in a cell, `- ` a bullet and `1. ` a numbered item. A cell of `^` joins the cell above, and `<` the cell on its left: + +```markdown +[[LISTTABLE: header, widths=8,62,30]] +| Step | Action | Who | +| 1 | Fit the controller.
Check before you start:
- The supply is isolated
- The enclosure is dry | Installer | +| 2 | Connect the sensor cable to terminal S1. | ^ | +``` + +A diagram or picture can go in a cell too, as `[[BLOCK: name]]`, with the diagram between `[[BLOCK: name]]` and `[[ENDBLOCK]]` lines below the table. + ## Pictures and diagrams Put a picture with its caption on the line before it: diff --git a/docs/requirements/01_functional.md b/docs/requirements/01_functional.md index 7b426cc..8cd46ef 100644 --- a/docs/requirements/01_functional.md +++ b/docs/requirements/01_functional.md @@ -35,6 +35,7 @@ title: Functional requirements | FR-026 | Fenced blocks of Mermaid, PlantUML, Graphviz, ditaa, the blockdiag family, WaveDrom, bytefield-svg, Pikchr, railroad, WireViz, Vega and Vega-Lite shall become pictures in every output. | Must | Met | | FR-027 | A diagram that cannot be drawn shall stop the build with the file, the line and the tool's message. `--keep-going` shall list every failure. | Must | Met | | FR-028 | `diagrams.mermaid` in `py-doc.yml` shall give every Mermaid diagram of documents, sites and posters the same Mermaid settings, under the diagram's own. | Could | Met | +| FR-029 | `[[LISTTABLE]]` followed by `\| ... \|` rows shall make a table with an optional header row and relative column widths. Its cells shall hold several paragraphs (`
`), bullet and numbered lists (`- `, `1. `) nested by indentation, cells merged across (`<`) and down (`^`), and blocks such as diagrams (`[[BLOCK:]]`). In Word its body rows may break across pages, only a header row shall be formatted and repeated as one, and a picture shall fit its cell. These mistakes shall be warnings that name the file and line: a wrong number of cells, an impossible merge, a marker with no rows, an undefined block. A block never closed shall be an error. | Should | Met | ## Outputs @@ -59,6 +60,7 @@ title: Functional requirements | FR-061 | `py-doc import` shall turn a PowerPoint deck into Markdown that `py-doc slides` builds into the same slides. It keeps the title slide, headings, nested bullets, columns, tables, pictures with captions, code, callouts and speaker notes. | Should | Met | | FR-062 | What an import cannot carry over shall be counted in a warning, never dropped quietly. | Must | Met | | FR-063 | An import shall not replace an existing Markdown file unless told to. | Must | Met | +| FR-064 | A Word table with several paragraphs or a list in a cell, merged cells or no header row shall import as a `[[LISTTABLE]]`, with its column widths. | Should | Met | ## C4 architecture diff --git a/docs/requirements/04_traceability.md b/docs/requirements/04_traceability.md index 4891db8..0216a16 100644 --- a/docs/requirements/04_traceability.md +++ b/docs/requirements/04_traceability.md @@ -14,7 +14,7 @@ title: Traceability | FR-020 to FR-025 | `directives.py`, `pandoc/resources/*.lua`, `pandoc/docx_post.py` | `test_directives.py`, `test_pandoc.py`, `test_word_control.py`, `test_word_layout.py` | | FR-026, FR-027, QR-080 | `diagrams/` | `test_diagrams.py` | | FR-040 to FR-049 | `pandoc/engine.py`, `pandoc/reference.py`, `pandoc/docx_finish.py`, `pandoc/pptx_finish.py`, `naming.py`, `report.py`, `template_check.py`, `convert.py`, `_cli.py` | `test_pandoc.py`, `test_pptx_pandoc.py`, `test_report.py`, `test_template_check.py`, `test_cli.py`, `test_slides_cli.py`, `test_convert.py`, `test_examples.py`, `test_heading_numbers.py` | -| FR-060, FR-062, FR-063 | `docx_import.py`, `import_cli.py` | `test_docx_import.py` | +| FR-060, FR-062 to FR-064 | `docx_import.py`, `import_cli.py` | `test_docx_import.py` | | FR-061 | `pptx_import.py` | `test_pptx_import.py` | | FR-080 to FR-086, QR-081 | `c4/` | `test_c4.py` | | FR-100 to FR-106 | `lint/`, `markdown/` | `test_lint.py`, `test_lint_cli.py`, `test_structure.py`, `test_vale.py`, `test_markdownlint.py`, `test_markdown_ast.py` | @@ -23,6 +23,7 @@ title: Traceability | FR-161 | `_cli.py`, `convert.py`, `watch.py`, `deck.py`, `pandoc/engine.py`, `pandoc/resources/preview.css` | `test_cli.py`, `test_slides_cli.py`, `test_convert.py`, `test_preview_cases.py` | | FR-162 | `editors/vscode/extension.js`, `editors/vscode/src/` | `editors/vscode/test/`, `test_preview_cases.py` | | FR-028 | `diagrams/render.py`, `config.py` | `test_diagrams.py` | +| FR-029 | `listtable.py`, `directives.py`, `pandoc/resources/listtable.lua`, `pandoc/docx_post.py`, `pandoc/docx_finish.py`, `lint/text.py`, `lint/structure.py`, `report.py` | `test_listtable.py`, `test_structure.py`, `test_examples.py` | | FR-167, FR-168 | `diagrams/blocks.py`, `diagrams/render.py`, `diagrams/scan.py`, `diagrams/registry.py`, `diagrams/tools/block_png.js`, `poster_blocks.py`, `config.py`, `runtime/definition/package.json` | `test_document_blocks.py`, `test_poster_diagrams.py`, `test_diagrams.py` | | FR-169 | `diagram_cli.py`, `diagram_api.py`, `diagrams/tools/block_png.js`, `app.py`, `__init__.py` | `test_diagram_cli.py` | | FR-171 | `icons.py`, `icons_cli.py`, `poster_blocks.py`, `poster.py`, `poster_settings.py`, `report.py`, `report.schema.json` | `test_icons.py` | diff --git a/editors/vscode/CHANGELOG.md b/editors/vscode/CHANGELOG.md index 1d59ad8..cd1da82 100644 --- a/editors/vscode/CHANGELOG.md +++ b/editors/vscode/CHANGELOG.md @@ -1,5 +1,9 @@ # Changes +## 2.2.0 + +- Highlighting of py-doc directives in Markdown, such as `[[INCLUDE: a.md]]`, `[[LISTTABLE: header]]` and `[[BLOCK: name]]`, and of `{{variables}}`. Code is left alone. + ## 2.1.0 - Highlighting of the [bpmn-text](https://github.com/markbac/bpmn-text) notation, in `.bpmnt` files and in ```` ```bpmnt ```` fences of Markdown files, with `#` comments toggled by `Ctrl+/`. diff --git a/editors/vscode/README.md b/editors/vscode/README.md index 1b0334a..997881c 100644 --- a/editors/vscode/README.md +++ b/editors/vscode/README.md @@ -34,6 +34,12 @@ VS Code's own Markdown preview (`Ctrl+Shift+V`) still works, but shows plain Mar --- +## Directive highlighting + +py-doc's directives at the start of a line in Markdown, such as `[[INCLUDE: a.md]]`, `[[CAPTION:Table]]`, `[[LISTTABLE: header]]` and `[[BLOCK: name]]`, are highlighted as keywords with their arguments, and `{{variables}}` as variables. Code blocks and inline code are left alone. + +--- + ## Install The extension is not on the Marketplace yet. Build it and install the file: @@ -70,7 +76,7 @@ editors/vscode/ │ ├── frontmatter.js # front matter (js-yaml) │ ├── text.js # front matter values as text │ └── watch.js # the live preview process -├── syntaxes/ # bpmn-text highlighting: .bpmnt files and ```bpmnt fences +├── syntaxes/ # highlighting: .bpmnt files, ```bpmnt fences, and py-doc directives in Markdown ├── images/icon.png ├── examples/ # a document, a deck and a poster to try it on └── test/ diff --git a/editors/vscode/package-lock.json b/editors/vscode/package-lock.json index 28e7921..ceee9ed 100644 --- a/editors/vscode/package-lock.json +++ b/editors/vscode/package-lock.json @@ -1,12 +1,12 @@ { "name": "vscode-py-doc-preview", - "version": "2.1.0", + "version": "2.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "vscode-py-doc-preview", - "version": "2.1.0", + "version": "2.2.0", "license": "MIT", "dependencies": { "js-yaml": "^4.1.0" diff --git a/editors/vscode/package.json b/editors/vscode/package.json index fc5ae08..21a7ed0 100644 --- a/editors/vscode/package.json +++ b/editors/vscode/package.json @@ -1,8 +1,8 @@ { "name": "vscode-py-doc-preview", "displayName": "py-doc Live Preview", - "description": "Live preview of py-doc documents, slide decks and posters in VS Code: py-doc's own output, rebuilt on every save. Highlights bpmn-text (.bpmnt) files and ```bpmnt fences.", - "version": "2.1.0", + "description": "Live preview of py-doc documents, slide decks and posters in VS Code: py-doc's own output, rebuilt on every save. Highlights py-doc directives and variables in Markdown, bpmn-text (.bpmnt) files and ```bpmnt fences.", + "version": "2.2.0", "publisher": "py-doc", "license": "MIT", "icon": "images/icon.png", @@ -65,6 +65,13 @@ "embeddedLanguages": { "meta.embedded.block.bpmnt": "bpmnt" } + }, + { + "scopeName": "markdown.pydoc.directives", + "path": "./syntaxes/pydoc.markdown.tmLanguage.json", + "injectTo": [ + "text.html.markdown" + ] } ], "commands": [ diff --git a/editors/vscode/syntaxes/pydoc.markdown.tmLanguage.json b/editors/vscode/syntaxes/pydoc.markdown.tmLanguage.json new file mode 100644 index 0000000..78454d3 --- /dev/null +++ b/editors/vscode/syntaxes/pydoc.markdown.tmLanguage.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json", + "comment": "py-doc directives such as [[INCLUDE: a.md]] and [[LISTTABLE: header]], and {{variables}}, in Markdown outside code", + "scopeName": "markdown.pydoc.directives", + "injectionSelector": "L:text.html.markdown -markup.fenced_code -markup.inline.raw", + "patterns": [ + { + "include": "#directive" + }, + { + "include": "#variable" + } + ], + "repository": { + "directive": { + "match": "^(\\[\\[)([A-Z][A-Z-]*)(?:(:)\\s*([^\\]]*))?(\\]\\])", + "captures": { + "1": { + "name": "punctuation.definition.tag.begin.pydoc" + }, + "2": { + "name": "keyword.control.directive.pydoc" + }, + "3": { + "name": "punctuation.separator.key-value.pydoc" + }, + "4": { + "name": "string.unquoted.argument.pydoc" + }, + "5": { + "name": "punctuation.definition.tag.end.pydoc" + } + } + }, + "variable": { + "match": "\\{\\{\\s*[A-Za-z_][A-Za-z0-9_]*\\s*\\}\\}", + "name": "variable.other.pydoc" + } + } +} diff --git a/editors/vscode/test/grammar.test.js b/editors/vscode/test/grammar.test.js index 85c52cb..f9cb948 100644 --- a/editors/vscode/test/grammar.test.js +++ b/editors/vscode/test/grammar.test.js @@ -1,6 +1,6 @@ 'use strict'; -// bpmn-text highlighting: .bpmnt files, and ```bpmnt fences in Markdown. +// Highlighting: .bpmnt files, ```bpmnt fences in Markdown, and py-doc's directives and variables in Markdown. const { test } = require('node:test'); const assert = require('node:assert'); @@ -37,3 +37,16 @@ test('the keywords are the statement keywords of the bpmn-text notation', () => } assert.doesNotMatch(' Ship parcel', new RegExp(statement)); }); + +test('py-doc directives and variables are highlighted in Markdown', () => { + const grammar = read('syntaxes/pydoc.markdown.tmLanguage.json'); + const directive = new RegExp(grammar.repository.directive.match); + for (const line of ['[[LISTTABLE: header, widths=5,95]]', '[[BLOCK: wiring]]', '[[ENDBLOCK]]', '[[INCLUDE: a.md]]', '[[CAPTION:Table]] Sizes']) { + assert.match(line, directive, line); + } + assert.strictEqual('[[LISTTABLE: header]]'.match(directive)[2], 'LISTTABLE'); + assert.doesNotMatch('[[not a directive]]', directive); + assert.doesNotMatch('text [[INCLUDE: a.md]]', directive); + assert.match('Version {{ version }}', new RegExp(grammar.repository.variable.match)); + assert.match(grammar.injectionSelector, /-markup\.fenced_code/); +}); diff --git a/examples/README.md b/examples/README.md index bebe74c..5203f3d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -37,7 +37,7 @@ What needs Docker: Word, PowerPoint, PDF and HTML output, every diagram type, dr |---|---| | `py-doc.yml` | Five documents, output defaults, per-document variables and conditions, file naming, the build report, C4, lint and site settings. Every setting has a comment | | `site/` | The MkDocs site project: `mkdocs.yml` (yours to change) and `py-doc-nav.yml` (written by `py-doc site`) | -| `docs/guide/` | The guide, in five files: front matter and document properties, variables, conditions, nested includes, every callout style, all heading levels, block quotes, captions, lists, task lists, footnotes, definition lists, maths, code includes, a landscape section, page breaks, an appendix with a shifted heading level, links between chapters. `DRAFT.md` is listed with `include: false` | +| `docs/guide/` | The guide, in five files: front matter and document properties, variables, conditions, nested includes, every callout style, all heading levels, block quotes, captions, lists, a rich table with a header row, nested and numbered lists, merged cells and a diagram in a cell (`[[LISTTABLE]]`), task lists, footnotes, definition lists, maths, code includes, a landscape section, page breaks, an appendix with a shifted heading level, links between chapters. `DRAFT.md` is listed with `include: false` | | `docs/guide/05_diagram_types.md` | One example of every other diagram type: blockdiag, actdiag, seqdiag, nwdiag, packetdiag, ditaa, wavedrom, bytefield-svg, wireviz, railroad, pikchr, vega, vega-lite and bpmnt (BPMN, drawn by bpmn-text). Graphviz, Mermaid and PlantUML are in `docs/guide/02_design.md` | | `docs/datasheet.md` | A short datasheet that includes the diagram chapter, to try the datasheet template (`@@` placeholders) and the LaTeX template | | `docs/release-notes.md` | A short document whose version comes from Git (`version: git`) | @@ -45,7 +45,7 @@ What needs Docker: Word, PowerPoint, PDF and HTML output, every diagram type, dr | `docs/deck/slides.md` | The deck: section slides, bullets, columns, incremental lists, both kinds of speaker notes, diagrams, a picture, a table, maths, callouts, code, an include, a condition, a page break | | `poster/` | One-page HTML posters, listed under `posters:` in `py-doc.yml`. Between them they use every poster setting and block, and every diagram the tool container draws. `poster.md` is portrait in the default theme, with a picture, a Mermaid diagram, charts, the poster blocks and a QR code, all set in its front matter. `landscape.md` has the words and the layout (`orientation: landscape`) in its front matter. The company's look is in `py-doc.yml`: its theme (`brand/example-instruments.yml`, a made-up company with its logo), embedded fonts, header and footer. It has a gallery and steps. `release.md` uses the dark theme, a two-series chart and a `{{version}}` variable. `reliability.md` is a three-column A0 report in the print theme, with PlantUML, Graphviz, Vega-Lite and ditaa, includes, conditions and `version: git`. They are outside `docs/` because a poster is not a chapter of the guide | | `docs/includes/` | Files pulled in with `[[INCLUDE:]]` and `[[INCLUDE-CODE:]]` | -| `templates/sensor.docx` | The Word template: logo, title page with `Replace-with-*` placeholders, header, footer with page numbers, callout styles, a `Lead Paragraph` style, `[[INSERTCONTENT]]` where the body goes | +| `templates/sensor.docx` | The Word template: logo, title page with `Replace-with-*` placeholders, header, footer with page numbers, callout styles, a `Lead Paragraph` style, `Highlight Yellow` and `Highlight Grey` character styles, `[[INSERTCONTENT]]` where the body goes | | `templates/sensor_Draft.docx` | The same template with a red "DRAFT" header. A document with `status: Draft` uses it without being told to | | `templates/sensor.pptx`, `sensor_Draft.pptx` | The PowerPoint template, and the one a `status: Draft` deck uses. Coloured titles and a `{{VERSION}}` shape on the master, filled in when the deck is built | | `templates/datasheet.docx` | A Word template with its own placeholder prefix (`@@Title`) and body marker (`<>`) | diff --git a/examples/docs/guide/03_operations.md b/examples/docs/guide/03_operations.md index f25f626..5ca373f 100644 --- a/examples/docs/guide/03_operations.md +++ b/examples/docs/guide/03_operations.md @@ -20,6 +20,29 @@ Customers see only the public build. This paragraph is shown to everyone except staff. [[ENDIF]] +## Commissioning + +A rich table holds paragraphs, lists, merged cells and diagrams in its cells. `
` starts a new paragraph, `- ` a bullet and `1. ` a numbered item, and two spaces put an item inside the one above. A cell of `<` joins the cell on its left, and `^` the cell above. + +[[CAPTION:Table]] Commissioning steps + +[[LISTTABLE: header, widths=8,52,40]] +| Step | What to do | Check | +| 1 | [Prepare the node:]{.underline}
Fit the batteries and close the enclosure.
Before you go to site:
- The firmware is version {{version}}
- The node is registered on the broker
- with its own client certificate | The light flashes green once. | +| 2 | Pair the node over Bluetooth:
1. Hold the button for five seconds.
2. Choose the node in the app. | ^ | +| 3 | The first reading goes this way:
[[BLOCK: reading]] | < | +| 4 | [Report a node that publishes nothing to support.]{custom-style="Highlight Yellow"} | < | + +[[BLOCK: reading]] +```dot +digraph reading { + rankdir=LR; + node [shape=box, style=rounded]; + Sensor -> Gateway -> Broker -> Dashboard; +} +``` +[[ENDBLOCK]] + ## Glossary [[INCLUDE: ../includes/glossary.md]] diff --git a/examples/templates/datasheet.docx b/examples/templates/datasheet.docx index bec128c..c3390d8 100644 Binary files a/examples/templates/datasheet.docx and b/examples/templates/datasheet.docx differ diff --git a/examples/templates/make_templates.py b/examples/templates/make_templates.py index 019a9b4..802cc04 100644 --- a/examples/templates/make_templates.py +++ b/examples/templates/make_templates.py @@ -26,7 +26,7 @@ from docx import Document from docx.enum.style import WD_STYLE_TYPE -from docx.enum.text import WD_ALIGN_PARAGRAPH +from docx.enum.text import WD_ALIGN_PARAGRAPH, WD_COLOR_INDEX from docx.oxml import parse_xml as parse_word_xml from docx.oxml.ns import nsdecls as word_ns from docx.shared import Cm, Pt, RGBColor @@ -103,12 +103,21 @@ def set_styles(document) -> None: lead.base_style = find(styles, "Normal") lead.font.size, lead.font.color.rgb = Pt(13), GREY lead.paragraph_format.space_after = Pt(10) + highlight_styles(document) for section in document.sections: section.page_width, section.page_height = Cm(21.0), Cm(29.7) section.left_margin = section.right_margin = Cm(2.4) section.top_margin = section.bottom_margin = Cm(2.4) +def highlight_styles(document) -> None: + """The character styles that ``[text]{custom-style="Highlight Yellow"}`` uses, in a [[LISTTABLE]] or anywhere.""" + names = {s.name for s in document.styles} + for name, colour in (("Highlight Yellow", WD_COLOR_INDEX.YELLOW), ("Highlight Grey", WD_COLOR_INDEX.GRAY_25)): + if name not in names: + document.styles.add_style(name, WD_STYLE_TYPE.CHARACTER).font.highlight_color = colour + + def title_page(document) -> None: """Logo, title, description and a table of document facts, all filled in from the front matter.""" for paragraph in list(document.paragraphs): diff --git a/examples/templates/report.tex b/examples/templates/report.tex index 8b89a4f..e3111a2 100644 --- a/examples/templates/report.tex +++ b/examples/templates/report.tex @@ -30,7 +30,7 @@ \makeatother \setkeys{Gin}{keepaspectratio} \usepackage{adjustbox} -\usepackage{longtable,booktabs,array,calc} +\usepackage{longtable,booktabs,array,calc,multirow} % multirow: cells merged down, in [[LISTTABLE]] tables \usepackage{xcolor} \usepackage{amsmath} % \text, align and the other commands pandoc writes for maths \usepackage{amssymb} % \boxtimes and \square, which pandoc writes for task lists diff --git a/examples/templates/report_Draft.tex b/examples/templates/report_Draft.tex index 02b1f5d..7994039 100644 --- a/examples/templates/report_Draft.tex +++ b/examples/templates/report_Draft.tex @@ -31,7 +31,7 @@ \makeatother \setkeys{Gin}{keepaspectratio} \usepackage{adjustbox} -\usepackage{longtable,booktabs,array,calc} +\usepackage{longtable,booktabs,array,calc,multirow} % multirow: cells merged down, in [[LISTTABLE]] tables \usepackage{xcolor} \usepackage{amsmath} % \text, align and the other commands pandoc writes for maths \usepackage{amssymb} % \boxtimes and \square, which pandoc writes for task lists diff --git a/examples/templates/sensor.docx b/examples/templates/sensor.docx index da465c3..ac95f1c 100644 Binary files a/examples/templates/sensor.docx and b/examples/templates/sensor.docx differ diff --git a/examples/templates/sensor_Draft.docx b/examples/templates/sensor_Draft.docx index 1d44f9a..f186239 100644 Binary files a/examples/templates/sensor_Draft.docx and b/examples/templates/sensor_Draft.docx differ diff --git a/src/py_doc/directives.py b/src/py_doc/directives.py index 5af5d3d..47427d2 100644 --- a/src/py_doc/directives.py +++ b/src/py_doc/directives.py @@ -10,6 +10,9 @@ A page break (a new slide in a deck), the same as a ``::: {.pagebreak}`` block. ``{{name}}`` Replace with the value of a variable. +``[[LISTTABLE]]``, ``[[BLOCK: name]]`` ... ``[[ENDBLOCK]]`` + A table whose cells hold paragraphs, lists, merged cells and blocks: see :mod:`py_doc.listtable`. + Only when ``expand`` is asked for tables, as a document build does. Directives on their own line, and ``{{name}}`` in running text, are the only forms recognised. Anything in a fenced code block or an inline code span is left alone, so a document can show the syntax. Paths are @@ -26,6 +29,7 @@ from pathlib import Path from typing import Any +from . import listtable from .errors import ConversionError logger = logging.getLogger(__name__) @@ -134,13 +138,25 @@ def expand( file: Path, variables: dict[str, Any] | None = None, conditions: dict[str, Any] | None = None, + tables: bool = False, ) -> str: - """Apply every directive in ``text``, which was read from ``file``. Raises :class:`DirectiveError`.""" - lines = _expand(text, Path(file), variables or {}, conditions or {}, (), 0) + """Apply every directive in ``text``, which was read from ``file``. Raises :class:`DirectiveError`. + + With ``tables``, ``[[LISTTABLE]]`` tables become grid tables and ``[[BLOCK:]]`` definitions Divs. + """ + lines = _expand(text, Path(file), variables or {}, conditions or {}, (), 0, tables) return "\n".join(lines) + ("\n" if lines and text.endswith("\n") else "") -def _expand(text, file: Path, variables, conditions, parents: tuple[Path, ...], depth: int) -> list[str]: +def _substituted(line: str, variables, file: Path, number: int) -> str: + undefined: list[str] = [] + line = _substitute(line, variables, undefined) + for name in dict.fromkeys(undefined): + logger.warning(f"{file}:{number}: variable '{name}' is not defined, so '{{{{{name}}}}}' is left as written") + return line + + +def _expand(text, file: Path, variables, conditions, parents: tuple[Path, ...], depth: int, tables: bool) -> list[str]: def fail(number: int, message: str) -> DirectiveError: return DirectiveError(f"{file}:{number}: {message}") @@ -149,7 +165,13 @@ def fail(number: int, message: str) -> DirectiveError: out: list[str] = [] fence = Fence() blocks: list[tuple[bool, int]] = [] # (condition held, line of the [[IF:]]) + rich = listtable.Tables(str(file)) for number, line in enumerate(text.splitlines(), start=1): + if rich.table is not None: + if listtable.ROW.match(line): + rich.table.add(number, _substituted(line, variables, file, number)) + continue + out += rich.flush() if fence.inside(line): if all(held for held, _ in blocks): out.append(line) @@ -168,6 +190,9 @@ def fail(number: int, message: str) -> DirectiveError: continue if not all(held for held, _ in blocks): continue + if tables and (replaced := rich.directive(number, line)) is not None: + out += replaced + continue if _PAGEBREAK.match(line): out += ["", "::: {.pagebreak}", ":::", ""] continue @@ -182,28 +207,41 @@ def fail(number: int, message: str) -> DirectiveError: raise fail(number, f"{target.name} includes itself, directly or through other files") logger.debug(f"Including {target} from {file}:{number}") fragment = target.read_text(encoding="utf-8-sig") - out += _expand(fragment, target, variables, conditions, (*parents, file.resolve()), depth + 1) + out += _expand(fragment, target, variables, conditions, (*parents, file.resolve()), depth + 1, tables) continue - undefined: list[str] = [] - out.append(_substitute(line, variables, undefined)) - for name in dict.fromkeys(undefined): - logger.warning(f"{file}:{number}: variable '{name}' is not defined, so '{{{{{name}}}}}' is left as written") + out.append(_substituted(line, variables, file, number)) if blocks: raise fail(blocks[-1][1], "[[IF:]] or [[NOT:]] is never closed with [[ENDIF]]") + out += rich.close() + for number, message in rich.problems: + logger.warning(f"{file}:{number}: {message}") + if rich.errors: + raise fail(*rich.errors[0]) + if rich.count: + logger.info(f"{file}: {rich.count} [[LISTTABLE]] table(s) converted") return out def check(text: str, file: Path) -> list[Problem]: """Mistakes in the directives of one file that do not need variables or conditions to find. - That is: an [[ENDIF]] with nothing open, a block never closed, and an include whose file is missing. + That is: an [[ENDIF]] with nothing open, a block never closed, an include whose file is missing, and + the mistakes of [[LISTTABLE]] tables and their [[BLOCK:]]s. """ problems: list[Problem] = [] fence = Fence() open_lines: list[int] = [] + rich = listtable.Tables(str(file)) for number, line in enumerate(text.splitlines(), start=1): + if rich.table is not None: + if listtable.ROW.match(line): + rich.table.add(number, line) + continue + rich.flush() if fence.inside(line): continue + if rich.directive(number, line) is not None: + continue if _IF.match(line): open_lines.append(number) elif _ENDIF.match(line): @@ -216,4 +254,6 @@ def check(text: str, file: Path) -> list[Problem]: if not target.is_file(): problems.append(Problem(number, f"included file not found: {match['path']}")) problems += [Problem(n, "[[IF:]] or [[NOT:]] is never closed with [[ENDIF]]") for n in open_lines] + rich.close() + problems += [Problem(number, message) for number, message in rich.problems + rich.errors] return sorted(problems, key=lambda p: p.line) diff --git a/src/py_doc/docx_import.py b/src/py_doc/docx_import.py index 8c07940..b9ef104 100644 --- a/src/py_doc/docx_import.py +++ b/src/py_doc/docx_import.py @@ -24,6 +24,7 @@ from lxml import etree from .errors import ConversionError, EnvironmentProblem, ToolFailed +from .pandoc.docx_post import has_header_row from .runtime import Mount, Runtime, connect logger = logging.getLogger(__name__) @@ -342,33 +343,58 @@ def _list_item(self, depth: int, marker: str, text: str) -> None: # -- tables --------------------------------------------------------------------------------- def _table(self, tbl) -> None: + """A pipe table, or a ``[[LISTTABLE]]`` when a cell has several paragraphs or a list, cells are + merged, or the table has no header row: those a pipe table cannot hold.""" self._close_list() - rows: list[list[str]] = [] + rows: list[list[str]] = [] # the cells as a pipe table has them: the paragraphs joined, merges empty + rich_rows: list[list[str]] = [] # the cells as a [[LISTTABLE]] has them + header = has_header_row(tbl) + rich = not header for tr in tbl.findall(qn("w:tr")): cells: list[str] = [] + rich_cells: list[str] = [] for tc in tr.findall(qn("w:tc")): props = tc.find(qn("w:tcPr")) span = props.find(qn("w:gridSpan")) if props is not None else None merge = props.find(qn("w:vMerge")) if props is not None else None width = int(span.get(W_VAL)) if span is not None and span.get(W_VAL, "").isdigit() else 1 + rich = rich or width > 1 if merge is not None and merge.get(W_VAL, "continue") == "continue": - text = "" - else: - paragraphs = [self._render(self._tokens(p)).strip() for p in tc.iter(W_P)] - text = " ".join(x for x in paragraphs if x).replace("|", "\\|") - text = re.sub(r"\s*\\\n\s*", " ", text) - cells.extend([text] + [""] * (width - 1)) - rows.append(cells) - rows = [r for r in rows if r] + rich = True + cells.extend([""] * width) + rich_cells.extend(["^"] * width) + continue + paragraphs = [] + for p in tc.iter(W_P): + text = re.sub(r"\s*\\\n\s*", " ", self._render(self._tokens(p)).strip()).replace("|", "\\|") + if text: + item = self._list_info(p, self._style_name(p).lower()) + paragraphs.append(f"{' ' * item[0]}{item[1]} {text}" if item else text) + rich = rich or item is not None + rich = rich or len(paragraphs) > 1 + cells.extend([" ".join(paragraphs)] + [""] * (width - 1)) + rich_cells.extend(["
".join(paragraphs)] + ["<"] * (width - 1)) + if cells: + rows.append(cells) + rich_rows.append(rich_cells) if not rows: return columns = max(len(r) for r in rows) rows = [r + [""] * (columns - len(r)) for r in rows] + rich_rows = [r + [""] * (columns - len(r)) for r in rich_rows] def line(cells: list[str]) -> str: return "|" + "|".join(f" {c} " if c else " " for c in cells) + "|" - self._add("table", [line(rows[0]), line(["---"] * columns), *(line(r) for r in rows[1:])]) + if not rich: + self._add("table", [line(rows[0]), line(["---"] * columns), *(line(r) for r in rows[1:])]) + return + grid = [int(c.get(qn("w:w"), "0")) for c in tbl.findall(f"{qn('w:tblGrid')}/{qn('w:gridCol')}")] + options = ["header"] if header else [] + if len(grid) == columns and sum(grid) and len(set(grid)) > 1: + options.append("widths=" + ",".join(str(max(1, round(100 * w / sum(grid)))) for w in grid)) + marker = f"[[LISTTABLE: {', '.join(options)}]]" if options else "[[LISTTABLE]]" + self._add("table", [marker, *(line(r) for r in rich_rows)]) # -- inline content ------------------------------------------------------------------------- diff --git a/src/py_doc/lint/structure.py b/src/py_doc/lint/structure.py index a410af0..1d4945a 100644 --- a/src/py_doc/lint/structure.py +++ b/src/py_doc/lint/structure.py @@ -182,6 +182,9 @@ def load_rules(path: Path) -> list[StructureRule]: _BACKTICKS = re.compile(r"`+[^`]*`+") _URL = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE) _TABLE_ROW = re.compile(r"^\s*\|.*\|\s*$") +_LISTTABLE = re.compile(r"^\[\[LISTTABLE\b[^\]]*\]\]\s*$") +_CELL_BLOCK = re.compile(r"^\[\[BLOCK:[^\]]*\]\]\s*$") # a block of a [[LISTTABLE]] cell, up to [[ENDBLOCK]] +_END_CELL_BLOCK = re.compile(r"^\[\[ENDBLOCK\]\]\s*$") _IMAGE = re.compile(r"!\[[^\]]*\]\([^)]+\)") _DIV = re.compile(r"^(:{3,})\s*\{?\.?([A-Za-z0-9_-]*)") _CODE = re.compile(r"^ {0,3}(`{3,}|~{3,})\s*([^\s`{]*)") @@ -326,6 +329,10 @@ def captioned(after: int, rule: StructureRule, first: int | None = None) -> bool i = 0 while i < count: line = lines[i] + if _CELL_BLOCK.match(line): # what is in a table cell needs no caption of its own + while i < count and not _END_CELL_BLOCK.match(lines[i]): + i += 1 + continue if code := _CODE.match(line): marker, language = code.group(1), code.group(2).lower() end = i + 1 @@ -348,7 +355,7 @@ def captioned(after: int, rule: StructureRule, first: int | None = None) -> bool i = end + 1 continue if table and _TABLE_ROW.match(line): - first = i + first = i - 1 if i and _LISTTABLE.match(lines[i - 1]) else i # a caption goes before the marker while i < count and _TABLE_ROW.match(lines[i]): i += 1 if not captioned(i, table, first): diff --git a/src/py_doc/lint/text.py b/src/py_doc/lint/text.py index 77f846b..e9c522e 100644 --- a/src/py_doc/lint/text.py +++ b/src/py_doc/lint/text.py @@ -75,11 +75,48 @@ def _mask(text: str) -> str: return text +_LISTTABLE = re.compile(r"\[\[LISTTABLE\b") +_BR = re.compile(r"", re.IGNORECASE) + + +def _listtable_segments(paragraph: n.Paragraph) -> list[Segment]: + """A ``[[LISTTABLE]]`` and its rows, read as one paragraph, as a segment per cell paragraph. + + A row is a line, a cell ends at a ``|``, and ``
`` starts a new paragraph in a cell. The parser has + already turned an escaped ``\\|`` into ``|``, so a cell holding one is read as two: harmless for prose rules. + """ + found: list[Segment] = [] + builder = _Builder(paragraph.line) + + def close(line: int) -> None: + nonlocal builder + segment = builder.segment() + if segment.text.strip(): + found.append(segment) + builder = _Builder(line) + + for node in paragraph.children: + if isinstance(node, n.SoftBreak) or (isinstance(node, n.HtmlInline) and _BR.fullmatch(node.text.strip())): + close(node.line) + elif isinstance(node, n.Text): + for index, text in enumerate(node.text.split("|")): + if index: + close(node.line) + builder.add(text, node.line) + else: + builder.inlines([node]) + close(paragraph.line) + return found + + def segments(document: n.Document) -> list[Segment]: """The prose of ``document`` in source order.""" found: list[Segment] = [] for node in n.walk(document): - if isinstance(node, (n.Heading, n.Paragraph, n.TableCell)): + first = node.children[0] if isinstance(node, n.Paragraph) and node.children else None + if isinstance(first, n.Text) and _LISTTABLE.match(first.text): + found += _listtable_segments(node) + elif isinstance(node, (n.Heading, n.Paragraph, n.TableCell)): builder = _Builder(node.line) builder.inlines(node.children) segment = builder.segment() diff --git a/src/py_doc/listtable.py b/src/py_doc/listtable.py new file mode 100644 index 0000000..fdb9d12 --- /dev/null +++ b/src/py_doc/listtable.py @@ -0,0 +1,339 @@ +"""``[[LISTTABLE]]``: tables whose cells hold paragraphs, lists, merged cells and blocks such as diagrams. + +A marker line, then one row per line:: + + [[LISTTABLE: header, widths=5,95]] + | Step | What happens | + | 1 | First paragraph
Second paragraph
- a bullet
- a bullet inside it
1. a numbered item | + | 2 | [[BLOCK: wiring]] | + | < | ^ | + +``
`` starts a new paragraph in a cell. A paragraph that starts with ``- `` is a bullet and one that starts +with ``1. `` a numbered item; two spaces before the marker put the item inside the one above. A cell of only +``<`` is merged with the cell on its left, and one of only ``^`` with the cell above. ``header`` makes the first +row the header row. ``[[BLOCK: name]]`` as a paragraph of its own is replaced by the blocks between +``[[BLOCK: name]]`` and ``[[ENDBLOCK]]`` lines elsewhere in the same file: a diagram, a picture, code. + +Each table becomes a pandoc grid table, which takes several blocks per cell, in a Div of class ``listtable``. +The blocks go in a Div of class ``listtable-block`` each, which ``listtable.lua`` moves into their cells. +Pandoc reads a grid table by character position, so every line is padded exactly. + +:class:`Tables` is fed the lines of one file by :mod:`py_doc.directives`, after conditions and variables. +""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass, field + +MARKER = re.compile(r"^\[\[LISTTABLE(?::(?P[^\]]*))?\]\]\s*$") +ROW = re.compile(r"^\|.*\|\s*$") +BLOCK = re.compile(r"^\[\[BLOCK:\s*(?P[\w-]+)\s*\]\]\s*$") +ENDBLOCK = re.compile(r"^\[\[ENDBLOCK\]\]\s*$") +_OPTIONS = re.compile( + r"\s*(?:(?Pheader)\s*(?:,\s*|$))?(?:widths\s*=\s*(?P\d+(?:\s*,\s*\d+)*))?\s*(?:,\s*(?Pheader))?\s*" +) +_CELL_SEPARATOR = re.compile(r"(?", re.IGNORECASE) +_ITEM = re.compile(r"^(?P-|\d+\.)\s+(?P.*)$") +_MARKER_LIKE = re.compile(r"^(?:[-+*]|\d+[.)])$") # a word pandoc would read as a list item at the start of a line +TOTAL_WIDTH = 100 # wider than pandoc's 72 columns, so pandoc keeps the relative widths +MIN_WIDTH = 3 + + +@dataclass(frozen=True) +class Options: + header: bool + widths: list[int] + + +def options(text: str | None) -> Options | None: + """The options of a marker, or None when they are not ``header`` and ``widths=a,b,c``.""" + match = _OPTIONS.fullmatch(text or "") + if not match or (match["before"] and match["after"]): + return None + widths = [int(w) for w in match["widths"].split(",")] if match["widths"] else [] + return Options(bool(match["before"] or match["after"]), widths) + + +def _cells(row: str) -> list[str]: + return [cell.strip() for cell in _CELL_SEPARATOR.split(row.strip()[1:-1])] + + +def _scale(weights: list[int]) -> list[int]: + """Column widths in characters, ``TOTAL_WIDTH`` in all, at least ``MIN_WIDTH`` each.""" + weights = weights if sum(weights) else [1] * len(weights) + widths = [max(MIN_WIDTH, w * TOTAL_WIDTH // sum(weights)) for w in weights] + widest = widths.index(max(widths)) + widths[widest] = max(MIN_WIDTH, widths[widest] + TOTAL_WIDTH - sum(widths)) + return widths + + +def _words(text: str) -> list[str]: + """Split at spaces, but never inside a ``{...}`` attribute block or an inline code span.""" + words, current, depth, ticks = [], "", 0, 0 + for token in re.findall(r"`+|[^`]", text): + if token[0] == "`": + ticks = len(token) if not ticks else 0 if len(token) == ticks else ticks + elif not ticks and token == "{": + depth += 1 + elif not ticks and token == "}" and depth: + depth -= 1 + if token == " " and not depth and not ticks: + if current: + words.append(current) + current = "" + else: + current += token + return [*words, current] if current else words + + +def _wrap(text: str, width: int) -> list[str]: + """Lines of at most ``width``. A line never starts with a word pandoc would take for a list marker.""" + lines: list[str] = [] + for word in _words(text): + if lines and len(lines[-1]) + 1 + len(word) <= width: + lines[-1] += " " + word + else: + lines.append(re.sub(r"^(\d*)(.)$", r"\1\\\2", word) if _MARKER_LIKE.match(word) else word) + return lines or [""] + + +@dataclass +class _Cell: + text: str + row: int + column: int + rows: int = 1 + columns: int = 1 + lines: list[str] = field(default_factory=list) + + +class Table: + """One ``[[LISTTABLE]]``: its marker, its rows as they are added, then its grid table.""" + + def __init__(self, marker: str, line: int, settings: Options, scope: str) -> None: + self.marker, self.line, self.settings, self.scope = marker, line, settings, scope + self.rows: list[tuple[int, str]] = [] + self.problems: list[tuple[int, str]] = [] + self.slots: dict[str, int] = {} # block name: the line of the first row that uses it + + def add(self, number: int, row: str) -> None: + self.rows.append((number, row)) + + def _cell_lines(self, text: str, width: int, number: int) -> list[str]: + """The lines of one cell: paragraphs apart by a blank line, a list after one, items together.""" + lines: list[str] = [] + indents: list[int] = [] # the content column of each open list level + top_marker = None + for piece in _BREAK.split(text): + paragraph = piece.strip() + if not paragraph: + continue + item = _ITEM.match(paragraph) + if item: + kind = "-" if item["marker"] == "-" else "1." + level = min((len(piece) - len(piece.lstrip(" "))) // 2, len(indents)) + if not indents or (level == 0 and kind != top_marker): + if lines: + lines.append("") + indents, level, top_marker = [], 0, kind + del indents[level:] + lead = " " * sum(indents) + marker = item["marker"] + " " + wrapped = _wrap(item["text"], width - len(lead) - len(marker)) + lines += [lead + marker + wrapped[0]] + [lead + " " * len(marker) + w for w in wrapped[1:]] + indents.append(len(marker)) + continue + if lines: + lines.append("") + indents = [] + slot = BLOCK.match(paragraph) + if slot: + self.slots.setdefault(slot["name"], number) + paragraph = f"§{list(self.slots).index(slot['name']) + 1}" + lines += _wrap(paragraph, width) + return lines + + def _layout(self, columns: int) -> list[_Cell]: + """The cells that are not merged into another, with their spans. Merges that are not a rectangle are undone.""" + grid: list[list[_Cell | None]] = [] + owner: dict[tuple[int, int], _Cell] = {} + for r, (number, row) in enumerate(self.rows): + texts = _cells(row) + if len(texts) != columns: + self.problems.append( + ( + number, + f"[[LISTTABLE]] row has {len(texts)} cell(s), not {columns}, so it is " + f"{'padded' if len(texts) < columns else 'cut'} to {columns}", + ) + ) + texts = (texts + [""] * columns)[:columns] + grid.append([]) + for c, text in enumerate(texts): + merge = text if text in ("<", "^") else None + if merge == "<" and c == 0: + self.problems.append((number, "'<' in the first column has no cell on its left, so it is empty")) + merge, text = None, "" + if merge == "^" and (r == 0 or (r == 1 and self.settings.header)): + where = "the first row has no cell above" if r == 0 else "a body cell cannot join the header row" + self.problems.append((number, f"'^' cannot be merged: {where}, so it is empty")) + merge, text = None, "" + if merge: + owner[(r, c)] = owner[(r, c - 1) if merge == "<" else (r - 1, c)] + grid[r].append(None) + else: + owner[(r, c)] = _Cell(text, r, c) + grid[r].append(owner[(r, c)]) + regions: dict[int, list[tuple[int, int]]] = {} # the spots of each cell, by its id + for spot, cell in owner.items(): + regions.setdefault(id(cell), []).append(spot) + for anchor in [cell for row in grid for cell in row if cell]: + spots = regions[id(anchor)] + rows = {r for r, _ in spots} + cols = {c for _, c in spots} + box = {(r, c) for r in range(min(rows), max(rows) + 1) for c in range(min(cols), max(cols) + 1)} + if set(spots) != box: + self.problems.append( + (self.rows[anchor.row][0], "these merged cells are not a rectangle, so they are not merged") + ) + for spot in spots: + if spot != (anchor.row, anchor.column): + owner[spot] = _Cell("", *spot) + grid[spot[0]][spot[1]] = owner[spot] + continue + anchor.rows, anchor.columns = len(rows), len(cols) + return [cell for row in grid for cell in row if cell] + + def grid(self) -> list[str]: + """The grid table, with every line the same width. Records problems on the way.""" + weights = self.settings.widths or [1] * len(_cells(self.rows[0][1])) + widths = _scale(weights) + columns, count = len(widths), len(self.rows) + cells = self._layout(columns) + xs = [sum(widths[:c]) + c for c in range(columns + 1)] # the | of each column boundary + for cell in cells: + width = xs[cell.column + cell.columns] - xs[cell.column] - 3 + number = self.rows[cell.row][0] + cell.lines = self._cell_lines(cell.text, width, number) + if any(len(line) > width for line in cell.lines): + self.problems.append( + ( + number, + f"a word in a [[LISTTABLE]] cell is wider than its column of {width} characters, so the " + "table cannot be read. Widen the column or shorten the word", + ) + ) + heights = [1] * count + for cell in sorted(cells, key=lambda cell: cell.rows): + room = sum(heights[cell.row : cell.row + cell.rows]) + cell.rows - 1 + heights[cell.row + cell.rows - 1] += max(0, len(cell.lines) - room) + ys = [sum(heights[:r]) + r for r in range(count + 1)] + canvas = [[" "] * (xs[-1] + 1) for _ in range(ys[-1] + 1)] + + def put(y: int, x: int, char: str) -> None: + if canvas[y][x] == " " or char == "+": + canvas[y][x] = char + + for cell in cells: + top, bottom = ys[cell.row], ys[cell.row + cell.rows] + left, right = xs[cell.column], xs[cell.column + cell.columns] + for y in (top, bottom): + rule = "=" if self.settings.header and y == ys[1] and count > 1 else "-" + for x in range(left + 1, right): + put(y, x, rule) + for x in (left, right): + for y in range(top + 1, bottom): + put(y, x, "|") + for y in (top, bottom): + for x in (left, right): + put(y, x, "+") + for i, line in enumerate(cell.lines): + canvas[top + 1 + i][left + 2 : left + 2 + len(line)] = list(line) + return ["".join(row) for row in canvas] + + def finish(self) -> list[str]: + """The lines that replace the marker and the rows.""" + if not self.rows: + self.problems.append((self.line, "[[LISTTABLE]] is left as written: no '| ... |' rows follow it")) + return [self.marker] + lines = self.grid() + keys = " ".join(f"{self.scope}-{name}" for name in self.slots) + div = f'::: {{.listtable blocks="{keys}"}}' if keys else "::: {.listtable}" + return ["", div, *lines, ":::", ""] + + +class Tables: + """The ``[[LISTTABLE]]`` tables and ``[[BLOCK:]]`` definitions of one file, fed to it line by line. + + ``problems`` are warnings: the table is still made. ``errors`` (a block never closed, an + ``[[ENDBLOCK]]`` with no block) stop a build. + """ + + def __init__(self, file: str) -> None: + self.scope = hashlib.sha1(file.encode("utf-8")).hexdigest()[:8] + self.table: Table | None = None + self.problems: list[tuple[int, str]] = [] + self.errors: list[tuple[int, str]] = [] + self.count = 0 + self._open: tuple[int, str] | None = None + self._defined: dict[str, int] = {} + self._used: dict[str, int] = {} + + def flush(self) -> list[str]: + """End the table being read, if there is one, and return its lines.""" + if self.table is None: + return [] + table, self.table = self.table, None + lines = table.finish() + self.problems += table.problems + if table.rows: + self.count += 1 + for name, number in table.slots.items(): + self._used.setdefault(name, number) + return lines + + def directive(self, number: int, line: str) -> list[str] | None: + """The lines that replace ``line`` when it is a marker or a block line, else None.""" + if match := MARKER.match(line): + settings = options(match["options"]) + if settings is None: + self.problems.append( + (number, "[[LISTTABLE]] is left as written: its options are not 'header' and 'widths=a,b,c'") + ) + return [line] + self.table = Table(line, number, settings, self.scope) + return [] + if match := BLOCK.match(line): + if self._open: + self.errors.append((number, f"[[BLOCK: {match['name']}]] is inside [[BLOCK: {self._open[1]}]]")) + return [] + if match["name"] in self._defined: + self.errors.append((number, f"[[BLOCK: {match['name']}]] is defined twice in this file")) + self._open = (number, match["name"]) + self._defined.setdefault(match["name"], number) + return ["", f'::: {{.listtable-block key="{self.scope}-{match["name"]}"}}', ""] + if ENDBLOCK.match(line): + if not self._open: + self.errors.append((number, "[[ENDBLOCK]] has no [[BLOCK:]] before it")) + return [] + self._open = None + return ["", ":::", ""] + return None + + def close(self) -> list[str]: + """At the end of the file: the last table's lines, and the checks of the blocks.""" + lines = self.flush() + if self._open: + self.errors.append((self._open[0], f"[[BLOCK: {self._open[1]}]] is never closed with [[ENDBLOCK]]")) + for name, number in self._used.items(): + if name not in self._defined: + self.problems.append( + (number, f"[[BLOCK: {name}]] is used in a table, but this file does not define it") + ) + for name, number in self._defined.items(): + if name not in self._used: + self.problems.append((number, f"[[BLOCK: {name}]] is defined, but no table in this file uses it")) + return lines diff --git a/src/py_doc/pandoc/docx_finish.py b/src/py_doc/pandoc/docx_finish.py index be7da02..e304831 100644 --- a/src/py_doc/pandoc/docx_finish.py +++ b/src/py_doc/pandoc/docx_finish.py @@ -8,6 +8,8 @@ from typing import Any from docx import Document +from docx.oxml import parse_xml +from docx.oxml.ns import nsdecls from py_doc.markdown import Metadata @@ -21,7 +23,20 @@ def _composer(): with warnings.catch_warnings(): warnings.simplefilter("ignore") from docxcompose.composer import Composer - return Composer + from docxcompose.utils import xpath + + class OneListAtATime(Composer): + """docxcompose gives every list of one body element the same new number, so the lists in the cells + of a table (a [[LISTTABLE]]) would all share the first one's bullets or numbers. Each list is + added on its own first, so each gets a number of its own; then the element's references follow.""" + + def add_numberings(self, doc, element): + for num_id in sorted({int(n) for n in xpath(element, ".//w:numId/@w:val")}): + one = f'' + super().add_numberings(doc, parse_xml(one)) + super().add_numberings(doc, element) + + return OneListAtATime DEFAULT_PREFIX = "Replace-with-" @@ -104,8 +119,9 @@ def finish_docx(body_path: Path, template: Path | None, metadata: Metadata, opti post.apply_layout_fixes(document) post.rescale_landscape_images(document) post.apply_explicit_styles(document) - post.format_all_tables(document) - post.keep_captions_with_tables(document) # after the [[CAPTION:]] lines are captions + rich = post.take_rich_table_marks(document) + post.format_all_tables(document, rich=rich) + post.keep_captions_with_tables(document, rich) # after the [[CAPTION:]] lines are captions post.update_core_properties( document, title=text(metadata.title), diff --git a/src/py_doc/pandoc/docx_post.py b/src/py_doc/pandoc/docx_post.py index 3fd88cc..99d04c2 100644 --- a/src/py_doc/pandoc/docx_post.py +++ b/src/py_doc/pandoc/docx_post.py @@ -28,8 +28,11 @@ from docx.oxml import OxmlElement from docx.oxml.ns import nsmap as _DOCX_NSMAP from docx.oxml.ns import qn +from docx.shape import InlineShape from docx.shared import Cm, RGBColor from docx.text.paragraph import Paragraph + +from .reference import DEFAULT_TEXT_WIDTH_CM from lxml import etree @@ -493,12 +496,104 @@ def apply_explicit_styles(doc: Document) -> None: # --------------------------------------------------------------------------- +def has_header_row(tbl) -> bool: + """True when the table ```` says its first row is a header. + + That is the ``firstRow`` of its ``w:tblLook`` (or the same bit of the older ``w:val`` form), and when + the table has no ``w:tblLook``, a ``w:tblHeader`` on its first row. Pandoc writes ``firstRow="0"`` for a + table without a header, such as a ``[[LISTTABLE]]``. + """ + look = tbl.tblPr.find(qn("w:tblLook")) if tbl.tblPr is not None else None + if look is not None: + if look.get(qn("w:firstRow")) is not None: + return look.get(qn("w:firstRow")) in ("1", "true", "on") + if look.get(qn("w:val")) is not None: + return bool(int(look.get(qn("w:val")), 16) & 0x0020) + first = tbl.find(qn("w:tr")) + return first is not None and first.find(f"{qn('w:trPr')}/{qn('w:tblHeader')}") is not None + + +RICH_TABLE_MARK = "_PyDocRichTable" # the bookmark listtable.lua puts before a [[LISTTABLE]] +_CELL_MARGINS = 2 * 68580 # Word's default left and right cell margins, 0.075 inch each, in EMU + + +def take_rich_table_marks(doc: Document) -> set: + """The ```` of each ``[[LISTTABLE]]``, found by the bookmark before it, which is taken out.""" + rich = set() + marks = [m for m in doc.element.body.iter(qn("w:bookmarkStart")) if m.get(qn("w:name")) == RICH_TABLE_MARK] + for mark in marks: + end = mark.getnext() + if end is not None and end.tag == qn("w:bookmarkEnd"): # its own end: docxcompose renumbers only the start + end.getparent().remove(end) + table = mark.getnext() + if table is not None and table.tag == qn("w:tbl"): + rich.add(table) + mark.getparent().remove(mark) + return rich + + +def _text_width_at(doc: Document, element) -> int: + """The text-area width (EMU) of the section that holds ``element``, a child of the body. + + A section without a page size (pandoc's own reference document has none) has pandoc's default width. + """ + sections = list(doc.sections) + index = 0 + for child in doc.element.body.iterchildren(): + if child is element: + break + if child.tag == qn("w:p") and child.find(f"{qn('w:pPr')}/{qn('w:sectPr')}") is not None: + index += 1 + section = sections[min(index, len(sections) - 1)] + if None in (section.page_width, section.left_margin, section.right_margin): + return Cm(DEFAULT_TEXT_WIDTH_CM) + return section.page_width - section.left_margin - section.right_margin + + +def _fit_pictures_to_cells(tbl, text_width: int) -> None: + """Make each picture in a cell of ``tbl`` (a fixed layout table) no wider than its cell.""" + columns = [int(c.get(qn("w:w"), "0")) for c in tbl._tbl.tblGrid.findall(qn("w:gridCol"))] + total = sum(columns) or 1 + for tr in tbl._tbl.findall(qn("w:tr")): + column = 0 + for tc in tr.findall(qn("w:tc")): + room = int(text_width * sum(columns[column : column + tc.grid_span]) / total) - _CELL_MARGINS + for inline in tc.iter(qn("wp:inline")): + picture = InlineShape(inline) + if room > 0 and picture.width and picture.width > room: + picture.height = int(picture.height * room / picture.width) + picture.width = room + column += tc.grid_span + + +def _format_header_row(row) -> None: + """Row 1 as the header: repeated on each page, kept with the next row, in bold white.""" + trPr = row._tr.get_or_add_trPr() + if trPr.find(qn("w:tblHeader")) is None: + trPr.append(OxmlElement("w:tblHeader")) + for cell in row.cells: + for p in cell.paragraphs: + p.paragraph_format.keep_with_next = True + p.paragraph_format.keep_together = True + # Direct run formatting, not reliance on the table + # style's conditional "first row" formatting alone: + # whatever paragraph style Pandoc assigned to this cell + # (commonly "Compact", which inherits Normal's explicit + # navy colour) outranks the table style's white-on-navy + # override in Word's precedence, silently making header + # text invisible. Direct formatting always wins. + for run in p.runs: + run.font.bold = True + run.font.color.rgb = RGBColor(0xFF, 0xFF, 0xFF) + + def format_all_tables( doc: Document, style_name: str = "Table Grid", repeat_header: bool = True, prevent_row_split: bool = True, autofit: bool = True, + rich: set | frozenset = frozenset(), ) -> None: """ Apply a consistent appearance to every table in *doc*. @@ -515,6 +610,12 @@ def format_all_tables( * Enables autofit and removes fixed cell widths. * Marks the first row as a repeating header. * Prevents rows from splitting across pages. + + A ``[[LISTTABLE]]`` (its ```` is in ``rich``, see :func:`take_rich_table_marks`) and a table + without a header row (see :func:`has_header_row`) keep the column widths pandoc wrote, in a fixed + layout, and their rows may break across pages, so a tall row does not leave a gap by moving whole to + the next page. A picture in one of their cells is made no wider than its cell. Only a header row, if + there is one, is formatted and repeated as a header, and kept on one page. """ try: _ = doc.styles[style_name] @@ -536,6 +637,17 @@ def format_all_tables( tblW.set(qn("w:type"), "pct") tblW.set(qn("w:w"), "5000") # 5000 = 100% in Word's fiftieths-of-a-percent units + header = has_header_row(tbl._tbl) + if tbl._tbl in rich or not header: + tbl.autofit = False # a fixed layout, on the column widths pandoc wrote + if header and repeat_header and tbl.rows: + _format_header_row(tbl.rows[0]) + trPr = tbl.rows[0]._tr.get_or_add_trPr() + if trPr.find(qn("w:cantSplit")) is None: + trPr.append(OxmlElement("w:cantSplit")) + _fit_pictures_to_cells(tbl, _text_width_at(doc, tbl._tbl)) + continue + if autofit: try: tbl.autofit = True @@ -549,24 +661,7 @@ def format_all_tables( tcPr.remove(tcW) if repeat_header and tbl.rows: - hdr = tbl.rows[0] - trPr = hdr._tr.get_or_add_trPr() - if trPr.find(qn("w:tblHeader")) is None: - trPr.append(OxmlElement("w:tblHeader")) - for cell in hdr.cells: - for p in cell.paragraphs: - p.paragraph_format.keep_with_next = True - p.paragraph_format.keep_together = True - # Direct run formatting, not reliance on the table - # style's conditional "first row" formatting alone: - # whatever paragraph style Pandoc assigned to this cell - # (commonly "Compact", which inherits Normal's explicit - # navy colour) outranks the table style's white-on-navy - # override in Word's precedence, silently making header - # text invisible. Direct formatting always wins. - for run in p.runs: - run.font.bold = True - run.font.color.rgb = RGBColor(0xFF, 0xFF, 0xFF) + _format_header_row(tbl.rows[0]) if prevent_row_split: for row in tbl.rows: @@ -701,12 +796,15 @@ def _keep_with_next(paragraph) -> None: Paragraph(paragraph, None).paragraph_format.keep_with_next = True # in the order the schema wants -def keep_captions_with_tables(doc: Document) -> None: +def keep_captions_with_tables(doc: Document, rich: set | frozenset = frozenset()) -> None: """A table caption stays on the page of its table. Word has no "keep with previous", so a caption below a table is held by keeping the table's last - row (which may not split) with the next paragraph. A caption above a table is kept with the table. - A caption between two tables is the next one's, as in the other formats (markers.lua). + row (which may not split) with the next paragraph. In a ``[[LISTTABLE]]`` (``rich``) or a table without + a header row the last row may still split, so only the last paragraph of each of its cells is kept + with the caption. A caption above + a table is kept with the table. A caption between two tables is the next one's, as in the other + formats (markers.lua). """ style_names = {style.style_id: style.name for style in doc.styles} held = 0 @@ -714,7 +812,13 @@ def keep_captions_with_tables(doc: Document) -> None: before, after = table.getprevious(), table.getnext() rows = table.findall(qn("w:tr")) below = _is_caption(after, style_names) and (after.getnext() is None or after.getnext().tag != qn("w:tbl")) - if rows and below: + if rows and below and (table in rich or not has_header_row(table)): + for cell in rows[-1].iter(qn("w:tc")): # the row may still split: only its end is held + paragraphs = cell.findall(qn("w:p")) + if paragraphs: + _keep_with_next(paragraphs[-1]) + held += 1 + elif rows and below: trPr = rows[-1].get_or_add_trPr() if trPr.find(qn("w:cantSplit")) is None: trPr.append(OxmlElement("w:cantSplit")) diff --git a/src/py_doc/pandoc/engine.py b/src/py_doc/pandoc/engine.py index 838e8bf..02abd26 100644 --- a/src/py_doc/pandoc/engine.py +++ b/src/py_doc/pandoc/engine.py @@ -30,10 +30,10 @@ DEFAULT_TOC_DEPTH = 3 INPUT_FORMAT = "markdown+fenced_divs+raw_html+raw_attribute+table_captions+strikeout+task_lists" FILTERS = { - "docx": ("admonitions.lua", "pagebreak.lua", "widths.lua"), - "pptx": ("admonitions.lua", "pagebreak.lua", "markers.lua"), - "html": ("admonitions.lua", "pagebreak.lua", "markers.lua"), - "pdf": ("admonitions.lua", "pagebreak.lua", "center-div.lua", "markers.lua"), + "docx": ("listtable.lua", "admonitions.lua", "pagebreak.lua", "widths.lua"), + "pptx": ("listtable.lua", "admonitions.lua", "pagebreak.lua", "markers.lua"), + "html": ("listtable.lua", "admonitions.lua", "pagebreak.lua", "markers.lua"), + "pdf": ("listtable.lua", "admonitions.lua", "pagebreak.lua", "center-div.lua", "markers.lua"), } THEME = RESOURCES / "createdocs.theme" diff --git a/src/py_doc/pandoc/resources/latex_template.tex b/src/py_doc/pandoc/resources/latex_template.tex index 9b60c7d..c0a427f 100644 --- a/src/py_doc/pandoc/resources/latex_template.tex +++ b/src/py_doc/pandoc/resources/latex_template.tex @@ -40,7 +40,7 @@ \makeatother \setkeys{Gin}{keepaspectratio} \usepackage{adjustbox} -\usepackage{longtable,booktabs,array,calc} +\usepackage{longtable,booktabs,array,calc,multirow} % multirow: cells merged down, in [[LISTTABLE]] tables \usepackage{xcolor} \usepackage{amsmath} % \text, align and the other commands pandoc writes for maths \usepackage{amssymb} % \boxtimes and \square, which pandoc writes for task lists diff --git a/src/py_doc/pandoc/resources/listtable.lua b/src/py_doc/pandoc/resources/listtable.lua new file mode 100644 index 0000000..45b9681 --- /dev/null +++ b/src/py_doc/pandoc/resources/listtable.lua @@ -0,0 +1,50 @@ +-- [[LISTTABLE]] tables, which py-doc writes as a grid table in a Div of class "listtable". +-- +-- A cell paragraph §N becomes the blocks of the Nth key in the Div's "blocks" attribute: a [[BLOCK:]] +-- that py-doc wrote as a Div of class "listtable-block" elsewhere in the document, and which is taken out +-- of the place it was written. A picture or diagram in a cell is not a figure: it has no caption or number +-- of its own. The table then leaves its Div. In Word a hidden bookmark goes before it, +-- so the Word finishing knows a rich table (its rows may break across pages) from an ordinary one. + +local blocks = {} + +local function collect(div) + if div.classes:includes("listtable-block") then + blocks[div.attributes.key] = div.content + return {} + end +end + +local function place(div) + if not div.classes:includes("listtable") then + return nil + end + local keys = {} + for key in (div.attributes.blocks or ""):gmatch("%S+") do + keys[#keys + 1] = key + end + local filled = div:walk({ + Blocks = function(list) + local out = pandoc.List() + for _, block in ipairs(list) do + local slot = (block.t == "Plain" or block.t == "Para") and #block.content == 1 + and block.content[1].t == "Str" and block.content[1].text:match("^§(%d+)$") + local found = slot and blocks[keys[tonumber(slot)]] + if found then + out:extend(pandoc.Blocks(found):walk({ Figure = function(figure) return figure.content end })) + else + out:insert(block) + end + end + return out + end, + }) + local out = pandoc.List() + if FORMAT == "docx" then + out:insert(pandoc.RawBlock("openxml", '')) + end + out:extend(filled.content) + return out +end + +return { { Div = collect }, { Div = place } } diff --git a/src/py_doc/pandoc/source.py b/src/py_doc/pandoc/source.py index aacef54..673ab72 100644 --- a/src/py_doc/pandoc/source.py +++ b/src/py_doc/pandoc/source.py @@ -174,7 +174,7 @@ def join_sources(sources, variables: dict, conditions: dict, doc_id: str) -> Joi text = source.file.read_text(encoding="utf-8-sig") except UnicodeDecodeError as exc: raise ConversionError(f"Cannot read '{source.file}': it is not valid UTF-8 ({exc.reason})") from exc - text = directives.expand(text, source.file, variables, conditions) + text = directives.expand(text, source.file, variables, conditions, tables=True) warn_unclosed_fence(text, str(source.file)) front, body = split_front_matter(text) metadata = read_front_matter(front, source.file) diff --git a/src/py_doc/report.py b/src/py_doc/report.py index 68623ce..52feba6 100644 --- a/src/py_doc/report.py +++ b/src/py_doc/report.py @@ -23,6 +23,7 @@ _VERSION_FLAGS = {"dot": ["-V"], "java": ["-version"], "plantuml": ["-version"]} _WORDS = re.compile(r"\w+(?:['’-]\w+)*") _SENTENCE_END = re.compile(r"[.!?]+(?:\s|$)") +_RICH_TABLE = re.compile(r"^::: \{\.listtable[ }]", re.M) # the Div py-doc puts each [[LISTTABLE]] in def counts(text: str) -> dict[str, int]: @@ -107,7 +108,10 @@ def document(self, doc_id: str) -> dict[str, Any]: return entry def sources(self, doc_id: str, parts: list[tuple[Path, str]]) -> None: - self.document(doc_id)["sources"] = [{"file": self.shown(file), **counts(text)} for file, text in parts] + self.document(doc_id)["sources"] = [ + {"file": self.shown(file), **counts(text), "rich_tables": len(_RICH_TABLE.findall(text))} + for file, text in parts + ] def output(self, doc_id: str, fmt: str, path: Path) -> None: data = Path(path).read_bytes() diff --git a/src/py_doc/report.schema.json b/src/py_doc/report.schema.json index b212dee..35b69ef 100644 --- a/src/py_doc/report.schema.json +++ b/src/py_doc/report.schema.json @@ -44,13 +44,14 @@ "type": "array", "items": { "type": "object", - "required": ["file", "words", "sentences", "paragraphs"], + "required": ["file", "words", "sentences", "paragraphs", "rich_tables"], "additionalProperties": false, "properties": { "file": {"type": "string"}, "words": {"type": "integer", "minimum": 0}, "sentences": {"type": "integer", "minimum": 0}, - "paragraphs": {"type": "integer", "minimum": 0} + "paragraphs": {"type": "integer", "minimum": 0}, + "rich_tables": {"type": "integer", "minimum": 0, "description": "The [[LISTTABLE]] tables of the file and the files it includes"} } } }, diff --git a/src/py_doc/site/pages.py b/src/py_doc/site/pages.py index cabf2ea..9abc4dd 100644 --- a/src/py_doc/site/pages.py +++ b/src/py_doc/site/pages.py @@ -41,7 +41,7 @@ MANIFEST = ".py-doc-site.json" DOWNLOAD_LABELS = {"pdf": "PDF", "docx": "Word", "pptx": "PowerPoint", "html": "HTML"} -FILTERS = ("admonitions.lua", "pagebreak.lua", "markers.lua", "site.lua") +FILTERS = ("listtable.lua", "admonitions.lua", "pagebreak.lua", "markers.lua", "site.lua") OUTPUT_FORMAT = "gfm" # pipe tables, task lists and raw HTML. site.lua writes the maths TIMEOUT = 900 _LINK = re.compile(r"(?P!?\[(?:[^\]\\]|\\.)*\]\()(?P<[^>]*>|[^)\s]+)(?P[^)]*\))") diff --git a/tests/test_docx_import.py b/tests/test_docx_import.py index 8c65d7a..277fe85 100644 --- a/tests/test_docx_import.py +++ b/tests/test_docx_import.py @@ -11,7 +11,7 @@ from docx.enum.style import WD_STYLE_TYPE from docx.opc.constants import RELATIONSHIP_TYPE as RT from docx.oxml import parse_xml -from docx.oxml.ns import nsdecls +from docx.oxml.ns import nsdecls, qn from PIL import Image from py_doc.app import main @@ -119,7 +119,7 @@ def test_the_structure_is_kept(sample, tmp_path): assert "Plain **bold** and *italic* with a_b\\* star.[a link](https://example.com)" in text assert "- one\n - child\n- two\n" in text assert "1. first\n1. second" in text - assert "| A | B | C |\n| --- | --- | --- |\n| wide \\| cell | | x |\n| 1 | | |" in text + assert "[[LISTTABLE: header]]\n| A | B | C |\n| wide \\| cell | < | x |\n| 1 | | |" in text # a merged cell assert "```\nprint('hi')\nprint('there')\n```" in text assert "> [!NOTE]\n> Careful now." in text assert "> As they said." in text @@ -259,3 +259,32 @@ def test_a_round_trip_keeps_headings_lists_tables_and_pictures(sample, tmp_path, assert sum("List" in s or s.startswith("Compact") for s in styles) >= 3 assert len(back.tables) == 1 assert len(back.inline_shapes) == 1 + + +def test_a_table_of_paragraphs_lists_and_merged_rows_imports_as_a_listtable(tmp_path): + doc = Document() + table = doc.add_table(rows=3, cols=2) + table.cell(0, 0).text, table.cell(0, 1).text = "Step", "What" + table.cell(1, 0).merge(table.cell(2, 0)).text = "1" + cell = table.cell(1, 1) + cell.text = "Fit the node." + cell.add_paragraph("Check:") + cell.add_paragraph("dry", style="List Bullet") + cell.add_paragraph("isolated", style="List Bullet 2") + table.cell(2, 1).text = "Done." + table._tbl.tblGrid.findall(qn("w:gridCol"))[0].set(qn("w:w"), "1134") + table._tbl.tblGrid.findall(qn("w:gridCol"))[1].set(qn("w:w"), "7938") + doc.save(tmp_path / "t.docx") + text = read(import_docx(tmp_path / "t.docx", tmp_path / "out")) + assert ( + "[[LISTTABLE: header, widths=12,88]]\n| Step | What |\n" + "| 1 | Fit the node.
Check:
- dry
- isolated |\n| ^ | Done. |" + ) in text + + +def test_a_simple_table_stays_a_pipe_table(tmp_path): + doc = Document() + table = doc.add_table(rows=2, cols=2) + table.cell(0, 0).text, table.cell(0, 1).text, table.cell(1, 0).text, table.cell(1, 1).text = "a", "b", "1", "2" + doc.save(tmp_path / "t.docx") + assert "| a | b |\n| --- | --- |\n| 1 | 2 |" in read(import_docx(tmp_path / "t.docx", tmp_path / "out")) diff --git a/tests/test_examples.py b/tests/test_examples.py index d066dc9..c461fca 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -12,6 +12,7 @@ import pytest from docx import Document +from docx.oxml.ns import qn from pptx import Presentation from py_doc.app import main @@ -212,6 +213,11 @@ def test_the_guide_shows_the_word_features(examples, pandoc_runtime): assert "footnoteReference" in xml assert 'w:orient="landscape"' in xml, "a landscape section" assert "SEQ" in xml, "captions are numbered with fields" + rich = next(t for t in document.tables if t.cell(0, 0).text == "Step") + assert "The firmware is version 1.0" in rich.cell(1, 1).text, "a [[LISTTABLE]] takes variables" + assert 'w:rStyle w:val="HighlightYellow"' in rich._tbl.xml + assert rich._tbl.xpath(".//w:vMerge") and rich._tbl.xpath(".//w:gridSpan[@w:val='2']"), "merged cells" + assert rich._tbl.findall(qn("w:tr"))[3].xpath(".//wp:inline"), "the diagram is in its cell" with zipfile.ZipFile(examples / "build" / "guide-v1.0.docx") as archive: core = archive.read("docProps/core.xml").decode("utf-8") assert "Integrating the SN-100" in core and "Technical documentation" in core and "IoT, sensor, MQTT" in core diff --git a/tests/test_listtable.py b/tests/test_listtable.py new file mode 100644 index 0000000..94b1a77 --- /dev/null +++ b/tests/test_listtable.py @@ -0,0 +1,348 @@ +"""[[LISTTABLE]]: tables of paragraphs, lists, merged cells and blocks, as grid tables, laid out in Word.""" + +from __future__ import annotations + +import logging +import re +import shutil +import subprocess +import zipfile +from pathlib import Path + +import pytest +from docx import Document +from docx.oxml.ns import qn +from docx.shared import Cm +from PIL import Image + +from py_doc.config import Source +from py_doc.directives import DirectiveError, check, expand +from py_doc.listtable import _wrap +from py_doc.pandoc.reference import DEFAULT_TEXT_WIDTH_CM +from py_doc.pandoc.source import join_sources +from py_doc.report import BuildReport + +LUA = Path(__file__).parent.parent / "src" / "py_doc" / "pandoc" / "resources" / "listtable.lua" + +SAMPLE = """Before. + +[[LISTTABLE: widths=5,95]] +| 1 | [Meter installation:]{.underline}
The installer attends the premises.
Meter default settings:
- Mode = PAYG (prepayment)
- Meter balance = 15 m3 | +| 2 | Inform DNO about successful installation:
The installer notifies the DNO backend. | +| 3 | The message contains the following fields:
- Supply Meter Point (SMP)
- [Address details (postcode is required)]{custom-style="Highlight Yellow"} | + +| Name | Value | +|---|---| +| a | 1 | +""" + +needs_pandoc = pytest.mark.skipif(shutil.which("pandoc") is None, reason="pandoc is not installed") + + +def convert(text: str) -> str: + return expand(text, Path("t.md"), tables=True) + + +def grid(text: str) -> list[str]: + return [line for line in convert(text).splitlines() if line.startswith(("+", "|"))] + + +def cells(text: str, column: int = 2) -> list[str]: + """The text of one column of a one-row table, line by line, without its padding on the right.""" + return [line.split("|")[column].rstrip() for line in grid(text)[1:-1]] + + +class TestConversion: + def test_the_sample_is_a_grid_table_with_every_line_the_same_width(self): + lines = grid(SAMPLE.split("\n\n| Name")[0]) + assert lines[0].startswith("+-----+---") and lines[-1] == lines[0] + assert len({len(line) for line in lines}) == 1 + assert "[[LISTTABLE" not in convert(SAMPLE) + + def test_paragraphs_are_apart_and_bullets_together(self): + assert cells("[[LISTTABLE]]\n| a | p
q
- x
- y
r |\n") == [ + " p", + "", + " q", + "", + " - x", + " - y", + "", + " r", + ] + + def test_widths_add_up_to_a_hundred_with_three_at_least(self): + border = grid("[[LISTTABLE: widths=1,99]]\n| a | b |\n")[0] + assert [len(part) for part in border.strip("+").split("+")] == [3, 97] + assert [len(part) for part in grid("[[LISTTABLE]]\n| a | b | c |\n")[0].strip("+").split("+")] == [34, 33, 33] + + def test_a_long_bullet_wraps_inside_its_bullet_and_never_inside_an_attribute(self): + text = '[[LISTTABLE: widths=1,1]]\n| a | - [some words that go on]{custom-style="Highlight Yellow"} and on and on and on |\n' + lines = cells(text) + assert len(lines) > 2 + assert lines[0].startswith(" - ") and all(c.startswith(" ") for c in lines[1:]) + assert any('{custom-style="Highlight Yellow"}' in c for c in lines) + + def test_an_escaped_pipe_is_text(self): + assert len(grid("[[LISTTABLE]]\n| a \\| b | c |\n")[0].strip("+").split("+")) == 2 + + def test_a_short_row_is_padded_and_a_long_one_cut_with_a_warning(self, caplog): + with caplog.at_level(logging.WARNING): + lines = grid("[[LISTTABLE: widths=1,1]]\n| a |\n| a | b | c |\n") + assert len({len(line) for line in lines}) == 1 and "c" not in "".join(lines) + assert "t.md:2: [[LISTTABLE]] row has 1 cell(s), not 2, so it is padded" in caplog.text + assert "t.md:3: [[LISTTABLE]] row has 3 cell(s), not 2, so it is cut" in caplog.text + + def test_a_marker_without_rows_or_with_unknown_options_is_left_with_a_warning(self, caplog): + with caplog.at_level(logging.WARNING): + assert convert("[[LISTTABLE]]\n\ntext\n") == "[[LISTTABLE]]\n\ntext\n" + assert convert("[[LISTTABLE: wide]]\n| a |\n") == "[[LISTTABLE: wide]]\n| a |\n" + assert caplog.text.count("[[LISTTABLE]] is left as written") == 2 + + def test_a_marker_in_code_is_left_alone(self): + text = "```markdown\n[[LISTTABLE]]\n| a | b |\n```\n\n`[[LISTTABLE]]`\n" + assert convert(text) == text + + def test_a_pipe_table_is_unchanged_and_the_count_is_logged(self, caplog): + with caplog.at_level(logging.INFO): + out = convert(SAMPLE) + assert out.endswith("| Name | Value |\n|---|---|\n| a | 1 |\n") + assert "t.md: 1 [[LISTTABLE]] table(s) converted" in caplog.text + + def test_the_same_input_gives_the_same_output(self): + assert convert(SAMPLE) == convert(SAMPLE) + + def test_tables_work_in_included_files_and_take_variables(self, tmp_path): + (tmp_path / "part.md").write_text("[[LISTTABLE]]\n| {{who}} | b |\n", encoding="utf-8") + (tmp_path / "main.md").write_text("# T\n\n[[INCLUDE: part.md]]\nAfter.\n", encoding="utf-8") + joined = join_sources([Source(tmp_path / "main.md")], {"who": "Ann"}, {}, "d") + assert "| Ann" in joined.markdown and "+---" in joined.markdown + assert "\n\nAfter." in joined.markdown # not part of the table + + +class TestLists: + def test_numbered_items_make_one_list(self): + assert cells("[[LISTTABLE]]\n| a | Do:
1. one
2. two |\n") == [" Do:", "", " 1. one", " 2. two"] + + def test_two_spaces_put_an_item_inside_the_one_above(self): + text = "[[LISTTABLE]]\n| a | - a
- b
- c
- d
1. e
- f |\n" + assert cells(text) == [" - a", " - b", " - c", " - d", "", " 1. e", " - f"] + + def test_an_item_cannot_go_more_than_one_level_deeper(self): + assert cells("[[LISTTABLE]]\n| a | - a
- b |\n") == [" - a", " - b"] + + def test_a_wrapped_word_that_looks_like_a_list_marker_is_escaped(self): + assert _wrap("aaaa - b 1. c", 5) == ["aaaa", "\\- b", "1\\. c"] + assert _wrap("+ starts", 20) == ["\\+ starts"] + + +class TestHeaderAndMerges: + def test_a_header_row_is_ruled_off_with_equals_signs(self): + lines = grid("[[LISTTABLE: header, widths=1,3]]\n| A | B |\n| a | b |\n") + assert lines[2].startswith("+=") and lines[2].count("+") == 3 and "=" in lines[2] and "-" not in lines[2] + assert grid("[[LISTTABLE: widths=1,3, header]]\n| A | B |\n| a | b |\n") == lines + assert convert("[[LISTTABLE: header, header]]\n| a |\n").startswith("[[LISTTABLE: header, header]]") + + def test_a_cell_of_lt_joins_the_cell_on_its_left(self): + lines = grid("[[LISTTABLE]]\n| a | < |\n| b | c |\n") + assert lines[0].count("+") == 2 and lines[1].count("|") == 2 # one cell across the top + assert lines[2].count("+") == 3 + + def test_a_cell_of_caret_joins_the_cell_above(self): + lines = grid("[[LISTTABLE]]\n| a | b |\n| ^ | c |\n") + assert lines[2].startswith("| ") and "+-" in lines[2] # no rule under the merged cell + + def test_impossible_merges_are_empty_cells_with_a_warning(self, caplog): + with caplog.at_level(logging.WARNING): + convert("[[LISTTABLE]]\n| < | ^ |\n| a | b |\n") + convert("[[LISTTABLE]]\n| a | b |\n| ^ | < |\n") + convert("[[LISTTABLE: header]]\n| A | B |\n| ^ | b |\n") + assert "t.md:2: '<' in the first column has no cell on its left" in caplog.text + assert "t.md:2: '^' cannot be merged: the first row has no cell above" in caplog.text + assert "these merged cells are not a rectangle" in caplog.text + assert "a body cell cannot join the header row" in caplog.text + + @needs_pandoc + def test_pandoc_reads_the_merges_and_the_header(self): + text = "[[LISTTABLE: header]]\n| A | B | C |\n| a | < | b |\n| ^ | ^ | c |\n" + native = subprocess.run( + ["pandoc", "-f", "markdown", "-t", "native"], input=convert(text), capture_output=True, text=True + ) + assert native.stderr == "" + assert "(RowSpan 2)\n" in native.stdout and "(ColSpan 2)" in native.stdout + assert re.search(r"TableHead.*?Str \"A\".*?TableBody", native.stdout, re.S) + + +class TestBlocks: + TEXT = "[[LISTTABLE]]\n| a | See:
[[BLOCK: pic]] |\n\n[[BLOCK: pic]]\n![](p.png)\n[[ENDBLOCK]]\n" + + def test_a_block_is_a_slot_in_its_cell_and_a_div_of_its_own(self): + out = convert(self.TEXT) + key = re.search(r'::: \{\.listtable blocks="(\w+-pic)"\}', out)[1] + assert "§1" in out and f'::: {{.listtable-block key="{key}"}}' in out and "![](p.png)" in out + + def test_a_block_used_but_not_defined_or_defined_but_not_used_is_a_warning(self, caplog): + with caplog.at_level(logging.WARNING): + convert("[[LISTTABLE]]\n| a | [[BLOCK: gone]] |\n\n[[BLOCK: spare]]\nx\n[[ENDBLOCK]]\n") + assert "t.md:2: [[BLOCK: gone]] is used in a table, but this file does not define it" in caplog.text + assert "t.md:4: [[BLOCK: spare]] is defined, but no table in this file uses it" in caplog.text + + def test_an_unclosed_block_or_a_stray_end_stops_the_build(self): + with pytest.raises(DirectiveError, match="t.md:1: \\[\\[BLOCK: a\\]\\] is never closed"): + convert("[[BLOCK: a]]\nx\n") + with pytest.raises(DirectiveError, match="t.md:1: \\[\\[ENDBLOCK\\]\\] has no"): + convert("[[ENDBLOCK]]\n") + + @needs_pandoc + def test_the_filter_moves_the_block_into_its_cell(self, tmp_path): + html = subprocess.run( + ["pandoc", "-f", "markdown", "-t", "html", "--lua-filter", str(LUA)], + input=convert(self.TEXT), capture_output=True, text=True, check=True, + ).stdout # fmt: skip + assert re.search(r"

See:

\s*

", html) + assert html.count("', body) + + +def test_a_rich_table_is_headerless_fixed_and_may_split(docx): + document, body = docx + rich = document.tables[0]._tbl + assert rich.tblPr.find(qn("w:tblLook")).get(qn("w:firstRow")) == "0" + assert rich.tblPr.find(qn("w:tblLayout")).get(qn("w:type")) == "fixed" + assert rich.xpath(".//w:tblHeader") == [] and rich.xpath(".//w:cantSplit") == [] + assert rich.xpath(".//w:r/w:rPr/w:color[@w:val='FFFFFF']") == [] + assert rich.xpath(".//w:pPr/w:keepLines") == [] + + +def test_a_pipe_table_keeps_its_header_and_rows_together(docx): + document, _ = docx + pipe = document.tables[1]._tbl + assert pipe.tblPr.find(qn("w:tblLook")).get(qn("w:firstRow")) == "1" + rows = pipe.findall(qn("w:tr")) + assert rows[0].find(f"{qn('w:trPr')}/{qn('w:tblHeader')}") is not None + assert all(row.find(f"{qn('w:trPr')}/{qn('w:cantSplit')}") is not None for row in rows) + layout = pipe.tblPr.find(qn("w:tblLayout")) + assert layout is None or layout.get(qn("w:type")) == "autofit" + + +def test_a_narrow_first_column_is_about_five_percent(docx): + document, _ = docx + columns = [int(c.get(qn("w:w"))) for c in document.tables[0]._tbl.tblGrid.findall(qn("w:gridCol"))] + assert 0.04 < columns[0] / sum(columns) < 0.08 + + +def test_lint_reads_each_cell_paragraph_as_prose_of_its_own(): + from py_doc.lint.text import segments + from py_doc.markdown import parse + + found = [s.text.strip() for s in segments(parse("[[LISTTABLE]]\n| 1 | One.
- Two |\n| 2 | Three |\n"))] + assert found == ["1", "One.", "- Two", "2", "Three"] + + +RICH = """# T + +[[LISTTABLE: header, widths=30,70]] +| Part | Picture | +| Board | [[BLOCK: photo]] | +| ^ | Fitted on the wall:
1. Drill
2. Fix | +| Spare | < | + +[[BLOCK: photo]] +![The board](big.png) +[[ENDBLOCK]] +""" + + +@pytest.fixture +def rich_docx(tmp_path, convert_text): + Image.new("RGB", (2000, 800), "red").save(tmp_path / "big.png") + return Document(str(convert_text(RICH))) + + +def test_a_rich_header_row_is_a_header_and_the_other_rows_may_split(rich_docx): + table = rich_docx.tables[0]._tbl + rows = table.findall(qn("w:tr")) + assert table.tblPr.find(qn("w:tblLayout")).get(qn("w:type")) == "fixed" + assert rows[0].find(f"{qn('w:trPr')}/{qn('w:tblHeader')}") is not None + assert rows[0].find(f"{qn('w:trPr')}/{qn('w:cantSplit')}") is not None + assert rows[0].xpath(".//w:r/w:rPr/w:color[@w:val='FFFFFF']") + assert all(row.find(f"{qn('w:trPr')}/{qn('w:cantSplit')}") is None for row in rows[1:]) + assert not rich_docx.element.body.xpath('.//w:bookmarkStart[@w:name="_PyDocRichTable"]') + + +def test_merged_cells_reach_word(rich_docx): + rows = rich_docx.tables[0]._tbl.findall(qn("w:tr")) + assert rows[1].find(qn("w:tc")).xpath("w:tcPr/w:vMerge/@w:val") == ["restart"] + assert rows[2].find(qn("w:tc")).xpath("w:tcPr/w:vMerge/@w:val") == ["continue"] + assert rows[3].find(qn("w:tc")).grid_span == 2 + + +def test_a_block_picture_is_in_its_cell_and_no_wider_than_it(rich_docx): + text_width = Cm(DEFAULT_TEXT_WIDTH_CM) # pandoc's reference document has no page size + pictures = rich_docx.element.body.xpath(".//wp:inline") + assert len(pictures) == 1 + cell = rich_docx.tables[0]._tbl.findall(qn("w:tr"))[1].findall(qn("w:tc"))[1] + assert cell.xpath(".//wp:inline") == pictures + assert int(pictures[0].find(qn("wp:extent")).get("cx")) <= text_width * 0.7 + assert not any("§" in p.text or "The board" in p.text for p in rich_docx.paragraphs), "no caption in a cell" + numbered = rich_docx.tables[0]._tbl.findall(qn("w:tr"))[2].xpath(".//w:numPr") + assert len(numbered) == 2 + + +def test_each_list_in_a_cell_keeps_its_own_bullets_or_numbers_with_a_template(convert_text): + template = Path(__file__).parent.parent / "examples" / "templates" / "sensor.docx" + document = Document(str(convert_text("[[LISTTABLE]]\n| a | - x
- y |\n| b | 1. one
2. two |\n", template))) + numbering = document.part.numbering_part.element + + def format_of(paragraph): + (num_id,) = paragraph._p.xpath("w:pPr/w:numPr/w:numId/@w:val") + (abstract,) = numbering.xpath(f"w:num[@w:numId='{num_id}']/w:abstractNumId/@w:val") + return numbering.xpath(f"w:abstractNum[@w:abstractNumId='{abstract}']/w:lvl[@w:ilvl='0']/w:numFmt/@w:val")[0] + + table = document.tables[-1] # after the template's own + assert [format_of(p) for p in table.cell(0, 1).paragraphs] == ["bullet", "bullet"] + assert [format_of(p) for p in table.cell(1, 1).paragraphs] == ["decimal", "decimal"] diff --git a/tests/test_pandoc.py b/tests/test_pandoc.py index 3ff6655..449040c 100644 --- a/tests/test_pandoc.py +++ b/tests/test_pandoc.py @@ -259,7 +259,7 @@ def test_the_command_mounts_resources_read_only_and_translates_paths(tmp_path, m assert any("py-doc-" in m and not m.endswith("readonly") for m in mounts) resource_path = next(w for w in run if w.startswith("--resource-path=")).replace("\\", "/") assert tmp_path.as_posix() in resource_path - assert run.count("--lua-filter") == 3 and "--embed-resources" in run + assert run.count("--lua-filter") == 4 and "--embed-resources" in run def test_pdf_and_html_need_a_runtime(tmp_path): diff --git a/tests/test_report.py b/tests/test_report.py index c8079ac..f3f2096 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -63,7 +63,7 @@ def test_a_good_build_is_described(tmp_path, run): data = load(tmp_path / "out" / "r.json") assert (data["status"], data["error"]) == ("ok", None) (document,) = data["documents"] - assert document["sources"] == [{"file": "a.md", "words": 3, "sentences": 2, "paragraphs": 1}] + assert document["sources"] == [{"file": "a.md", "words": 3, "sentences": 2, "paragraphs": 1, "rich_tables": 0}] (output,) = document["outputs"] built = tmp_path / "build" / "a.docx" assert output["path"] == "build/a.docx" and output["bytes"] == built.stat().st_size diff --git a/tests/test_structure.py b/tests/test_structure.py index 95d42e7..72b2b36 100644 --- a/tests/test_structure.py +++ b/tests/test_structure.py @@ -97,6 +97,14 @@ def test_a_caption_before_the_block_counts(self): assert found("[[CAPTION:Table]] a\n| a |\n|---|\n| 1 |\n", self.CAPTION) == [] assert found("text\n\n![x](a.png)\n", image) == [(3, "r/img", "warning")] + def test_a_picture_or_diagram_in_a_listtable_block_needs_no_caption(self): + image = {**self.CAPTION, "id": "r/img", "block": "image", "required_pattern": r"\[\[CAPTION:Figure\]\]"} + assert found("[[BLOCK: a]]\n![x](a.png)\n[[ENDBLOCK]]\n\n![x](b.png)\n", image) == [(5, "r/img", "warning")] + + def test_a_caption_before_a_listtable_marker_counts(self): + assert found("[[CAPTION:Table]] a\n\n[[LISTTABLE]]\n| a | b |\n", self.CAPTION) == [] + assert found("text\n\n[[LISTTABLE]]\n| a | b |\n", self.CAPTION) == [(4, "r/cap", "warning")] + def test_front_matter_is_not_checked_as_text(self): rule = {"id": "r/rule", "type": "regex-line", "severity": "warning", "pattern": r"^---\s*$", "message": "m"} assert found("---\ntitle: x\n---\n\ntext\n\n---\n", rule) == [(7, "r/rule", "warning")]