diff --git a/docs/architecture/02_pipeline.md b/docs/architecture/02_pipeline.md index b1d473a..1407988 100644 --- a/docs/architecture/02_pipeline.md +++ b/docs/architecture/02_pipeline.md @@ -38,14 +38,14 @@ A source with `structurizr:` is replaced by Markdown files before anything else | Filter | Job | Formats | |---|---|---| -| `listtable.lua` | The blocks of `[[LISTTABLE]]` cells move into their cells, without captions. In Word a hidden bookmark marks the table for the finishing | all, and site pages | +| `listtable.lua` | The blocks of `[[LISTTABLE]]` cells move into their cells, without captions. Cell colours and alignment become a style (HTML), `\cellcolor` (PDF) or a hidden bookmark the Word finishing reads. In PDF a table in a cell becomes a `tabular`. In Word a hidden bookmark marks the table for the finishing | all, and site pages | | `admonitions.lua` | Callouts in both spellings become a box of their kind | all | | `pagebreak.lua` | Page break blocks become a page break, or a new slide | all | | `markers.lua` | `[[CAPTION:]]`, `[[STYLE:]]` and orientation markers. On slides the caption also becomes the alt text pandoc prints under a picture | PowerPoint, HTML, PDF | | `center-div.lua` | Centring divs become LaTeX centring | PDF | | `site.lua` | Admonitions or asides, figures as picture and caption, no repeated title | site pages | -Word handles the markers itself after pandoc (`pandoc/docx_post.py`), because they become Word fields, sections and styles. `pandoc/docx_finish.py` merges the result into the template, fills the placeholders and applies the layout. It merges with docxcompose one list at a time, because docxcompose gives every list in one table the same numbering. Tables get the template's table style, centred at full width. An ordinary table with a header row is autofit, its header repeated and its rows kept whole. A `[[LISTTABLE]]` (found by its bookmark) or a table without a header keeps pandoc's column widths in a fixed layout. Its body rows may break across pages, and its pictures are made no wider than their cells. `pandoc/pptx_finish.py` keeps embedded fonts, fills the version token and the footer, and warns about crowded slides. The finished file is moved to its output path, and the build report records it with its checksum. +Word handles the markers itself after pandoc (`pandoc/docx_post.py`), because they become Word fields, sections and styles. `pandoc/docx_finish.py` merges the result into the template, fills the placeholders and applies the layout. It merges with docxcompose one list at a time, because docxcompose gives every list in one table the same numbering. Before the merge each table of the Markdown gets a hidden bookmark, so the template's own tables are left as they are. Tables get the template's table style, centred at full width, with the header row's colour set on its cells. An ordinary table with a header row is autofit, its header repeated and its rows kept whole. A `[[LISTTABLE]]` (found by its bookmark) or a table without a header keeps pandoc's column widths in a fixed layout. Its body rows may break across pages, and its pictures are made no wider than their cells. `pandoc/pptx_finish.py` keeps embedded fonts, fills the version token and the footer, and warns about crowded slides. The finished file is moved to its output path, and the build report records it with its checksum. ## Around the documents diff --git a/docs/guide/03_writing.md b/docs/guide/03_writing.md index 9b48d32..8dae79f 100644 --- a/docs/guide/03_writing.md +++ b/docs/guide/03_writing.md @@ -191,20 +191,27 @@ Word uses the `Div.note` to `Div.caution` paragraph styles. The built-in referen ## Rich table cells -A pipe table holds one line of text in each cell. For cells of several paragraphs, lists, merged cells or a diagram, put `[[LISTTABLE]]` on the line before the rows, and leave out the `|---|` line: +A pipe table holds one line of text in each cell. For cells of several paragraphs, lists, merged cells, colours, a diagram or another table, put `[[LISTTABLE]]` on the line before the rows, and leave out the `|---|` line: ````markdown [[LISTTABLE: header, widths=8,52,40]] | Step | What to do | Check | -| 1 | [Prepare the node:]{.underline}
Fit the batteries.
Before you go to site:
- The firmware is version {{version}}
- The node is registered
- with its own certificate | The light flashes green. | +| 1 | [Prepare the node:]{.underline}
Fit the batteries.
Before you go to site:
- The firmware is version {{version}}
- The node is registered
- with its own certificate | {bg=E8F5E9 valign=middle} The light flashes green. | | 2 | Pair the node:
1. Hold the button for five seconds.
2. Choose the node in the app. | ^ | | 3 | The first reading goes this way:
[[BLOCK: reading]] | < | +| 4 | Wire the terminals:
[[BLOCK: terminals]] | See the [sheet](https://example.com/sn-100/installation/terminal-wiring.pdf). | [[BLOCK: reading]] ```dot digraph reading { rankdir=LR; Sensor -> Gateway -> Broker; } ``` [[ENDBLOCK]] + +[[BLOCK: terminals]] +| Terminal | Wire | +|---|---| +| V+ | Red | +[[ENDBLOCK]] ```` - **Rows**: each row is one line that starts and ends with `|`, straight after the marker. `\|` is a pipe in the text. @@ -212,18 +219,20 @@ digraph reading { rankdir=LR; Sensor -> Gateway -> Broker; } - **Widths**: `widths=8,52,40` gives the relative widths of the columns. Without it the columns are the same width. - **Paragraphs and lists**: `
` starts a new paragraph in a cell. A paragraph that starts with `- ` is a bullet and one that starts with `1. ` a numbered item, and items next to each other make one list. Two spaces before the `- ` or `1. ` put the item inside the one above. - **Merged cells**: a cell of only `<` joins the cell on its left, and a cell of only `^` the cell above. Merged cells must make a rectangle, and a body cell cannot join the header row. `\<` and `\^` are the characters themselves. -- **Blocks**: a cell paragraph `[[BLOCK: name]]` is replaced by the lines between `[[BLOCK: name]]` and `[[ENDBLOCK]]` elsewhere in the same file. A block can hold a diagram, a picture, code or a callout. A picture or diagram in a cell has no caption of its own: caption the table. Plain pictures also work inline, as `![alt](picture.png)`. +- **Colour and alignment**: a cell that starts with `{bg=E8F5E9}` has that background colour, as six hex digits. `{valign=middle}` puts its text in the middle of the row, and `top` and `bottom` work too. Both can go in one `{...}`. In PDF a colour covers the first row of a cell merged down, and the alignment is not used. +- **Blocks**: a cell paragraph `[[BLOCK: name]]` is replaced by the lines between `[[BLOCK: name]]` and `[[ENDBLOCK]]`, written elsewhere in the same file or in a file it includes, or the file that includes it. A block can hold a diagram, a picture, code, a callout or a table, including another `[[LISTTABLE]]`. A picture or diagram in a cell has no caption of its own: caption the table. Plain pictures also work inline, as `![alt](picture.png)`. +- **Long links**: a link or picture too long for its column is written as a reference after the table, so it keeps the table readable. It is still the same link. - **Formatting**: bold, italic, links, `[text]{.underline}` and `[text]{custom-style="Name"}` work as anywhere else. A `custom-style` is a character style of the Word template. Pandoc makes one with no formatting if the template has none of that name, so add the styles you use to the template. -py-doc turns the table into a pandoc grid table after includes, conditions and variables, so rich tables work in included files and take `{{variables}}`. A marker in a code block is left alone. In Word the columns keep the widths given, the rows may break across pages, and a picture is made no wider than its cell. Only a header row is repeated on each page and printed as a header. HTML, PDF and sites show the same table. On a slide pandoc shows merged cells as separate cells, bullets as plain lines and no pictures. +py-doc turns the table into a pandoc grid table after includes, conditions and variables, so rich tables work in included files and take `{{variables}}`. A marker in a code block is left alone. In Word the columns keep the widths given, the rows may break across pages, and a picture is made no wider than its cell. Only a header row is repeated on each page and printed as a header. A table in a cell is formatted like any other. HTML, PDF and sites show the same table. On a slide pandoc shows merged cells as separate cells, bullets as plain lines and no pictures, colours or tables in cells. The build warns about a row with too many or too few cells, which is padded or cut. It warns about a merge that cannot be made, which is left empty, and a marker with no rows, which is left as written. It also warns about a block that is used but not defined, or defined but not used. A `[[BLOCK:]]` that is never closed with `[[ENDBLOCK]]` stops the build. `py-doc lint` and `py-doc check` report all of these with their line. Known limits: -- A word wider than its column breaks the table, and the build says so. Widen the column or shorten the word. +- A word wider than its column breaks the table, and the build says so. Widen the column or shorten the word. Links and pictures are shortened for you, but not a long bare address such as ``. - Includes and conditions cannot go inside a cell, only around the whole table or inside a block. -- A block belongs to its file: a table cannot use a block defined in another file. +- A block is shared by a source file and the files it includes, not by the other sources of a document. ## Includes, conditions and variables diff --git a/docs/guide/04_outputs.md b/docs/guide/04_outputs.md index b8c938b..633094a 100644 --- a/docs/guide/04_outputs.md +++ b/docs/guide/04_outputs.md @@ -21,6 +21,7 @@ Pandoc writes the document with the template as its reference document, so the t - **Merging**: when a template is set, the result is merged into it. The body goes where `[[INSERTCONTENT]]` stands, or at the end if there is no marker. `[[LANDSCAPE]]`, `[[PORTRAIT]]`, `[[STYLE:Name]]` and `[[CAPTION:Figure]]` lines are applied afterwards. - **Placeholders**: `Replace-with-Title`, `-Author`, `-Version`, `-Document-Number`, `-Date`, `-Description` and `-Classification` are filled in the template body, tables, headers and footers. `placeholder_prefix` (default `Replace-with-`) and `marker` (default `[[INSERTCONTENT]]`) can be changed. A placeholder the document has no value for is emptied with a warning, and one that no property can fill (`Replace-with-Mystery`) is left and named in a warning. - **Heading numbers**: a template can number its heading styles (a `Heading 1` with list numbering). The numbers written in the headings would then show twice, so py-doc leaves them out: `## 2.1 Design` becomes `## Design`, numbered by Word. `heading_numbers: strip` or `keep` decides it for any template. See [Heading numbers](#heading-numbers). +- **Tables** from the Markdown get the template's `Table Grid` style, centred at full width. The header row is bold white on the colour the style gives a header row, or navy, set on the cells themselves so that every program shows it. It repeats on each page. A picture in a cell is made no wider than its column, and a table in a cell is formatted the same way. The template's own tables, such as a title page table of document facts, keep the look the template gives them. - **Contents list**: `docx: {toc: true}` adds a table of contents to the depth of `toc_depth` (front matter). Word fills in the page numbers when the file is opened and asks to update fields. - **Document properties** (title, author, subject, keywords, category, comments) are capped at Word's 255 characters, with a warning when a value is cut. - **Code** is highlighted with the built-in theme. diff --git a/docs/requirements/01_functional.md b/docs/requirements/01_functional.md index 8cd46ef..1014d85 100644 --- a/docs/requirements/01_functional.md +++ b/docs/requirements/01_functional.md @@ -35,7 +35,7 @@ title: Functional requirements | FR-026 | Fenced blocks of Mermaid, PlantUML, Graphviz, ditaa, the blockdiag family, WaveDrom, bytefield-svg, Pikchr, railroad, WireViz, Vega and Vega-Lite shall become pictures in every output. | Must | Met | | FR-027 | A diagram that cannot be drawn shall stop the build with the file, the line and the tool's message. `--keep-going` shall list every failure. | Must | Met | | FR-028 | `diagrams.mermaid` in `py-doc.yml` shall give every Mermaid diagram of documents, sites and posters the same Mermaid settings, under the diagram's own. | Could | Met | -| FR-029 | `[[LISTTABLE]]` followed by `\| ... \|` rows shall make a table with an optional header row and relative column widths. Its cells shall hold several paragraphs (`
`), bullet and numbered lists (`- `, `1. `) nested by indentation, cells merged across (`<`) and down (`^`), and blocks such as diagrams (`[[BLOCK:]]`). In Word its body rows may break across pages, only a header row shall be formatted and repeated as one, and a picture shall fit its cell. These mistakes shall be warnings that name the file and line: a wrong number of cells, an impossible merge, a marker with no rows, an undefined block. A block never closed shall be an error. | Should | Met | +| FR-029 | `[[LISTTABLE]]` followed by `\| ... \|` rows shall make a table with an optional header row and relative column widths. Its cells shall hold several paragraphs (`
`), bullet and numbered lists (`- `, `1. `) nested by indentation, and blocks such as diagrams and tables (`[[BLOCK:]]`). Cells shall merge across (`<`) and down (`^`), and take a colour and alignment (`{bg=RRGGBB valign=middle}`). A block may be in the file or in a file it includes. A link too long for its column shall become a reference. In Word its body rows may break across pages, only a header row shall be formatted and repeated as one, and a picture shall fit its cell. These mistakes shall be warnings that name the file and line: a wrong number of cells, an impossible merge, a marker with no rows, an undefined block. A block never closed shall be an error. | Should | Met | ## Outputs @@ -51,6 +51,7 @@ title: Functional requirements | FR-047 | `py-doc template` shall check, list, compare and count the styles of Word and PowerPoint templates. | Should | Met | | FR-048 | `py-doc docx` and `py-doc slides` shall convert single files and folders without a `py-doc.yml`. | Must | Met | | FR-049 | Where the output numbers its headings itself, the section numbers written in headings shall be left out, so no section is numbered twice. That is a Word template with numbered heading styles, or numbered PDF and HTML sections. A setting shall force or prevent it. | Must | Met | +| FR-050 | In Word, the header row of a table from the Markdown shall show in every program, and its pictures shall be no wider than their column. The template's own tables shall keep their look. | Must | Met | ## Importing diff --git a/docs/requirements/04_traceability.md b/docs/requirements/04_traceability.md index 0216a16..61d9b69 100644 --- a/docs/requirements/04_traceability.md +++ b/docs/requirements/04_traceability.md @@ -13,7 +13,7 @@ title: Traceability | FR-008 | `migrate.py` | `test_migrate.py` | | FR-020 to FR-025 | `directives.py`, `pandoc/resources/*.lua`, `pandoc/docx_post.py` | `test_directives.py`, `test_pandoc.py`, `test_word_control.py`, `test_word_layout.py` | | FR-026, FR-027, QR-080 | `diagrams/` | `test_diagrams.py` | -| FR-040 to FR-049 | `pandoc/engine.py`, `pandoc/reference.py`, `pandoc/docx_finish.py`, `pandoc/pptx_finish.py`, `naming.py`, `report.py`, `template_check.py`, `convert.py`, `_cli.py` | `test_pandoc.py`, `test_pptx_pandoc.py`, `test_report.py`, `test_template_check.py`, `test_cli.py`, `test_slides_cli.py`, `test_convert.py`, `test_examples.py`, `test_heading_numbers.py` | +| FR-040 to FR-050 | `pandoc/engine.py`, `pandoc/reference.py`, `pandoc/docx_finish.py`, `pandoc/pptx_finish.py`, `naming.py`, `report.py`, `template_check.py`, `convert.py`, `_cli.py` | `test_pandoc.py`, `test_pptx_pandoc.py`, `test_report.py`, `test_template_check.py`, `test_cli.py`, `test_slides_cli.py`, `test_convert.py`, `test_examples.py`, `test_heading_numbers.py`, `test_listtable.py` | | FR-060, FR-062 to FR-064 | `docx_import.py`, `import_cli.py` | `test_docx_import.py` | | FR-061 | `pptx_import.py` | `test_pptx_import.py` | | FR-080 to FR-086, QR-081 | `c4/` | `test_c4.py` | diff --git a/examples/docs/guide/03_operations.md b/examples/docs/guide/03_operations.md index 5ca373f..05ca345 100644 --- a/examples/docs/guide/03_operations.md +++ b/examples/docs/guide/03_operations.md @@ -22,16 +22,24 @@ This paragraph is shown to everyone except staff. ## Commissioning -A rich table holds paragraphs, lists, merged cells and diagrams in its cells. `
` starts a new paragraph, `- ` a bullet and `1. ` a numbered item, and two spaces put an item inside the one above. A cell of `<` joins the cell on its left, and `^` the cell above. +A rich table holds paragraphs, lists, merged cells, diagrams and other tables in its cells. `
` starts a new paragraph, `- ` a bullet and `1. ` a numbered item, and two spaces put an item inside the one above. A cell of `<` joins the cell on its left, and `^` the cell above. `{bg=E8F5E9 valign=middle}` at the start of a cell gives it a colour and an alignment. [[CAPTION:Table]] Commissioning steps [[LISTTABLE: header, widths=8,52,40]] | Step | What to do | Check | -| 1 | [Prepare the node:]{.underline}
Fit the batteries and close the enclosure.
Before you go to site:
- The firmware is version {{version}}
- The node is registered on the broker
- with its own client certificate | The light flashes green once. | +| 1 | [Prepare the node:]{.underline}
Fit the batteries and close the enclosure.
Before you go to site:
- The firmware is version {{version}}
- The node is registered on the broker
- with its own client certificate | {bg=E8F5E9 valign=middle} The light flashes green once. | | 2 | Pair the node over Bluetooth:
1. Hold the button for five seconds.
2. Choose the node in the app. | ^ | | 3 | The first reading goes this way:
[[BLOCK: reading]] | < | -| 4 | [Report a node that publishes nothing to support.]{custom-style="Highlight Yellow"} | < | +| 4 | Wire the terminals:
[[BLOCK: terminals]] | See the [installation sheet](https://example.com/sn-100/installation/terminal-wiring-sheet.pdf). | +| 5 | [Report a node that publishes nothing to support.]{custom-style="Highlight Yellow"} | < | + +[[BLOCK: terminals]] +| Terminal | Wire | +|---|---| +| V+ | Red | +| 0 V | Black | +[[ENDBLOCK]] [[BLOCK: reading]] ```dot diff --git a/examples/templates/datasheet.docx b/examples/templates/datasheet.docx index c3390d8..15d46d3 100644 Binary files a/examples/templates/datasheet.docx and b/examples/templates/datasheet.docx differ diff --git a/examples/templates/make_templates.py b/examples/templates/make_templates.py index 802cc04..b2382f2 100644 --- a/examples/templates/make_templates.py +++ b/examples/templates/make_templates.py @@ -118,10 +118,15 @@ def highlight_styles(document) -> None: document.styles.add_style(name, WD_STYLE_TYPE.CHARACTER).font.highlight_color = colour +def remove_sample(document) -> None: + """Take out pandoc's sample text and sample table, which its reference document starts with.""" + for block in [*document.paragraphs, *document.tables]: + block._element.getparent().remove(block._element) + + def title_page(document) -> None: """Logo, title, description and a table of document facts, all filled in from the front matter.""" - for paragraph in list(document.paragraphs): - paragraph._element.getparent().remove(paragraph._element) # pandoc's sample text + remove_sample(document) styles = document.styles logo = document.add_paragraph(style=find(styles, "Normal")) logo.alignment = WD_ALIGN_PARAGRAPH.CENTER @@ -184,8 +189,7 @@ def datasheet_docx() -> None: ``py-doc.yml`` names them with ``placeholder_prefix`` and ``marker``.""" document = Document(io.BytesIO(pandoc_file("reference.docx"))) set_styles(document) - for paragraph in list(document.paragraphs): - paragraph._element.getparent().remove(paragraph._element) + remove_sample(document) document.add_paragraph("@@Title", style="Title") document.add_paragraph("@@Description", style="Lead Paragraph") document.add_paragraph("Version @@Version, @@Date", style="Normal") diff --git a/examples/templates/report.tex b/examples/templates/report.tex index e3111a2..084b1f2 100644 --- a/examples/templates/report.tex +++ b/examples/templates/report.tex @@ -32,6 +32,7 @@ \usepackage{adjustbox} \usepackage{longtable,booktabs,array,calc,multirow} % multirow: cells merged down, in [[LISTTABLE]] tables \usepackage{xcolor} +\usepackage{colortbl} % \cellcolor: cell colours, in [[LISTTABLE]] tables \usepackage{amsmath} % \text, align and the other commands pandoc writes for maths \usepackage{amssymb} % \boxtimes and \square, which pandoc writes for task lists % pandoc writes ~~struck out~~ as \st{} or \sout{}, and underlined text as \ul{} or \uline{} diff --git a/examples/templates/report_Draft.tex b/examples/templates/report_Draft.tex index 7994039..10683bf 100644 --- a/examples/templates/report_Draft.tex +++ b/examples/templates/report_Draft.tex @@ -33,6 +33,7 @@ \usepackage{adjustbox} \usepackage{longtable,booktabs,array,calc,multirow} % multirow: cells merged down, in [[LISTTABLE]] tables \usepackage{xcolor} +\usepackage{colortbl} % \cellcolor: cell colours, in [[LISTTABLE]] tables \usepackage{amsmath} % \text, align and the other commands pandoc writes for maths \usepackage{amssymb} % \boxtimes and \square, which pandoc writes for task lists % pandoc writes ~~struck out~~ as \st{} or \sout{}, and underlined text as \ul{} or \uline{} diff --git a/examples/templates/sensor.docx b/examples/templates/sensor.docx index ac95f1c..5da222c 100644 Binary files a/examples/templates/sensor.docx and b/examples/templates/sensor.docx differ diff --git a/examples/templates/sensor_Draft.docx b/examples/templates/sensor_Draft.docx index f186239..8d9c4c7 100644 Binary files a/examples/templates/sensor_Draft.docx and b/examples/templates/sensor_Draft.docx differ diff --git a/src/py_doc/directives.py b/src/py_doc/directives.py index 47427d2..e08b77f 100644 --- a/src/py_doc/directives.py +++ b/src/py_doc/directives.py @@ -144,7 +144,10 @@ def expand( With ``tables``, ``[[LISTTABLE]]`` tables become grid tables and ``[[BLOCK:]]`` definitions Divs. """ - lines = _expand(text, Path(file), variables or {}, conditions or {}, (), 0, tables) + shared = listtable.Shared.of(str(file)) if tables else None + lines = _expand(text, Path(file), variables or {}, conditions or {}, (), 0, shared) + for where, number, message in shared.problems() if shared else []: + logger.warning(f"{where}:{number}: {message}") return "\n".join(lines) + ("\n" if lines and text.endswith("\n") else "") @@ -156,7 +159,11 @@ def _substituted(line: str, variables, file: Path, number: int) -> str: return line -def _expand(text, file: Path, variables, conditions, parents: tuple[Path, ...], depth: int, tables: bool) -> list[str]: +def _expand( + text, file: Path, variables, conditions, parents: tuple[Path, ...], depth: int, shared: listtable.Shared | None +) -> list[str]: + """The lines of ``text``. ``shared`` holds the blocks of the source being expanded, or is None for no tables.""" + def fail(number: int, message: str) -> DirectiveError: return DirectiveError(f"{file}:{number}: {message}") @@ -165,7 +172,7 @@ def fail(number: int, message: str) -> DirectiveError: out: list[str] = [] fence = Fence() blocks: list[tuple[bool, int]] = [] # (condition held, line of the [[IF:]]) - rich = listtable.Tables(str(file)) + rich = listtable.Tables(str(file), shared or listtable.Shared.of(str(file))) for number, line in enumerate(text.splitlines(), start=1): if rich.table is not None: if listtable.ROW.match(line): @@ -190,7 +197,7 @@ def fail(number: int, message: str) -> DirectiveError: continue if not all(held for held, _ in blocks): continue - if tables and (replaced := rich.directive(number, line)) is not None: + if shared and (replaced := rich.directive(number, line)) is not None: out += replaced continue if _PAGEBREAK.match(line): @@ -207,7 +214,7 @@ def fail(number: int, message: str) -> DirectiveError: raise fail(number, f"{target.name} includes itself, directly or through other files") logger.debug(f"Including {target} from {file}:{number}") fragment = target.read_text(encoding="utf-8-sig") - out += _expand(fragment, target, variables, conditions, (*parents, file.resolve()), depth + 1, tables) + out += _expand(fragment, target, variables, conditions, (*parents, file.resolve()), depth + 1, shared) continue out.append(_substituted(line, variables, file, number)) if blocks: @@ -231,7 +238,8 @@ def check(text: str, file: Path) -> list[Problem]: problems: list[Problem] = [] fence = Fence() open_lines: list[int] = [] - rich = listtable.Tables(str(file)) + shared = listtable.Shared.of(str(file)) + rich = listtable.Tables(str(file), shared) for number, line in enumerate(text.splitlines(), start=1): if rich.table is not None: if listtable.ROW.match(line): @@ -255,5 +263,28 @@ def check(text: str, file: Path) -> list[Problem]: problems.append(Problem(number, f"included file not found: {match['path']}")) problems += [Problem(n, "[[IF:]] or [[NOT:]] is never closed with [[ENDIF]]") for n in open_lines] rich.close() + _block_names(text, Path(file), shared, (Path(file).resolve(),)) problems += [Problem(number, message) for number, message in rich.problems + rich.errors] + problems += [Problem(number, message) for where, number, message in shared.problems() if where == str(file)] return sorted(problems, key=lambda p: p.line) + + +_SLOT = re.compile(r"\[\[BLOCK:\s*(?P[\w-]+)\s*\]\]") + + +def _block_names(text: str, file: Path, shared: listtable.Shared, seen: tuple[Path, ...]) -> None: + """Add the blocks the files ``file`` includes define and use to ``shared``, so a block may be in either.""" + fence = Fence() + for number, line in enumerate(text.splitlines(), start=1): + if fence.inside(line): + continue + if seen[1:]: # an included file: its own definitions and uses + if match := listtable.BLOCK.match(line): + shared.defined.setdefault(match["name"], (str(file), number)) + elif listtable.ROW.match(line): + for slot in _SLOT.finditer(line): + shared.used.setdefault(slot["name"], (str(file), number)) + if match := _INCLUDE.match(line): + target = (file.parent / match["path"]).resolve() + if target.is_file() and target not in seen: + _block_names(target.read_text(encoding="utf-8-sig"), target, shared, (*seen, target)) diff --git a/src/py_doc/lint/text.py b/src/py_doc/lint/text.py index e9c522e..068922f 100644 --- a/src/py_doc/lint/text.py +++ b/src/py_doc/lint/text.py @@ -77,6 +77,7 @@ def _mask(text: str) -> str: _LISTTABLE = re.compile(r"\[\[LISTTABLE\b") _BR = re.compile(r"", re.IGNORECASE) +_CELL_STYLE = re.compile(r"^\s*\{[^}]*=[^}]*\}") def _listtable_segments(paragraph: n.Paragraph) -> list[Segment]: @@ -102,6 +103,7 @@ def close(line: int) -> None: for index, text in enumerate(node.text.split("|")): if index: close(node.line) + text = _CELL_STYLE.sub(lambda m: " " * len(m[0]), text) # {bg=... valign=...} is not prose builder.add(text, node.line) else: builder.inlines([node]) diff --git a/src/py_doc/listtable.py b/src/py_doc/listtable.py index fdb9d12..c32caee 100644 --- a/src/py_doc/listtable.py +++ b/src/py_doc/listtable.py @@ -11,8 +11,10 @@ ``
`` starts a new paragraph in a cell. A paragraph that starts with ``- `` is a bullet and one that starts with ``1. `` a numbered item; two spaces before the marker put the item inside the one above. A cell of only ``<`` is merged with the cell on its left, and one of only ``^`` with the cell above. ``header`` makes the first -row the header row. ``[[BLOCK: name]]`` as a paragraph of its own is replaced by the blocks between -``[[BLOCK: name]]`` and ``[[ENDBLOCK]]`` lines elsewhere in the same file: a diagram, a picture, code. +row the header row. A cell may start with ``{bg=RRGGBB valign=middle}`` for its colour and alignment. +``[[BLOCK: name]]`` as a paragraph of its own is replaced by the blocks between ``[[BLOCK: name]]`` and +``[[ENDBLOCK]]`` lines elsewhere in the same file or a file it includes: a diagram, a picture, code, a table. +A link or picture too long for its column becomes a reference, defined after the table. Each table becomes a pandoc grid table, which takes several blocks per cell, in a Div of class ``listtable``. The blocks go in a Div of class ``listtable-block`` each, which ``listtable.lua`` moves into their cells. @@ -37,6 +39,12 @@ _CELL_SEPARATOR = re.compile(r"(?", re.IGNORECASE) _ITEM = re.compile(r"^(?P-|\d+\.)\s+(?P.*)$") +_LINK = re.compile( # an inline link or picture: [text](url "title"){attributes} + r'(?P!?)\[(?P[^\]]*)\]\((?P[^)\s]+)(?:\s+"(?P[^"]*)")?\)(?P<attrs>\{[^}]*\})?' +) +_CELL_STYLE = re.compile(r"^\{(?P<body>[^}]*)\}\s*") # {bg=FFF2CC valign=middle} at the start of a cell +_COLOUR = re.compile(r"^#?(?P<hex>[0-9A-Fa-f]{6})$") +_VALIGN = ("top", "middle", "bottom") _MARKER_LIKE = re.compile(r"^(?:[-+*]|\d+[.)])$") # a word pandoc would read as a list item at the start of a line TOTAL_WIDTH = 100 # wider than pandoc's 72 columns, so pandoc keeps the relative widths MIN_WIDTH = 3 @@ -108,16 +116,82 @@ class _Cell: rows: int = 1 columns: int = 1 lines: list[str] = field(default_factory=list) + style: dict[str, str] = field(default_factory=dict) # bg and valign + + +@dataclass +class Shared: + """What a source file and the files it includes share: their blocks, and labels for references.""" + + scope: str # a short hash of the source file, so keys differ between the sources of a document + defined: dict[str, tuple[str, int]] = field(default_factory=dict) # block name: file and line + used: dict[str, tuple[str, int]] = field(default_factory=dict) + references: int = 0 + + @classmethod + def of(cls, file: str) -> Shared: + return cls(hashlib.sha1(file.encode("utf-8")).hexdigest()[:8]) + + def label(self) -> str: + self.references += 1 + return f"{self.scope}-{self.references}" + + def problems(self) -> list[tuple[str, int, str]]: + """Blocks used but not defined, or defined but not used: file, line and message.""" + found = [ + (file, line, f"[[BLOCK: {name}]] is used in a table, but neither this file nor its includes define it") + for name, (file, line) in self.used.items() + if name not in self.defined + ] + found += [ + (file, line, f"[[BLOCK: {name}]] is defined, but no table uses it") + for name, (file, line) in self.defined.items() + if name not in self.used + ] + return found + + +def _cell_style(text: str) -> tuple[dict[str, str], str, str | None]: + """The ``{bg=... valign=...}`` a cell starts with, the rest of the cell, and what is wrong with it.""" + match = _CELL_STYLE.match(text) + if not match or "=" not in match["body"]: + return {}, text, None # not a style: a cell may start with other {...} text + style: dict[str, str] = {} + for pair in match["body"].split(): + key, _, value = pair.partition("=") + colour = _COLOUR.match(value) + if key == "bg" and colour: + style["bg"] = colour["hex"].upper() + elif key == "valign" and value in _VALIGN: + style["valign"] = value + else: + return {}, text, f"'{pair}' is not bg=RRGGBB or valign=top, middle or bottom, so the cell keeps it as text" + return style, text[match.end() :], None class Table: """One ``[[LISTTABLE]]``: its marker, its rows as they are added, then its grid table.""" - def __init__(self, marker: str, line: int, settings: Options, scope: str) -> None: - self.marker, self.line, self.settings, self.scope = marker, line, settings, scope + def __init__(self, marker: str, line: int, settings: Options, shared: Shared) -> None: + self.marker, self.line, self.settings, self.shared = marker, line, settings, shared self.rows: list[tuple[int, str]] = [] self.problems: list[tuple[int, str]] = [] self.slots: dict[str, int] = {} # block name: the line of the first row that uses it + self.references: list[str] = [] # reference definitions for links too long for their column + self._cells: list[_Cell] = [] + + def _shorten_links(self, text: str, width: int) -> str: + """Links and pictures with a word wider than ``width`` become references, defined after the table.""" + + def shorten(match: re.Match) -> str: + if all(len(word) <= width for word in _words(match[0])): + return match[0] + label = self.shared.label() + title = f' "{match["title"]}"' if match["title"] is not None else "" + self.references.append(f"[{label}]: {match['url']}{title}{' ' + match['attrs'] if match['attrs'] else ''}") + return f"{match['bang']}[{match['text']}][{label}]" + + return _LINK.sub(shorten, text) def add(self, number: int, row: str) -> None: self.rows.append((number, row)) @@ -142,7 +216,8 @@ def _cell_lines(self, text: str, width: int, number: int) -> list[str]: del indents[level:] lead = " " * sum(indents) marker = item["marker"] + " " - wrapped = _wrap(item["text"], width - len(lead) - len(marker)) + room = width - len(lead) - len(marker) + wrapped = _wrap(self._shorten_links(item["text"], room), room) lines += [lead + marker + wrapped[0]] + [lead + " " * len(marker) + w for w in wrapped[1:]] indents.append(len(marker)) continue @@ -153,7 +228,7 @@ def _cell_lines(self, text: str, width: int, number: int) -> list[str]: if slot: self.slots.setdefault(slot["name"], number) paragraph = f"§{list(self.slots).index(slot['name']) + 1}" - lines += _wrap(paragraph, width) + lines += _wrap(self._shorten_links(paragraph, width), width) return lines def _layout(self, columns: int) -> list[_Cell]: @@ -185,7 +260,10 @@ def _layout(self, columns: int) -> list[_Cell]: owner[(r, c)] = owner[(r, c - 1) if merge == "<" else (r - 1, c)] grid[r].append(None) else: - owner[(r, c)] = _Cell(text, r, c) + style, text, wrong = _cell_style(text) + if wrong: + self.problems.append((number, wrong)) + owner[(r, c)] = _Cell(text, r, c, style=style) grid[r].append(owner[(r, c)]) regions: dict[int, list[tuple[int, int]]] = {} # the spots of each cell, by its id for spot, cell in owner.items(): @@ -212,7 +290,7 @@ def grid(self) -> list[str]: weights = self.settings.widths or [1] * len(_cells(self.rows[0][1])) widths = _scale(weights) columns, count = len(widths), len(self.rows) - cells = self._layout(columns) + cells = self._cells = self._layout(columns) xs = [sum(widths[:c]) + c for c in range(columns + 1)] # the | of each column boundary for cell in cells: width = xs[cell.column + cell.columns] - xs[cell.column] - 3 @@ -260,27 +338,38 @@ def finish(self) -> list[str]: self.problems.append((self.line, "[[LISTTABLE]] is left as written: no '| ... |' rows follow it")) return [self.marker] lines = self.grid() - keys = " ".join(f"{self.scope}-{name}" for name in self.slots) - div = f'::: {{.listtable blocks="{keys}"}}' if keys else "::: {.listtable}" - return ["", div, *lines, ":::", ""] + attributes = "" + if self.slots: + attributes += f' blocks="{" ".join(f"{self.shared.scope}-{name}" for name in self.slots)}"' + styles = [] # row.index:key=value,... where index counts the cells pandoc has in that row + for cell in self._cells: + if cell.style: + index = sum(1 for other in self._cells if other.row == cell.row and other.column < cell.column) + styles.append(f"{cell.row}.{index}:" + ",".join(f"{k}={v}" for k, v in sorted(cell.style.items()))) + if styles: + attributes += f' cells="{" ".join(styles)}"' + div = f"::: {{.listtable{attributes}}}" + definitions = ["", *self.references] if self.references else [] + return ["", div, *lines, ":::", *definitions, ""] class Tables: """The ``[[LISTTABLE]]`` tables and ``[[BLOCK:]]`` definitions of one file, fed to it line by line. + The blocks are kept in a :class:`Shared` with the other files of the same source, so a table can use a + block defined in a file it includes, or in the file that includes it. + ``problems`` are warnings: the table is still made. ``errors`` (a block never closed, an ``[[ENDBLOCK]]`` with no block) stop a build. """ - def __init__(self, file: str) -> None: - self.scope = hashlib.sha1(file.encode("utf-8")).hexdigest()[:8] + def __init__(self, file: str, shared: Shared) -> None: + self.file, self.shared = file, shared self.table: Table | None = None self.problems: list[tuple[int, str]] = [] self.errors: list[tuple[int, str]] = [] self.count = 0 self._open: tuple[int, str] | None = None - self._defined: dict[str, int] = {} - self._used: dict[str, int] = {} def flush(self) -> list[str]: """End the table being read, if there is one, and return its lines.""" @@ -292,7 +381,7 @@ def flush(self) -> list[str]: if table.rows: self.count += 1 for name, number in table.slots.items(): - self._used.setdefault(name, number) + self.shared.used.setdefault(name, (self.file, number)) return lines def directive(self, number: int, line: str) -> list[str] | None: @@ -304,17 +393,18 @@ def directive(self, number: int, line: str) -> list[str] | None: (number, "[[LISTTABLE]] is left as written: its options are not 'header' and 'widths=a,b,c'") ) return [line] - self.table = Table(line, number, settings, self.scope) + self.table = Table(line, number, settings, self.shared) return [] if match := BLOCK.match(line): if self._open: self.errors.append((number, f"[[BLOCK: {match['name']}]] is inside [[BLOCK: {self._open[1]}]]")) return [] - if match["name"] in self._defined: - self.errors.append((number, f"[[BLOCK: {match['name']}]] is defined twice in this file")) + if match["name"] in self.shared.defined: + where = ":".join(map(str, self.shared.defined[match["name"]])) + self.errors.append((number, f"[[BLOCK: {match['name']}]] is defined twice: it is also at {where}")) self._open = (number, match["name"]) - self._defined.setdefault(match["name"], number) - return ["", f'::: {{.listtable-block key="{self.scope}-{match["name"]}"}}', ""] + self.shared.defined.setdefault(match["name"], (self.file, number)) + return ["", f'::: {{.listtable-block key="{self.shared.scope}-{match["name"]}"}}', ""] if ENDBLOCK.match(line): if not self._open: self.errors.append((number, "[[ENDBLOCK]] has no [[BLOCK:]] before it")) @@ -324,16 +414,8 @@ def directive(self, number: int, line: str) -> list[str] | None: return None def close(self) -> list[str]: - """At the end of the file: the last table's lines, and the checks of the blocks.""" + """At the end of the file: the last table's lines. Whether each block is used is :meth:`Shared.problems`.""" lines = self.flush() if self._open: self.errors.append((self._open[0], f"[[BLOCK: {self._open[1]}]] is never closed with [[ENDBLOCK]]")) - for name, number in self._used.items(): - if name not in self._defined: - self.problems.append( - (number, f"[[BLOCK: {name}]] is used in a table, but this file does not define it") - ) - for name, number in self._defined.items(): - if name not in self._used: - self.problems.append((number, f"[[BLOCK: {name}]] is defined, but no table in this file uses it")) return lines diff --git a/src/py_doc/pandoc/docx_finish.py b/src/py_doc/pandoc/docx_finish.py index e304831..0319369 100644 --- a/src/py_doc/pandoc/docx_finish.py +++ b/src/py_doc/pandoc/docx_finish.py @@ -87,6 +87,7 @@ def finish_docx(body_path: Path, template: Path | None, metadata: Metadata, opti post.replace_placeholders(base, replacements) index = post.find_and_remove_marker(base, options.get("marker", DEFAULT_MARKER)) body = Document(str(body_path)) + post.mark_tables(body) # so the template's own tables keep their look if present: _drop_title_block(body) if index is not None: @@ -119,8 +120,10 @@ def finish_docx(body_path: Path, template: Path | None, metadata: Metadata, opti post.apply_layout_fixes(document) post.rescale_landscape_images(document) post.apply_explicit_styles(document) + post.apply_cell_marks(document) + own = post.take_rich_table_marks(document, post.OWN_TABLE_MARK) if template is not None else None rich = post.take_rich_table_marks(document) - post.format_all_tables(document, rich=rich) + post.format_all_tables(document, rich=rich, own=own) post.keep_captions_with_tables(document, rich) # after the [[CAPTION:]] lines are captions post.update_core_properties( document, diff --git a/src/py_doc/pandoc/docx_post.py b/src/py_doc/pandoc/docx_post.py index 99d04c2..e550208 100644 --- a/src/py_doc/pandoc/docx_post.py +++ b/src/py_doc/pandoc/docx_post.py @@ -23,12 +23,13 @@ from docx import Document from docx.enum.section import WD_ORIENT -from docx.enum.table import WD_TABLE_ALIGNMENT +from docx.enum.table import WD_CELL_VERTICAL_ALIGNMENT, WD_TABLE_ALIGNMENT from docx.enum.text import WD_ALIGN_PARAGRAPH -from docx.oxml import OxmlElement +from docx.oxml import OxmlElement, parse_xml from docx.oxml.ns import nsmap as _DOCX_NSMAP -from docx.oxml.ns import qn +from docx.oxml.ns import nsdecls, qn from docx.shape import InlineShape +from docx.table import Table from docx.shared import Cm, RGBColor from docx.text.paragraph import Paragraph @@ -513,23 +514,91 @@ def has_header_row(tbl) -> bool: return first is not None and first.find(f"{qn('w:trPr')}/{qn('w:tblHeader')}") is not None +_TCPR_ORDER = [ # the order the schema wants the children of <w:tcPr> in + "cnfStyle", "tcW", "gridSpan", "hMerge", "vMerge", "tcBorders", "shd", "noWrap", "tcMar", "textDirection", + "tcFitText", "vAlign", "hideMark", +] # fmt: skip + + +def _order_tcPr(tcPr) -> None: + """Put the children of a ``<w:tcPr>`` in schema order, after one was appended.""" + rank = {qn(f"w:{name}"): index for index, name in enumerate(_TCPR_ORDER)} + children = sorted(tcPr, key=lambda child: rank.get(child.tag, len(rank))) + for child in children: + tcPr.append(child) + + RICH_TABLE_MARK = "_PyDocRichTable" # the bookmark listtable.lua puts before a [[LISTTABLE]] _CELL_MARGINS = 2 * 68580 # Word's default left and right cell margins, 0.075 inch each, in EMU -def take_rich_table_marks(doc: Document) -> set: - """The ``<w:tbl>`` of each ``[[LISTTABLE]]``, found by the bookmark before it, which is taken out.""" - rich = set() - marks = [m for m in doc.element.body.iter(qn("w:bookmarkStart")) if m.get(qn("w:name")) == RICH_TABLE_MARK] +OWN_TABLE_MARK = "_PyDocTable" # the bookmark before each table of the Markdown, while it joins the template + + +def mark_tables(doc: Document) -> None: + """Put a hidden bookmark before each table of ``doc``, so they are told from the template's own.""" + for table in doc.element.body.findall(qn("w:tbl")): + table.addprevious(parse_xml(f'<w:bookmarkStart {nsdecls("w")} w:id="0" w:name="{OWN_TABLE_MARK}"/>')) + table.addprevious(parse_xml(f'<w:bookmarkEnd {nsdecls("w")} w:id="0"/>')) + + +def take_rich_table_marks(doc: Document, name: str = RICH_TABLE_MARK) -> set: + """The ``<w:tbl>`` after each bookmark called ``name``, which is taken out with its end. + + By default those of the ``[[LISTTABLE]]`` tables, which ``listtable.lua`` marks; with + :data:`OWN_TABLE_MARK` those :func:`mark_tables` marked. + """ + found = set() + marks = [m for m in doc.element.body.iter(qn("w:bookmarkStart")) if m.get(qn("w:name")) == name] for mark in marks: end = mark.getnext() if end is not None and end.tag == qn("w:bookmarkEnd"): # its own end: docxcompose renumbers only the start end.getparent().remove(end) table = mark.getnext() + while table is not None and table.tag in (qn("w:bookmarkStart"), qn("w:bookmarkEnd")): # another mark + table = table.getnext() if table is not None and table.tag == qn("w:tbl"): - rich.add(table) + found.add(table) + mark.getparent().remove(mark) + return found + + +CELL_MARK = "_PyDocCell_" # listtable.lua's bookmark in a styled cell: _PyDocCell_<RRGGBB or none>_<valign or none> + + +def _shade(tcPr, fill: str) -> None: + shd = tcPr.find(qn("w:shd")) + if shd is None: + shd = OxmlElement("w:shd") + tcPr.append(shd) + _order_tcPr(tcPr) + shd.set(qn("w:val"), "clear") + shd.set(qn("w:color"), "auto") + shd.set(qn("w:fill"), fill) + + +def apply_cell_marks(doc: Document) -> int: + """Give each cell with a ``[[LISTTABLE]]`` cell style its shading and vertical alignment. + + ``listtable.lua`` leaves a bookmark in the cell, named for the colour and the alignment, which is + taken out here. Returns how many cells were styled. + """ + marks = [m for m in doc.element.body.iter(qn("w:bookmarkStart")) if m.get(qn("w:name"), "").startswith(CELL_MARK)] + for mark in marks: + fill, _, valign = mark.get(qn("w:name"))[len(CELL_MARK) :].partition("_") + cell = next(mark.iterancestors(qn("w:tc")), None) + if cell is not None: + tcPr = cell.get_or_add_tcPr() + if fill != "none": + _shade(tcPr, fill) + if valign != "none": + tcPr.vAlign_val = {"top": WD_CELL_VERTICAL_ALIGNMENT.TOP, "center": WD_CELL_VERTICAL_ALIGNMENT.CENTER, + "bottom": WD_CELL_VERTICAL_ALIGNMENT.BOTTOM}[valign] # fmt: skip + end = mark.getnext() + if end is not None and end.tag == qn("w:bookmarkEnd"): + end.getparent().remove(end) mark.getparent().remove(mark) - return rich + return len(marks) def _text_width_at(doc: Document, element) -> int: @@ -550,27 +619,69 @@ def _text_width_at(doc: Document, element) -> int: return section.page_width - section.left_margin - section.right_margin -def _fit_pictures_to_cells(tbl, text_width: int) -> None: - """Make each picture in a cell of ``tbl`` (a fixed layout table) no wider than its cell.""" - columns = [int(c.get(qn("w:w"), "0")) for c in tbl._tbl.tblGrid.findall(qn("w:gridCol"))] - total = sum(columns) or 1 +def _cell_width(tc, table_width: int) -> int: + """The width (EMU) inside the cell ``tc`` of a table ``table_width`` wide: its share of the grid.""" + tbl = tc.getparent().getparent() + columns = [int(c.get(qn("w:w"), "0")) for c in tbl.findall(f"{qn('w:tblGrid')}/{qn('w:gridCol')}")] + first = sum(other.grid_span for other in tc.itersiblings(qn("w:tc"), preceding=True)) + share = sum(columns[first : first + tc.grid_span]) / (sum(columns) or 1) + return int(table_width * share) - _CELL_MARGINS + + +def _table_width(doc: Document, tbl) -> int: + """The width (EMU) a table ``<w:tbl>`` fills: the text width, or for a table in a cell that cell's.""" + cell = next(tbl.iterancestors(qn("w:tc")), None) + if cell is None: + return _text_width_at(doc, tbl) + return _cell_width(cell, _table_width(doc, cell.getparent().getparent())) + + +def _fit_pictures_to_cells(doc: Document, tbl) -> None: + """Make each picture in a cell of ``tbl`` no wider than its column's share of the table. + + In a fixed layout that is the cell's width. An autofit table may still widen a column, but the + picture then fits the column pandoc planned, not the whole page. + """ + width = _table_width(doc, tbl._tbl) for tr in tbl._tbl.findall(qn("w:tr")): - column = 0 for tc in tr.findall(qn("w:tc")): - room = int(text_width * sum(columns[column : column + tc.grid_span]) / total) - _CELL_MARGINS + room = _cell_width(tc, width) for inline in tc.iter(qn("wp:inline")): picture = InlineShape(inline) if room > 0 and picture.width and picture.width > room: picture.height = int(picture.height * room / picture.width) picture.width = room - column += tc.grid_span -def _format_header_row(row) -> None: - """Row 1 as the header: repeated on each page, kept with the next row, in bold white.""" +HEADER_FILL = "1F3864" # navy, behind the white header text when the table style has no header colour + + +def _header_fill(style) -> str: + """The header row colour of a table style (or the styles it is based on), else :data:`HEADER_FILL`.""" + for _ in range(5): + if style is None: + break + fill = style.element.xpath("w:tblStylePr[@w:type='firstRow']/w:tcPr/w:shd/@w:fill") + if fill and fill[0].lower() != "auto": + return fill[0] + style = style.base_style + return HEADER_FILL + + +def _format_header_row(row, fill: str) -> None: + """Row 1 as the header: repeated on each page, kept with the next row, in bold white on ``fill``. + + The colour is set on each cell as well as the text: a table style's own header colour is not shown by + every program (LibreOffice and Google Docs skip it), and py-doc's ``Table Grid`` has none, so white text + alone could be invisible. + """ trPr = row._tr.get_or_add_trPr() if trPr.find(qn("w:tblHeader")) is None: trPr.append(OxmlElement("w:tblHeader")) + for tc in row._tr.findall(qn("w:tc")): + tcPr = tc.get_or_add_tcPr() + if tcPr.find(qn("w:shd")) is None: # a colour the cell was given itself stays + _shade(tcPr, fill) for cell in row.cells: for p in cell.paragraphs: p.paragraph_format.keep_with_next = True @@ -594,6 +705,7 @@ def format_all_tables( prevent_row_split: bool = True, autofit: bool = True, rich: set | frozenset = frozenset(), + own: set | None = None, ) -> None: """ Apply a consistent appearance to every table in *doc*. @@ -608,23 +720,35 @@ def format_all_tables( tables don't end up a mix of auto-width/left-aligned and full-width/left-aligned depending on how Pandoc happened to emit them. * Enables autofit and removes fixed cell widths. - * Marks the first row as a repeating header. + * Marks the first row as a repeating header, in bold white on the table style's header colour (or + navy), set on the cells themselves. * Prevents rows from splitting across pages. + * Makes each picture in a cell no wider than its column. + * Does the same for tables inside table cells. + + With ``own``, only those tables (the Markdown's, see :func:`mark_tables`) and the tables inside them + are formatted: a template's own tables keep the look the template gives them. A ``[[LISTTABLE]]`` (its ``<w:tbl>`` is in ``rich``, see :func:`take_rich_table_marks`) and a table without a header row (see :func:`has_header_row`) keep the column widths pandoc wrote, in a fixed layout, and their rows may break across pages, so a tall row does not leave a gap by moving whole to - the next page. A picture in one of their cells is made no wider than its cell. Only a header row, if - there is one, is formatted and repeated as a header, and kept on one page. + the next page. Only a header row, if there is one, is formatted and repeated as a header, and kept on + one page. """ try: - _ = doc.styles[style_name] + fill = _header_fill(doc.styles[style_name]) style_ok = True except KeyError: log.warning("Table style '%s' not found in document — skipping", style_name) style_ok = False + fill = HEADER_FILL - for tbl in doc.tables: + elements = [ + element + for element in doc.element.body.iter(qn("w:tbl")) # nested ones too + if own is None or element in own or any(outer in own for outer in element.iterancestors(qn("w:tbl"))) + ] + for tbl in [Table(element, doc._body) for element in elements]: if style_ok: tbl.style = style_name @@ -641,11 +765,11 @@ def format_all_tables( if tbl._tbl in rich or not header: tbl.autofit = False # a fixed layout, on the column widths pandoc wrote if header and repeat_header and tbl.rows: - _format_header_row(tbl.rows[0]) + _format_header_row(tbl.rows[0], fill) trPr = tbl.rows[0]._tr.get_or_add_trPr() if trPr.find(qn("w:cantSplit")) is None: trPr.append(OxmlElement("w:cantSplit")) - _fit_pictures_to_cells(tbl, _text_width_at(doc, tbl._tbl)) + _fit_pictures_to_cells(doc, tbl) continue if autofit: @@ -661,7 +785,8 @@ def format_all_tables( tcPr.remove(tcW) if repeat_header and tbl.rows: - _format_header_row(tbl.rows[0]) + _format_header_row(tbl.rows[0], fill) + _fit_pictures_to_cells(doc, tbl) if prevent_row_split: for row in tbl.rows: diff --git a/src/py_doc/pandoc/resources/latex_template.tex b/src/py_doc/pandoc/resources/latex_template.tex index c0a427f..3e37728 100644 --- a/src/py_doc/pandoc/resources/latex_template.tex +++ b/src/py_doc/pandoc/resources/latex_template.tex @@ -42,6 +42,7 @@ \usepackage{adjustbox} \usepackage{longtable,booktabs,array,calc,multirow} % multirow: cells merged down, in [[LISTTABLE]] tables \usepackage{xcolor} +\usepackage{colortbl} % \cellcolor: cell colours, in [[LISTTABLE]] tables \usepackage{amsmath} % \text, align and the other commands pandoc writes for maths \usepackage{amssymb} % \boxtimes and \square, which pandoc writes for task lists % pandoc writes ~~struck out~~ as \st{} or \sout{}, and underlined text as \ul{} or \uline{} diff --git a/src/py_doc/pandoc/resources/listtable.lua b/src/py_doc/pandoc/resources/listtable.lua index 45b9681..19b36f9 100644 --- a/src/py_doc/pandoc/resources/listtable.lua +++ b/src/py_doc/pandoc/resources/listtable.lua @@ -3,10 +3,17 @@ -- A cell paragraph §N becomes the blocks of the Nth key in the Div's "blocks" attribute: a [[BLOCK:]] -- that py-doc wrote as a Div of class "listtable-block" elsewhere in the document, and which is taken out -- of the place it was written. A picture or diagram in a cell is not a figure: it has no caption or number --- of its own. The table then leaves its Div. In Word a hidden bookmark goes before it, --- so the Word finishing knows a rich table (its rows may break across pages) from an ordinary one. +-- of its own. A block may hold another [[LISTTABLE]], which is placed the same way. +-- +-- The Div's "cells" attribute gives the colour and vertical alignment of some cells, as +-- "row.index:bg=RRGGBB,valign=middle", where index counts the cells pandoc has in that row. HTML gets +-- them as a style, PDF the colour as \cellcolor, and Word a hidden bookmark in the cell that py-doc's Word +-- finishing turns into the cell's shading and alignment. The table then leaves its Div. In Word a hidden +-- bookmark before it also tells the finishing that it is a rich table (its rows may break across pages). +-- In PDF a table inside a cell is written as a tabular, as LaTeX cannot nest the longtables pandoc writes. local blocks = {} +local VALIGN = { top = "top", middle = "center", bottom = "bottom" } local function collect(div) if div.classes:includes("listtable-block") then @@ -15,7 +22,94 @@ local function collect(div) end end -local function place(div) +local function bookmark(name) + return pandoc.RawBlock("openxml", '<w:bookmarkStart w:id="0" w:name="' .. name .. '"/><w:bookmarkEnd w:id="0"/>') +end + +local function style_cell(cell, spec) + local bg = spec:match("bg=(%x+)") + local valign = spec:match("valign=(%a+)") + if FORMAT:match("html") or FORMAT == "gfm" or FORMAT == "markdown" then + local css = {} + if bg then css[#css + 1] = "background-color: #" .. bg end + if valign then css[#css + 1] = "vertical-align: " .. valign end + cell.attributes.style = table.concat(css, "; ") + elseif FORMAT == "docx" then + cell.contents:insert(1, bookmark("_PyDocCell_" .. (bg or "none") .. "_" .. (VALIGN[valign] or "none"))) + elseif FORMAT == "latex" and bg then + local colour = pandoc.RawInline("latex", "\\cellcolor[HTML]{" .. bg .. "}") + local first = cell.contents[1] + if first and (first.t == "Plain" or first.t == "Para") then + first.content:insert(1, colour) + else + cell.contents:insert(1, pandoc.Plain({ colour })) + end + end +end + +local function rows_of(tbl) + local rows = pandoc.List() + rows:extend(tbl.head.rows) + for _, body in ipairs(tbl.bodies) do + rows:extend(body.body) + end + return rows +end + +-- LaTeX cannot put a longtable, which pandoc writes for every table, inside another one. A table in a +-- cell becomes a tabular instead: the same columns and rules, without page breaks. +local function as_tabular(tbl) + local latex = pandoc.write(pandoc.Pandoc({ tbl }), "latex") + local head, foot, body = latex:match("^(.*)\\endhead\n(.-)\\endlastfoot\n(.*)$") + if head then -- a longtable gives its foot before its body: a tabular wants it last + local last = body:find("\\end{longtable}", 1, true) + latex = head .. body:sub(1, last - 1) .. foot .. body:sub(last) + end + latex = latex:gsub("\\begin{longtable}%[%]", "\\begin{tabular}"):gsub("\\end{longtable}", "\\end{tabular}") + return pandoc.RawBlock("latex", latex) +end + +local function nested_as_tabular(tbl) + for _, row in ipairs(rows_of(tbl)) do + for _, cell in ipairs(row.cells) do + cell.contents = pandoc.Blocks(cell.contents):walk({ Table = as_tabular }) + end + end + return tbl +end + +local function style_cells(tbl, specs) + local wanted = {} + for row, index, spec in specs:gmatch("(%d+)%.(%d+):(%S+)") do + wanted[row .. "." .. index] = spec + end + for r, row in ipairs(rows_of(tbl)) do + for i, cell in ipairs(row.cells) do + local spec = wanted[(r - 1) .. "." .. (i - 1)] + if spec then style_cell(cell, spec) end + end + end + return tbl +end + +local place + +local function fill(list, keys) + local out = pandoc.List() + for _, block in ipairs(list) do + local slot = (block.t == "Plain" or block.t == "Para") and #block.content == 1 + and block.content[1].t == "Str" and block.content[1].text:match("^§(%d+)$") + local found = slot and blocks[keys[tonumber(slot)]] + if found then + out:extend(pandoc.Blocks(found):walk({ Div = place, Figure = function(figure) return figure.content end })) + else + out:insert(block) + end + end + return out +end + +place = function(div) if not div.classes:includes("listtable") then return nil end @@ -23,25 +117,16 @@ local function place(div) for key in (div.attributes.blocks or ""):gmatch("%S+") do keys[#keys + 1] = key end - local filled = div:walk({ - Blocks = function(list) - local out = pandoc.List() - for _, block in ipairs(list) do - local slot = (block.t == "Plain" or block.t == "Para") and #block.content == 1 - and block.content[1].t == "Str" and block.content[1].text:match("^§(%d+)$") - local found = slot and blocks[keys[tonumber(slot)]] - if found then - out:extend(pandoc.Blocks(found):walk({ Figure = function(figure) return figure.content end })) - else - out:insert(block) - end - end - return out - end, - }) + local filled = div:walk({ Blocks = function(list) return fill(list, keys) end }) + if div.attributes.cells then + filled = filled:walk({ Table = function(tbl) return style_cells(tbl, div.attributes.cells) end }) + end + if FORMAT == "latex" then + filled = filled:walk({ Table = nested_as_tabular }) + end local out = pandoc.List() if FORMAT == "docx" then - out:insert(pandoc.RawBlock("openxml", '<w:bookmarkStart w:id="0" w:name="_PyDocRichTable"/><w:bookmarkEnd w:id="0"/>')) + out:insert(bookmark("_PyDocRichTable")) end out:extend(filled.content) return out diff --git a/tests/test_examples.py b/tests/test_examples.py index c461fca..d00ea87 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -218,6 +218,12 @@ def test_the_guide_shows_the_word_features(examples, pandoc_runtime): assert 'w:rStyle w:val="HighlightYellow"' in rich._tbl.xml assert rich._tbl.xpath(".//w:vMerge") and rich._tbl.xpath(".//w:gridSpan[@w:val='2']"), "merged cells" assert rich._tbl.findall(qn("w:tr"))[3].xpath(".//wp:inline"), "the diagram is in its cell" + assert rich._tbl.xpath(".//w:tcPr/w:shd[@w:fill='E8F5E9']") and rich._tbl.xpath(".//w:vAlign[@w:val='center']") + assert rich._tbl.findall(qn("w:tr"))[4].xpath(".//w:tbl"), "a table in a cell" + assert ( + "terminal-wiring-sheet.pdf" + in document.part.rels[rich._tbl.findall(qn("w:tr"))[4].xpath(".//w:hyperlink/@r:id")[0]].target_ref + ), "a link too long for its column" with zipfile.ZipFile(examples / "build" / "guide-v1.0.docx") as archive: core = archive.read("docProps/core.xml").decode("utf-8") assert "Integrating the SN-100" in core and "Technical documentation" in core and "IoT, sensor, MQTT" in core diff --git a/tests/test_listtable.py b/tests/test_listtable.py index 94b1a77..2a2454f 100644 --- a/tests/test_listtable.py +++ b/tests/test_listtable.py @@ -11,7 +11,8 @@ import pytest from docx import Document -from docx.oxml.ns import qn +from docx.oxml import parse_xml +from docx.oxml.ns import nsdecls, qn from docx.shared import Cm from PIL import Image @@ -184,8 +185,11 @@ def test_a_block_is_a_slot_in_its_cell_and_a_div_of_its_own(self): def test_a_block_used_but_not_defined_or_defined_but_not_used_is_a_warning(self, caplog): with caplog.at_level(logging.WARNING): convert("[[LISTTABLE]]\n| a | [[BLOCK: gone]] |\n\n[[BLOCK: spare]]\nx\n[[ENDBLOCK]]\n") - assert "t.md:2: [[BLOCK: gone]] is used in a table, but this file does not define it" in caplog.text - assert "t.md:4: [[BLOCK: spare]] is defined, but no table in this file uses it" in caplog.text + assert ( + "t.md:2: [[BLOCK: gone]] is used in a table, but neither this file nor its includes define it" + in caplog.text + ) + assert "t.md:4: [[BLOCK: spare]] is defined, but no table uses it" in caplog.text def test_an_unclosed_block_or_a_stray_end_stops_the_build(self): with pytest.raises(DirectiveError, match="t.md:1: \\[\\[BLOCK: a\\]\\] is never closed"): @@ -279,7 +283,9 @@ def test_lint_reads_each_cell_paragraph_as_prose_of_its_own(): from py_doc.lint.text import segments from py_doc.markdown import parse - found = [s.text.strip() for s in segments(parse("[[LISTTABLE]]\n| 1 | One.<br>- Two |\n| 2 | Three |\n"))] + found = [ + s.text.strip() for s in segments(parse("[[LISTTABLE]]\n| 1 | One.<br>- Two |\n| 2 | {bg=E8F5E9} Three |\n")) + ] assert found == ["1", "One.", "- Two", "2", "Three"] @@ -346,3 +352,176 @@ def format_of(paragraph): table = document.tables[-1] # after the template's own assert [format_of(p) for p in table.cell(0, 1).paragraphs] == ["bullet", "bullet"] assert [format_of(p) for p in table.cell(1, 1).paragraphs] == ["decimal", "decimal"] + + +class TestLongLinks: + LINK = "https://example.com/a/very/long/path/that/does/not/fit/in/the/column/at/all" + + def test_a_link_too_long_for_its_column_becomes_a_reference_after_the_table(self): + out = convert(f'[[LISTTABLE]]\n| a | See [the page]({self.LINK} "Title") and ![x](short.png){{width=50%}} |\n') + label = re.search(r"\[the page\]\[(\w+-1)\]", out)[1] + assert f'\n[{label}]: {self.LINK} "Title"\n' in out + assert "![x](short.png){width=50%}" in out, "a link that fits is left as written" + assert len({len(line) for line in grid(out.replace("\n\n", "\n"))}) == 1 + + def test_labels_differ_between_tables_and_files(self, tmp_path): + (tmp_path / "part.md").write_text(f"[[LISTTABLE]]\n| a | [x]({self.LINK}) |\n", encoding="utf-8") + main = tmp_path / "main.md" + main.write_text(f"[[LISTTABLE]]\n| a | [x]({self.LINK}) |\n\n[[INCLUDE: part.md]]\n", encoding="utf-8") + out = expand(main.read_text(encoding="utf-8"), main, tables=True) + assert len(set(re.findall(r"^\[(\w+-\d+)\]: ", out, re.M))) == 2 + + @needs_pandoc + def test_pandoc_links_the_reference(self): + html = subprocess.run( + ["pandoc", "-f", "markdown", "-t", "html"], input=convert(f"[[LISTTABLE]]\n| a | [x]({self.LINK}) |\n"), + capture_output=True, text=True, check=True, + ).stdout # fmt: skip + assert re.search(rf'<a\s+href="{re.escape(self.LINK)}">x</a>', html) + + +class TestBlocksAcrossIncludes: + def test_a_table_may_use_a_block_of_a_file_it_includes_or_of_the_file_that_includes_it(self, tmp_path, caplog): + (tmp_path / "part.md").write_text( + "[[BLOCK: inner]]\nIn the part.\n[[ENDBLOCK]]\n\n[[LISTTABLE]]\n| b | [[BLOCK: outer]] |\n", + encoding="utf-8", + ) + main = tmp_path / "main.md" + main.write_text( + "[[LISTTABLE]]\n| a | [[BLOCK: inner]] |\n\n[[INCLUDE: part.md]]\n\n[[BLOCK: outer]]\nIn main.\n[[ENDBLOCK]]\n", + encoding="utf-8", + ) + with caplog.at_level(logging.WARNING): + out = expand(main.read_text(encoding="utf-8"), main, tables=True) + assert "[[BLOCK" not in caplog.text + keys = re.findall(r'blocks="(\S+)"', out) + assert {key.split("-", 1)[1] for key in keys} == {"inner", "outer"} + assert len({key.split("-", 1)[0] for key in keys}) == 1, "one scope for the source and its includes" + + def test_a_block_defined_twice_across_files_stops_the_build(self, tmp_path): + (tmp_path / "part.md").write_text("[[BLOCK: a]]\nx\n[[ENDBLOCK]]\n", encoding="utf-8") + main = tmp_path / "main.md" + main.write_text("[[BLOCK: a]]\ny\n[[ENDBLOCK]]\n\n[[INCLUDE: part.md]]\n", encoding="utf-8") + with pytest.raises( + DirectiveError, match=r"part.md:1: \[\[BLOCK: a\]\] is defined twice: it is also at .*main.md:1" + ): + expand(main.read_text(encoding="utf-8"), main, tables=True) + + def test_check_looks_in_the_included_files(self, tmp_path): + (tmp_path / "part.md").write_text("[[BLOCK: inner]]\nx\n[[ENDBLOCK]]\n", encoding="utf-8") + main = tmp_path / "main.md" + main.write_text("[[LISTTABLE]]\n| a | [[BLOCK: inner]] |\n| b | [[BLOCK: gone]] |\n\n[[INCLUDE: part.md]]\n") + assert [(p.line, p.message[:20]) for p in check(main.read_text(), main)] == [(3, "[[BLOCK: gone]] is u")] + + +class TestCellStyles: + def test_a_cell_style_is_taken_off_the_text_and_given_by_position(self): + out = convert( + "[[LISTTABLE: header]]\n| A | B |\n| {bg=#fff2cc valign=middle} a | < |\n| b | {valign=top} c |\n" + ) + assert 'cells="1.0:bg=FFF2CC,valign=middle 2.1:valign=top"' in out + assert "{bg" not in "".join(grid(out)) and "| a " in out + + def test_a_cell_style_counts_only_the_cells_pandoc_has_in_the_row(self): + out = convert("[[LISTTABLE]]\n| a | b |\n| ^ | {bg=000000} c |\n") + assert 'cells="1.0:bg=000000"' in out # the merged cell is not in row 1 of pandoc's table + + def test_a_cell_style_that_is_not_understood_stays_as_text_with_a_warning(self, caplog): + with caplog.at_level(logging.WARNING): + out = convert("[[LISTTABLE]]\n| {bg=yellow} a | {.class} b |\n") + assert "'bg=yellow' is not bg=RRGGBB or valign=top, middle or bottom" in caplog.text + assert "{bg=yellow} a" in out and "{.class} b" in out and "cells=" not in out + + @needs_pandoc + def test_html_has_the_style_and_pdf_the_colour_and_a_nested_tabular(self): + text = convert( + "[[LISTTABLE]]\n| {bg=FFF2CC valign=middle} a | [[BLOCK: inner]] |\n\n" + "[[BLOCK: inner]]\n| P | Q |\n|---|---|\n| p | q |\n[[ENDBLOCK]]\n" + ) + + def to(fmt): + return subprocess.run( + ["pandoc", "-f", "markdown", "-t", fmt, "--lua-filter", str(LUA)], input=text, capture_output=True, + text=True, check=True, + ).stdout # fmt: skip + + assert '<td style="background-color: #FFF2CC; vertical-align: middle">a</td>' in to("html") + latex = to("latex") + assert "\\cellcolor[HTML]{FFF2CC}a" in latex + assert latex.count("\\begin{longtable}") == 1 and "\\begin{tabular}" in latex, "LaTeX cannot nest longtables" + assert latex.index("p & q") < latex.index("\\bottomrule\\noalign{}\n\\end{tabular}") + + +NESTED = """# T + +[[LISTTABLE: header]] +| Part | Detail | +| {bg=FFF2CC valign=middle} Board | [[BLOCK: pins]] | + +[[BLOCK: pins]] +| Pin | Use | +|---|---| +| 1 | Power | +[[ENDBLOCK]] + +| Name | Picture | +|---|---| +| a | ![](wide.png) | +""" + + +@pytest.fixture +def nested_docx(tmp_path, convert_text): + Image.new("RGB", (2000, 800), "red").save(tmp_path / "wide.png") + template = Path(__file__).parent.parent / "examples" / "templates" / "sensor.docx" + return Document(str(convert_text(NESTED, template))) + + +def _cells(table) -> list: + return [tc for tr in table.findall(qn("w:tr")) for tc in tr.findall(qn("w:tc"))] + + +def test_a_table_in_a_block_is_a_table_in_its_cell_and_formatted_like_one(nested_docx): + outer = next(t for t in nested_docx.tables if t.cell(0, 0).text == "Part")._tbl + (inner,) = _cells(outer)[3].findall(qn("w:tbl")) + assert inner.tblPr.find(qn("w:tblStyle")) is not None + assert inner.xpath("w:tr[1]/w:trPr/w:tblHeader") and inner.xpath("w:tr[1]//w:color[@w:val='FFFFFF']") + + +def test_a_cell_style_gives_word_shading_and_alignment(nested_docx): + outer = next(t for t in nested_docx.tables if t.cell(0, 0).text == "Part")._tbl + board = _cells(outer)[2] + assert board.xpath("w:tcPr/w:shd/@w:fill") == ["FFF2CC"] and board.xpath("w:tcPr/w:vAlign/@w:val") == ["center"] + assert not nested_docx.element.body.xpath('.//w:bookmarkStart[starts-with(@w:name, "_PyDoc")]') + + +def test_header_cells_are_shaded_so_their_white_text_shows(nested_docx): + pins = next(t for t in nested_docx.tables if t.cell(0, 0).text == "Name")._tbl + assert pins.xpath("w:tr[1]/w:tc/w:tcPr/w:shd/@w:fill") == ["1F3864", "1F3864"] + + +def test_the_templates_own_tables_keep_their_look(nested_docx): + facts = next(t for t in nested_docx.tables if t.cell(0, 0).text == "Document number")._tbl + assert not facts.xpath(".//w:tblHeader") and not facts.xpath(".//w:shd") and not facts.xpath(".//w:cantSplit") + assert not facts.xpath(".//w:color[@w:val='FFFFFF']") + + +def test_a_picture_in_an_ordinary_table_fits_its_column(nested_docx): + pins = next(t for t in nested_docx.tables if t.cell(0, 0).text == "Name")._tbl + (picture,) = pins.xpath(".//wp:inline/wp:extent/@cx") + section = nested_docx.sections[0] + assert int(picture) < (section.page_width - section.left_margin - section.right_margin) * 0.75 + + +def test_a_header_takes_the_colour_of_its_table_style(): + from py_doc.pandoc.docx_post import HEADER_FILL, _header_fill + + style = Document().styles["Table Grid"] + assert _header_fill(style) == HEADER_FILL, "py-doc's own Table Grid has no header colour" + style.element.append( + parse_xml( + f'<w:tblStylePr {nsdecls("w")} w:type="firstRow"><w:tcPr>' + '<w:shd w:val="clear" w:color="auto" w:fill="C00000"/></w:tcPr></w:tblStylePr>' + ) + ) + assert _header_fill(style) == "C00000" diff --git a/tests/test_site.py b/tests/test_site.py index 8efa33b..9f9b99f 100644 --- a/tests/test_site.py +++ b/tests/test_site.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re import shutil import pytest @@ -589,3 +590,33 @@ def test_site_decks_must_be_decks(tmp_path): (tmp_path / "py-doc.yml").write_text("site: {engine: mkdocs, decks: [nope]}\n", encoding="utf-8") with pytest.raises(ConfigError, match="site.decks names 'nope', which is not under documents"): load_config(tmp_path / "py-doc.yml") + + +RICH_TABLE = """ +[[LISTTABLE: header]] +| Part | Notes | +| Board | {bg=FFF2CC} Fitted first<br>- then wired | +| ^ | [[BLOCK: board]] | +| Spare | < | + +[[BLOCK: board]] +![The board](images/board.png) +[[ENDBLOCK]] +""" + + +@needs_sh +@pytest.mark.parametrize("engine_name", ["mkdocs", "starlight"]) +def test_a_rich_table_keeps_its_merges_colours_lists_and_picture_on_a_site(project, pandoc_runtime, engine_name): + overview = project / "docs" / "01_overview.md" + overview.write_text(overview.read_text(encoding="utf-8") + RICH_TABLE, encoding="utf-8") + config, runtime = _site(project, engine_name, pandoc_runtime) + generate_site(config, runtime) + folder = project / "site" / ("docs" if engine_name == "mkdocs" else "src/content/docs") / "guide" + page = (folder / "01-overview.md").read_text(encoding="utf-8") + table = page[page.index("<table") : page.index("</table>")] + assert 'rowspan="2"' in table and 'colspan="2"' in table + assert "background-color: #FFF2CC" in table and "<li>then wired</li>" in table + (picture,) = [p for p in (folder / "media").iterdir() if p.name.endswith("-board.png")] + assert re.search(rf'<img src="[^"]*{re.escape(picture.name)}"', table), "the picture is in its cell" + assert "§" not in page and "listtable" not in page