From d761a88a9491c076b42f00fbe72653b3496dae8d Mon Sep 17 00:00:00 2001 From: Anna Petrasova Date: Tue, 22 Sep 2026 23:15:11 -0400 Subject: [PATCH] docs: Make edit links and sitemap independent of the site generator Write the GitHub edit URL of each page into its front matter in mkmarkdown.py and read it in the actions template, replacing the MkDocs hook which scraped it from the rendered pages. Add generate_sitemap.py which lists every built page, and run it in the documentation workflow so pages outside the navigation stay in the sitemap. Written with Claude Code. --- .github/workflows/documentation.yml | 9 +++ man/CMakeLists.txt | 7 +- man/Makefile | 7 -- man/mkdocs/mkdocs.yml | 4 - man/mkdocs/overrides/partials/actions.html | 44 +++++----- man/mkdocs/scripts/hook_list_scripts.py | 83 ------------------- utils/generate_sitemap.py | 82 +++++++++++++++++++ utils/mkmarkdown.py | 29 ++++++- utils/tests/generate_sitemap_test.py | 93 ++++++++++++++++++++++ utils/tests/mkmarkdown_test.py | 68 ++++++++++++++++ 10 files changed, 298 insertions(+), 128 deletions(-) delete mode 100644 man/mkdocs/scripts/hook_list_scripts.py create mode 100755 utils/generate_sitemap.py create mode 100644 utils/tests/generate_sitemap_test.py create mode 100644 utils/tests/mkmarkdown_test.py diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index d4413ade598..ae43acbf464 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -238,6 +238,15 @@ jobs: cd "${MKDOCS_DIR}" || exit mkdocs build + - name: Generate manuals sitemap + # List every built page, not only those in the site navigation, so + # the tool pages stay in the sitemap regardless of the site generator. + # Runs before libpython is moved in; Sphinx provides that sitemap. + run: | + python grass/utils/generate_sitemap.py \ + --site-dir "${MKDOCS_DIR}/site" \ + --site-url https://grass.osgeo.org/grass-stable/manuals/ + - name: Build Sphinx documentation run: | pip install -r "grass/python/grass/docs/requirements.txt" diff --git a/man/CMakeLists.txt b/man/CMakeLists.txt index bc962806516..a6a6d75a882 100644 --- a/man/CMakeLists.txt +++ b/man/CMakeLists.txt @@ -131,8 +131,7 @@ if(NOT WITH_FHS) add_custom_target( copy_mkdocs_files COMMAND - ${CMAKE_COMMAND} -E make_directory ${mkdocs_dir}/scripts - ${mkdocs_dir}/overrides/partials + ${CMAKE_COMMAND} -E make_directory ${mkdocs_dir}/overrides/partials ${mkdocs_dir}/overrides/fragments/tags/default COMMAND ${CMAKE_COMMAND} -E copy_if_different @@ -147,10 +146,6 @@ if(NOT WITH_FHS) ${CMAKE_CURRENT_SOURCE_DIR}/grass_logo.svg ${CMAKE_CURRENT_SOURCE_DIR}/grass_icon.png ${CMAKE_CURRENT_SOURCE_DIR}/favicon.ico ${mkdocs_dir}/source - COMMAND - ${CMAKE_COMMAND} -E copy_if_different - ${CMAKE_CURRENT_SOURCE_DIR}/mkdocs/scripts/hook_list_scripts.py - ${mkdocs_dir}/scripts COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/mkdocs/overrides/partials/copyright.html diff --git a/man/Makefile b/man/Makefile index 3f71bf7cd99..9c4735a780e 100644 --- a/man/Makefile +++ b/man/Makefile @@ -29,7 +29,6 @@ DSTFILES := \ $(MDDIR)/source/grass_icon.png \ $(MDDIR)/source/favicon.ico \ $(MDDIR)/source/grassdocs.css \ - $(MDDIR)/scripts/hook_list_scripts.py \ $(MDDIR)/source/index.md \ $(MDDIR)/overrides/main.html \ $(MDDIR)/overrides/partials/copyright.html \ @@ -281,12 +280,6 @@ $(MDDIR)/source/development_intro.md: mkdocs/docs/development_intro.md $(MDDIR)/source/command_line_intro.md: mkdocs/docs/command_line_intro.md $(INSTALL_DATA) $< $@ -$(MDDIR)/scripts/hook_list_scripts.py: mkdocs/scripts/hook_list_scripts.py | $(MDDIR)/scripts - $(INSTALL_DATA) $< $@ - -$(MDDIR)/scripts: - $(MKDIR) $@ - $(MDDIR)/overrides/main.html: mkdocs/overrides/main.html | $(MDDIR)/overrides $(INSTALL_DATA) $< $@ diff --git a/man/mkdocs/mkdocs.yml b/man/mkdocs/mkdocs.yml index 3b12a567a6a..3df1775290a 100644 --- a/man/mkdocs/mkdocs.yml +++ b/man/mkdocs/mkdocs.yml @@ -75,10 +75,6 @@ extra: grass: grass grass_addons: grass-addons -# Hooks -hooks: - - scripts/hook_list_scripts.py - # Custom CSS extra_css: - grassdocs.css diff --git a/man/mkdocs/overrides/partials/actions.html b/man/mkdocs/overrides/partials/actions.html index 3256f22728b..82ab35c1d88 100644 --- a/man/mkdocs/overrides/partials/actions.html +++ b/man/mkdocs/overrides/partials/actions.html @@ -1,27 +1,21 @@ -{% if page.edit_url %} - - {% set toolname = page.file.src_path.split("/")[-1] | replace(".md", "") %} - {# Default to the normal edit URL #} - {% set ghpath = github_path(toolname, config.extra.source_docs) %} - {% if ghpath %} - {% set custom_edit_url = page.edit_url | replace(page.edit_url, ghpath ~ "/" ~ toolname ~ ".md") %} - {% if "content.action.edit" in features %} - - {% set icon = config.theme.icon.edit or "material/file-edit-outline" %} - {% include ".icons/" ~ icon ~ ".svg" %} - - {% endif %} - - {% if "content.action.view" in features %} - {% if "/blob/" in custom_edit_url %} - {% set part = "/blob/" %} - {% else %} - {% set part = "/edit/" %} - {% endif %} - - {% set icon = config.theme.icon.view or "material/file-eye-outline" %} - {% include ".icons/" ~ icon ~ ".svg" %} - - {% endif %} +{# + The generated pages carry the URL of their Markdown source in the front + matter (source_edit_url, written by utils/mkmarkdown.py), so the edit and + view buttons can point to the tool directory in the repository rather than + to the generated documentation tree, without any Python in the site build. +#} +{% if page.meta and page.meta.source_edit_url %} + {% set edit_url = page.meta.source_edit_url %} + {% if "content.action.edit" in features %} + + {% set icon = config.theme.icon.edit or "material/file-edit-outline" %} + {% include ".icons/" ~ icon ~ ".svg" %} + + {% endif %} + {% if "content.action.view" in features %} + + {% set icon = config.theme.icon.view or "material/file-eye-outline" %} + {% include ".icons/" ~ icon ~ ".svg" %} + {% endif %} {% endif %} diff --git a/man/mkdocs/scripts/hook_list_scripts.py b/man/mkdocs/scripts/hook_list_scripts.py deleted file mode 100644 index f815e5bdd2b..00000000000 --- a/man/mkdocs/scripts/hook_list_scripts.py +++ /dev/null @@ -1,83 +0,0 @@ -import re -from pathlib import Path - -from jinja2 import Environment - - -def github_path(toolname: str, source_docs: dict[str, str]): - """Return the GitHub path for the given toolname""" - try: - return source_docs[toolname] - except KeyError: - return None - - -def github_edit_path(ghpath: str, base_repo_url: str): - """Return the GitHub edit path for the given path""" - path_parts = ghpath.split("/") - - # Gracefully handle invalid paths - if len(path_parts) < 3: - return None - repo = path_parts[0].strip() - doc_path = "/".join(path_parts[2:]).strip().rstrip("/") # Drop repo_name/tree - return f"{base_repo_url}/{repo}/edit/{doc_path}" - - -def on_env(env: Environment, config, files): - """Enable loopcontrols extension in Jinja2""" - env.add_extension("jinja2.ext.loopcontrols") - env.globals["github_path"] = github_path - env.trim_blocks = True - env.lstrip_blocks = True - return env - - -def set_extra_source_docs(source_dir: str, config: dict): - """ - Read the list of tools from the source directory and - store it in MkDocs extra config. These are used to generate - the correct links to the documentation in GitHub. - """ - - base_repo_url = config["repo_url"] # Set in mkdocs.yml as repo_url - source_dir = Path(source_dir) - - # Dict to store the documentation path for each tool in GitHub - # the key is the doc name and the value is the path in the GitHub repository - source_docs = {} - - # Read the source files and extract the GitHub path from the Available at link at the bottom of the page - pattern = re.compile( - r"Available at:\s*\[(?P.*?)\]\(https://github\.com/OSGeo/(?P.*?)\)" - ) - - for file in source_dir.rglob("*.md"): - with file.open() as f: - for line in f: - match = pattern.search(line) - if match: - # Extract the entire link text and the GitHub path. - text = match.group("text") - gh_path = match.group("gh_path").strip() - - # Remove the trailing "source code" from the text to get the tool name. - toolname = re.sub( - r"\s*source\s+code\s*$", "", text, flags=re.IGNORECASE - ).strip() - print(f"Found {toolname} at {gh_path}") - source_docs[toolname] = github_edit_path(gh_path, base_repo_url) - - # Store in MkDocs extra config - return source_docs - - -def on_config(config): - """ - Runs on MkDocs config to set the extra config with the source_docs - """ - - # Store in MkDocs extra config - config["extra"]["source_docs"] = set_extra_source_docs("source", config) - - return config diff --git a/utils/generate_sitemap.py b/utils/generate_sitemap.py new file mode 100755 index 00000000000..68a90bcf6aa --- /dev/null +++ b/utils/generate_sitemap.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2026 by the GRASS Development Team +# SPDX-License-Identifier: GPL-2.0-or-later + +"""Generate a sitemap listing every HTML page of a built documentation site. + +Static site generators list only the pages they know about, which for +Zensical means the pages in the navigation. The GRASS manual keeps the +hundreds of tool pages out of the navigation, so the sitemap is generated +from the built pages instead. +""" + +import argparse +import datetime +from pathlib import Path +from xml.sax.saxutils import escape + + +def page_paths(site_dir: Path) -> list[str]: + """Return the site-relative paths of all HTML pages, sorted. + + The error page is not a page to be indexed and is left out. + """ + return sorted( + path.relative_to(site_dir).as_posix() + for path in site_dir.rglob("*.html") + if path.relative_to(site_dir).as_posix() != "404.html" + ) + + +def sitemap_xml(site_url: str, paths: list[str], lastmod: datetime.date) -> str: + """Return the sitemap document for the given page paths.""" + base = site_url.rstrip("/") + "/" + lines = [ + '', + '', + ] + for path in paths: + lines += [ + " ", + f" {escape(base + path)}", + f" {lastmod.isoformat()}", + " ", + ] + lines.append("") + return "\n".join(lines) + "\n" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--site-dir", + type=Path, + required=True, + help="directory with the built HTML pages", + ) + parser.add_argument( + "--site-url", + required=True, + help="URL the site directory is published at", + ) + parser.add_argument( + "--output", + type=Path, + help="output file (default: sitemap.xml in the site directory)", + ) + args = parser.parse_args() + + if not args.site_dir.is_dir(): + parser.error(f"{args.site_dir} is not a directory") + output = args.output or args.site_dir / "sitemap.xml" + + paths = page_paths(args.site_dir) + output.write_text( + sitemap_xml(args.site_url, paths, datetime.date.today()), encoding="utf-8" + ) + print(f"Wrote {len(paths)} pages to {output}") + + +if __name__ == "__main__": + main() diff --git a/utils/mkmarkdown.py b/utils/mkmarkdown.py index 4a62c25f120..92349c85dfe 100644 --- a/utils/mkmarkdown.py +++ b/utils/mkmarkdown.py @@ -171,8 +171,11 @@ def modify_keyword_links(text, prefix): return text.replace(keywords_content, modified_keywords_content) -def merge_md_files(md1, md2, content_modifier1, content_modifier2): - """Merge two markdown files by concatenating their YAML headers and content.""" +def merge_md_files(md1, md2, content_modifier1, content_modifier2, extra_yaml=None): + """Merge two markdown files by concatenating their YAML headers and content. + + The optional extra_yaml is appended to the combined YAML header. + """ yaml1, content1 = extract_yaml_header(md1) yaml2, content2 = extract_yaml_header(md2) @@ -187,6 +190,8 @@ def merge_md_files(md1, md2, content_modifier1, content_modifier2): yaml_items.append(yaml1) if yaml2: yaml_items.append(yaml2) + if extra_yaml: + yaml_items.append(extra_yaml) if yaml_items: # Add YAML header only when non-empty. combined_yaml = "\n".join(yaml_items) @@ -204,6 +209,18 @@ def merge_md_files(md1, md2, content_modifier1, content_modifier2): return result +def source_edit_url(url_source, pgm): + """Return the URL for editing the tool's Markdown source on GitHub. + + The source URL points to the tool directory in the repository tree, + so the edit URL is the same path in the edit view plus the file name. + Returns None when the source location is unknown. + """ + if not url_source: + return None + return f"{url_source.rstrip('/').replace('/tree/', '/edit/', 1)}/{pgm}.md" + + def main(): pgm = sys.argv[1] @@ -226,6 +243,12 @@ def modify_generated(text): prefix = os.getenv("HTML_PAGE_FOOTER_PAGES_PATH") or None return modify_keyword_links(text, prefix=prefix) + url_source, url_log, date_tag = parse_source(pgm) + # The edit URL goes into the front matter so that the website templates + # can link to the source file without any Python hook in the site build. + edit_url = source_edit_url(url_source, pgm) + extra_yaml = f"source_edit_url: {edit_url}" if edit_url else None + # process header/usage generated by --md-description tool_generated = read_file(tmp_file) # process body @@ -236,6 +259,7 @@ def modify_generated(text): source_code, content_modifier1=modify_generated, content_modifier2=None, + extra_yaml=extra_yaml, ) for item in items: if not item: @@ -243,7 +267,6 @@ def modify_generated(text): sys.stdout.write(item) # process footer - url_source, url_log, date_tag = parse_source(pgm) sys.stdout.write( sourcecode.substitute( URL_SOURCE=url_source, diff --git a/utils/tests/generate_sitemap_test.py b/utils/tests/generate_sitemap_test.py new file mode 100644 index 00000000000..6ae18c0d431 --- /dev/null +++ b/utils/tests/generate_sitemap_test.py @@ -0,0 +1,93 @@ +# Copyright (C) 2026 by the GRASS Development Team +# SPDX-License-Identifier: GPL-2.0-or-later + +"""Tests for the sitemap generation from a built site.""" + +import datetime +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +UTILS_DIR = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(UTILS_DIR)) + +import generate_sitemap # noqa: E402 + +NS = "{http://www.sitemaps.org/schemas/sitemap/0.9}" + + +@pytest.fixture +def site_dir(tmp_path): + """Built site with pages at the root and in a subdirectory, plus non-pages.""" + for name in [ + "index.html", + "r.slope.aspect.html", + "404.html", + "addons/r.tree.html", + "assets/javascripts/bundle.js", + "colortables/aspect.png", + "search.json", + ]: + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("") + return tmp_path + + +def test_page_paths(site_dir): + assert generate_sitemap.page_paths(site_dir) == [ + "addons/r.tree.html", + "index.html", + "r.slope.aspect.html", + ] + + +@pytest.mark.parametrize( + "site_url", + ["https://example.org/manuals", "https://example.org/manuals/"], +) +def test_sitemap_xml(site_url): + xml = generate_sitemap.sitemap_xml( + site_url, ["index.html", "addons/r.tree.html"], datetime.date(2026, 9, 22) + ) + urls = ET.fromstring(xml).findall(f"{NS}url") + assert [url.find(f"{NS}loc").text for url in urls] == [ + "https://example.org/manuals/index.html", + "https://example.org/manuals/addons/r.tree.html", + ] + assert {url.find(f"{NS}lastmod").text for url in urls} == {"2026-09-22"} + + +def test_sitemap_xml_escapes_url(): + xml = generate_sitemap.sitemap_xml( + "https://example.org/a&b", ["x.html"], datetime.date(2026, 9, 22) + ) + assert ET.fromstring(xml).find(f"{NS}url/{NS}loc").text == ( + "https://example.org/a&b/x.html" + ) + + +def test_script_writes_sitemap_into_site_dir(site_dir): + subprocess.run( + [ + sys.executable, + str(UTILS_DIR / "generate_sitemap.py"), + "--site-dir", + str(site_dir), + "--site-url", + "https://example.org/manuals/", + ], + check=True, + ) + locs = [ + loc.text + for loc in ET.parse(site_dir / "sitemap.xml").getroot().iter(f"{NS}loc") + ] + assert locs == [ + "https://example.org/manuals/addons/r.tree.html", + "https://example.org/manuals/index.html", + "https://example.org/manuals/r.slope.aspect.html", + ] diff --git a/utils/tests/mkmarkdown_test.py b/utils/tests/mkmarkdown_test.py new file mode 100644 index 00000000000..079dbd6eb4e --- /dev/null +++ b/utils/tests/mkmarkdown_test.py @@ -0,0 +1,68 @@ +# Copyright (C) 2026 by the GRASS Development Team +# SPDX-License-Identifier: GPL-2.0-or-later + +"""Tests for the Markdown page assembly in mkmarkdown.py.""" + +import os +import sys +from pathlib import Path + +import pytest + +UTILS_DIR = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(UTILS_DIR)) +# The shared helpers resolve the repository root from the build environment +# at import time, so they need it set even when the functions under test +# never use it. +os.environ.setdefault("MODULE_TOPDIR", str(UTILS_DIR.parent)) + +import mkmarkdown # noqa: E402 + + +@pytest.mark.parametrize( + ("url_source", "pgm", "expected"), + [ + ( + "https://github.com/OSGeo/grass/tree/main/raster/r.slope.aspect", + "r.slope.aspect", + "https://github.com/OSGeo/grass/edit/main/raster/r.slope.aspect/r.slope.aspect.md", + ), + ( + "https://github.com/OSGeo/grass/tree/main/gui/wxpython/docs/", + "wxGUI.nviz", + "https://github.com/OSGeo/grass/edit/main/gui/wxpython/docs/wxGUI.nviz.md", + ), + ( + "https://github.com/OSGeo/grass-addons/tree/grass8/src/raster/r.tree", + "r.tree", + "https://github.com/OSGeo/grass-addons/edit/grass8/src/raster/r.tree/r.tree.md", + ), + ], +) +def test_source_edit_url(url_source, pgm, expected): + assert mkmarkdown.source_edit_url(url_source, pgm) == expected + + +def test_source_edit_url_unknown_source(): + assert mkmarkdown.source_edit_url("", "r.slope.aspect") is None + + +def test_merge_md_files_appends_extra_yaml(): + generated = "---\nname: r.tool\n---\n\n# r.tool\n" + source = "---\ndescription: Tool\n---\n\nBody\n" + result = "".join( + mkmarkdown.merge_md_files( + generated, source, None, None, extra_yaml="source_edit_url: URL" + ) + ) + assert result.startswith( + "---\nname: r.tool\ndescription: Tool\nsource_edit_url: URL\n---\n" + ) + assert "Body" in result + + +def test_merge_md_files_extra_yaml_creates_header(): + result = "".join( + mkmarkdown.merge_md_files("", "# Page\n", None, None, extra_yaml="key: value") + ) + assert result.startswith("---\nkey: value\n---\n")