diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index d4413ade598..ae43acbf464 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -238,6 +238,15 @@ jobs: cd "${MKDOCS_DIR}" || exit mkdocs build + - name: Generate manuals sitemap + # List every built page, not only those in the site navigation, so + # the tool pages stay in the sitemap regardless of the site generator. + # Runs before libpython is moved in; Sphinx provides that sitemap. + run: | + python grass/utils/generate_sitemap.py \ + --site-dir "${MKDOCS_DIR}/site" \ + --site-url https://grass.osgeo.org/grass-stable/manuals/ + - name: Build Sphinx documentation run: | pip install -r "grass/python/grass/docs/requirements.txt" diff --git a/man/CMakeLists.txt b/man/CMakeLists.txt index bc962806516..a6a6d75a882 100644 --- a/man/CMakeLists.txt +++ b/man/CMakeLists.txt @@ -131,8 +131,7 @@ if(NOT WITH_FHS) add_custom_target( copy_mkdocs_files COMMAND - ${CMAKE_COMMAND} -E make_directory ${mkdocs_dir}/scripts - ${mkdocs_dir}/overrides/partials + ${CMAKE_COMMAND} -E make_directory ${mkdocs_dir}/overrides/partials ${mkdocs_dir}/overrides/fragments/tags/default COMMAND ${CMAKE_COMMAND} -E copy_if_different @@ -147,10 +146,6 @@ if(NOT WITH_FHS) ${CMAKE_CURRENT_SOURCE_DIR}/grass_logo.svg ${CMAKE_CURRENT_SOURCE_DIR}/grass_icon.png ${CMAKE_CURRENT_SOURCE_DIR}/favicon.ico ${mkdocs_dir}/source - COMMAND - ${CMAKE_COMMAND} -E copy_if_different - ${CMAKE_CURRENT_SOURCE_DIR}/mkdocs/scripts/hook_list_scripts.py - ${mkdocs_dir}/scripts COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/mkdocs/overrides/partials/copyright.html diff --git a/man/Makefile b/man/Makefile index 3f71bf7cd99..9c4735a780e 100644 --- a/man/Makefile +++ b/man/Makefile @@ -29,7 +29,6 @@ DSTFILES := \ $(MDDIR)/source/grass_icon.png \ $(MDDIR)/source/favicon.ico \ $(MDDIR)/source/grassdocs.css \ - $(MDDIR)/scripts/hook_list_scripts.py \ $(MDDIR)/source/index.md \ $(MDDIR)/overrides/main.html \ $(MDDIR)/overrides/partials/copyright.html \ @@ -281,12 +280,6 @@ $(MDDIR)/source/development_intro.md: mkdocs/docs/development_intro.md $(MDDIR)/source/command_line_intro.md: mkdocs/docs/command_line_intro.md $(INSTALL_DATA) $< $@ -$(MDDIR)/scripts/hook_list_scripts.py: mkdocs/scripts/hook_list_scripts.py | $(MDDIR)/scripts - $(INSTALL_DATA) $< $@ - -$(MDDIR)/scripts: - $(MKDIR) $@ - $(MDDIR)/overrides/main.html: mkdocs/overrides/main.html | $(MDDIR)/overrides $(INSTALL_DATA) $< $@ diff --git a/man/mkdocs/mkdocs.yml b/man/mkdocs/mkdocs.yml index 3b12a567a6a..3df1775290a 100644 --- a/man/mkdocs/mkdocs.yml +++ b/man/mkdocs/mkdocs.yml @@ -75,10 +75,6 @@ extra: grass: grass grass_addons: grass-addons -# Hooks -hooks: - - scripts/hook_list_scripts.py - # Custom CSS extra_css: - grassdocs.css diff --git a/man/mkdocs/overrides/partials/actions.html b/man/mkdocs/overrides/partials/actions.html index 3256f22728b..82ab35c1d88 100644 --- a/man/mkdocs/overrides/partials/actions.html +++ b/man/mkdocs/overrides/partials/actions.html @@ -1,27 +1,21 @@ -{% if page.edit_url %} - - {% set toolname = page.file.src_path.split("/")[-1] | replace(".md", "") %} - {# Default to the normal edit URL #} - {% set ghpath = github_path(toolname, config.extra.source_docs) %} - {% if ghpath %} - {% set custom_edit_url = page.edit_url | replace(page.edit_url, ghpath ~ "/" ~ toolname ~ ".md") %} - {% if "content.action.edit" in features %} - - {% set icon = config.theme.icon.edit or "material/file-edit-outline" %} - {% include ".icons/" ~ icon ~ ".svg" %} - - {% endif %} - - {% if "content.action.view" in features %} - {% if "/blob/" in custom_edit_url %} - {% set part = "/blob/" %} - {% else %} - {% set part = "/edit/" %} - {% endif %} - - {% set icon = config.theme.icon.view or "material/file-eye-outline" %} - {% include ".icons/" ~ icon ~ ".svg" %} - - {% endif %} +{# + The generated pages carry the URL of their Markdown source in the front + matter (source_edit_url, written by utils/mkmarkdown.py), so the edit and + view buttons can point to the tool directory in the repository rather than + to the generated documentation tree, without any Python in the site build. +#} +{% if page.meta and page.meta.source_edit_url %} + {% set edit_url = page.meta.source_edit_url %} + {% if "content.action.edit" in features %} + + {% set icon = config.theme.icon.edit or "material/file-edit-outline" %} + {% include ".icons/" ~ icon ~ ".svg" %} + + {% endif %} + {% if "content.action.view" in features %} + + {% set icon = config.theme.icon.view or "material/file-eye-outline" %} + {% include ".icons/" ~ icon ~ ".svg" %} + {% endif %} {% endif %} diff --git a/man/mkdocs/scripts/hook_list_scripts.py b/man/mkdocs/scripts/hook_list_scripts.py deleted file mode 100644 index f815e5bdd2b..00000000000 --- a/man/mkdocs/scripts/hook_list_scripts.py +++ /dev/null @@ -1,83 +0,0 @@ -import re -from pathlib import Path - -from jinja2 import Environment - - -def github_path(toolname: str, source_docs: dict[str, str]): - """Return the GitHub path for the given toolname""" - try: - return source_docs[toolname] - except KeyError: - return None - - -def github_edit_path(ghpath: str, base_repo_url: str): - """Return the GitHub edit path for the given path""" - path_parts = ghpath.split("/") - - # Gracefully handle invalid paths - if len(path_parts) < 3: - return None - repo = path_parts[0].strip() - doc_path = "/".join(path_parts[2:]).strip().rstrip("/") # Drop repo_name/tree - return f"{base_repo_url}/{repo}/edit/{doc_path}" - - -def on_env(env: Environment, config, files): - """Enable loopcontrols extension in Jinja2""" - env.add_extension("jinja2.ext.loopcontrols") - env.globals["github_path"] = github_path - env.trim_blocks = True - env.lstrip_blocks = True - return env - - -def set_extra_source_docs(source_dir: str, config: dict): - """ - Read the list of tools from the source directory and - store it in MkDocs extra config. These are used to generate - the correct links to the documentation in GitHub. - """ - - base_repo_url = config["repo_url"] # Set in mkdocs.yml as repo_url - source_dir = Path(source_dir) - - # Dict to store the documentation path for each tool in GitHub - # the key is the doc name and the value is the path in the GitHub repository - source_docs = {} - - # Read the source files and extract the GitHub path from the Available at link at the bottom of the page - pattern = re.compile( - r"Available at:\s*\[(?P.*?)\]\(https://github\.com/OSGeo/(?P.*?)\)" - ) - - for file in source_dir.rglob("*.md"): - with file.open() as f: - for line in f: - match = pattern.search(line) - if match: - # Extract the entire link text and the GitHub path. - text = match.group("text") - gh_path = match.group("gh_path").strip() - - # Remove the trailing "source code" from the text to get the tool name. - toolname = re.sub( - r"\s*source\s+code\s*$", "", text, flags=re.IGNORECASE - ).strip() - print(f"Found {toolname} at {gh_path}") - source_docs[toolname] = github_edit_path(gh_path, base_repo_url) - - # Store in MkDocs extra config - return source_docs - - -def on_config(config): - """ - Runs on MkDocs config to set the extra config with the source_docs - """ - - # Store in MkDocs extra config - config["extra"]["source_docs"] = set_extra_source_docs("source", config) - - return config diff --git a/utils/generate_sitemap.py b/utils/generate_sitemap.py new file mode 100755 index 00000000000..68a90bcf6aa --- /dev/null +++ b/utils/generate_sitemap.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2026 by the GRASS Development Team +# SPDX-License-Identifier: GPL-2.0-or-later + +"""Generate a sitemap listing every HTML page of a built documentation site. + +Static site generators list only the pages they know about, which for +Zensical means the pages in the navigation. The GRASS manual keeps the +hundreds of tool pages out of the navigation, so the sitemap is generated +from the built pages instead. +""" + +import argparse +import datetime +from pathlib import Path +from xml.sax.saxutils import escape + + +def page_paths(site_dir: Path) -> list[str]: + """Return the site-relative paths of all HTML pages, sorted. + + The error page is not a page to be indexed and is left out. + """ + return sorted( + path.relative_to(site_dir).as_posix() + for path in site_dir.rglob("*.html") + if path.relative_to(site_dir).as_posix() != "404.html" + ) + + +def sitemap_xml(site_url: str, paths: list[str], lastmod: datetime.date) -> str: + """Return the sitemap document for the given page paths.""" + base = site_url.rstrip("/") + "/" + lines = [ + '', + '', + ] + for path in paths: + lines += [ + " ", + f" {escape(base + path)}", + f" {lastmod.isoformat()}", + " ", + ] + lines.append("") + return "\n".join(lines) + "\n" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--site-dir", + type=Path, + required=True, + help="directory with the built HTML pages", + ) + parser.add_argument( + "--site-url", + required=True, + help="URL the site directory is published at", + ) + parser.add_argument( + "--output", + type=Path, + help="output file (default: sitemap.xml in the site directory)", + ) + args = parser.parse_args() + + if not args.site_dir.is_dir(): + parser.error(f"{args.site_dir} is not a directory") + output = args.output or args.site_dir / "sitemap.xml" + + paths = page_paths(args.site_dir) + output.write_text( + sitemap_xml(args.site_url, paths, datetime.date.today()), encoding="utf-8" + ) + print(f"Wrote {len(paths)} pages to {output}") + + +if __name__ == "__main__": + main() diff --git a/utils/mkmarkdown.py b/utils/mkmarkdown.py index 4a62c25f120..92349c85dfe 100644 --- a/utils/mkmarkdown.py +++ b/utils/mkmarkdown.py @@ -171,8 +171,11 @@ def modify_keyword_links(text, prefix): return text.replace(keywords_content, modified_keywords_content) -def merge_md_files(md1, md2, content_modifier1, content_modifier2): - """Merge two markdown files by concatenating their YAML headers and content.""" +def merge_md_files(md1, md2, content_modifier1, content_modifier2, extra_yaml=None): + """Merge two markdown files by concatenating their YAML headers and content. + + The optional extra_yaml is appended to the combined YAML header. + """ yaml1, content1 = extract_yaml_header(md1) yaml2, content2 = extract_yaml_header(md2) @@ -187,6 +190,8 @@ def merge_md_files(md1, md2, content_modifier1, content_modifier2): yaml_items.append(yaml1) if yaml2: yaml_items.append(yaml2) + if extra_yaml: + yaml_items.append(extra_yaml) if yaml_items: # Add YAML header only when non-empty. combined_yaml = "\n".join(yaml_items) @@ -204,6 +209,18 @@ def merge_md_files(md1, md2, content_modifier1, content_modifier2): return result +def source_edit_url(url_source, pgm): + """Return the URL for editing the tool's Markdown source on GitHub. + + The source URL points to the tool directory in the repository tree, + so the edit URL is the same path in the edit view plus the file name. + Returns None when the source location is unknown. + """ + if not url_source: + return None + return f"{url_source.rstrip('/').replace('/tree/', '/edit/', 1)}/{pgm}.md" + + def main(): pgm = sys.argv[1] @@ -226,6 +243,12 @@ def modify_generated(text): prefix = os.getenv("HTML_PAGE_FOOTER_PAGES_PATH") or None return modify_keyword_links(text, prefix=prefix) + url_source, url_log, date_tag = parse_source(pgm) + # The edit URL goes into the front matter so that the website templates + # can link to the source file without any Python hook in the site build. + edit_url = source_edit_url(url_source, pgm) + extra_yaml = f"source_edit_url: {edit_url}" if edit_url else None + # process header/usage generated by --md-description tool_generated = read_file(tmp_file) # process body @@ -236,6 +259,7 @@ def modify_generated(text): source_code, content_modifier1=modify_generated, content_modifier2=None, + extra_yaml=extra_yaml, ) for item in items: if not item: @@ -243,7 +267,6 @@ def modify_generated(text): sys.stdout.write(item) # process footer - url_source, url_log, date_tag = parse_source(pgm) sys.stdout.write( sourcecode.substitute( URL_SOURCE=url_source, diff --git a/utils/tests/generate_sitemap_test.py b/utils/tests/generate_sitemap_test.py new file mode 100644 index 00000000000..6ae18c0d431 --- /dev/null +++ b/utils/tests/generate_sitemap_test.py @@ -0,0 +1,93 @@ +# Copyright (C) 2026 by the GRASS Development Team +# SPDX-License-Identifier: GPL-2.0-or-later + +"""Tests for the sitemap generation from a built site.""" + +import datetime +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +UTILS_DIR = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(UTILS_DIR)) + +import generate_sitemap # noqa: E402 + +NS = "{http://www.sitemaps.org/schemas/sitemap/0.9}" + + +@pytest.fixture +def site_dir(tmp_path): + """Built site with pages at the root and in a subdirectory, plus non-pages.""" + for name in [ + "index.html", + "r.slope.aspect.html", + "404.html", + "addons/r.tree.html", + "assets/javascripts/bundle.js", + "colortables/aspect.png", + "search.json", + ]: + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("") + return tmp_path + + +def test_page_paths(site_dir): + assert generate_sitemap.page_paths(site_dir) == [ + "addons/r.tree.html", + "index.html", + "r.slope.aspect.html", + ] + + +@pytest.mark.parametrize( + "site_url", + ["https://example.org/manuals", "https://example.org/manuals/"], +) +def test_sitemap_xml(site_url): + xml = generate_sitemap.sitemap_xml( + site_url, ["index.html", "addons/r.tree.html"], datetime.date(2026, 9, 22) + ) + urls = ET.fromstring(xml).findall(f"{NS}url") + assert [url.find(f"{NS}loc").text for url in urls] == [ + "https://example.org/manuals/index.html", + "https://example.org/manuals/addons/r.tree.html", + ] + assert {url.find(f"{NS}lastmod").text for url in urls} == {"2026-09-22"} + + +def test_sitemap_xml_escapes_url(): + xml = generate_sitemap.sitemap_xml( + "https://example.org/a&b", ["x.html"], datetime.date(2026, 9, 22) + ) + assert ET.fromstring(xml).find(f"{NS}url/{NS}loc").text == ( + "https://example.org/a&b/x.html" + ) + + +def test_script_writes_sitemap_into_site_dir(site_dir): + subprocess.run( + [ + sys.executable, + str(UTILS_DIR / "generate_sitemap.py"), + "--site-dir", + str(site_dir), + "--site-url", + "https://example.org/manuals/", + ], + check=True, + ) + locs = [ + loc.text + for loc in ET.parse(site_dir / "sitemap.xml").getroot().iter(f"{NS}loc") + ] + assert locs == [ + "https://example.org/manuals/addons/r.tree.html", + "https://example.org/manuals/index.html", + "https://example.org/manuals/r.slope.aspect.html", + ] diff --git a/utils/tests/mkmarkdown_test.py b/utils/tests/mkmarkdown_test.py new file mode 100644 index 00000000000..079dbd6eb4e --- /dev/null +++ b/utils/tests/mkmarkdown_test.py @@ -0,0 +1,68 @@ +# Copyright (C) 2026 by the GRASS Development Team +# SPDX-License-Identifier: GPL-2.0-or-later + +"""Tests for the Markdown page assembly in mkmarkdown.py.""" + +import os +import sys +from pathlib import Path + +import pytest + +UTILS_DIR = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(UTILS_DIR)) +# The shared helpers resolve the repository root from the build environment +# at import time, so they need it set even when the functions under test +# never use it. +os.environ.setdefault("MODULE_TOPDIR", str(UTILS_DIR.parent)) + +import mkmarkdown # noqa: E402 + + +@pytest.mark.parametrize( + ("url_source", "pgm", "expected"), + [ + ( + "https://github.com/OSGeo/grass/tree/main/raster/r.slope.aspect", + "r.slope.aspect", + "https://github.com/OSGeo/grass/edit/main/raster/r.slope.aspect/r.slope.aspect.md", + ), + ( + "https://github.com/OSGeo/grass/tree/main/gui/wxpython/docs/", + "wxGUI.nviz", + "https://github.com/OSGeo/grass/edit/main/gui/wxpython/docs/wxGUI.nviz.md", + ), + ( + "https://github.com/OSGeo/grass-addons/tree/grass8/src/raster/r.tree", + "r.tree", + "https://github.com/OSGeo/grass-addons/edit/grass8/src/raster/r.tree/r.tree.md", + ), + ], +) +def test_source_edit_url(url_source, pgm, expected): + assert mkmarkdown.source_edit_url(url_source, pgm) == expected + + +def test_source_edit_url_unknown_source(): + assert mkmarkdown.source_edit_url("", "r.slope.aspect") is None + + +def test_merge_md_files_appends_extra_yaml(): + generated = "---\nname: r.tool\n---\n\n# r.tool\n" + source = "---\ndescription: Tool\n---\n\nBody\n" + result = "".join( + mkmarkdown.merge_md_files( + generated, source, None, None, extra_yaml="source_edit_url: URL" + ) + ) + assert result.startswith( + "---\nname: r.tool\ndescription: Tool\nsource_edit_url: URL\n---\n" + ) + assert "Body" in result + + +def test_merge_md_files_extra_yaml_creates_header(): + result = "".join( + mkmarkdown.merge_md_files("", "# Page\n", None, None, extra_yaml="key: value") + ) + assert result.startswith("---\nkey: value\n---\n")