diff --git a/man/parser_standard_options.py b/man/parser_standard_options.py index febbaa5d781..fb5c8f21aac 100644 --- a/man/parser_standard_options.py +++ b/man/parser_standard_options.py @@ -45,7 +45,7 @@ def count_par_diff(line: str) -> int: elif line.startswith("G_"): diff = count_par_diff(line) elif diff > 0: - diff -= count_par_diff(line) + diff += count_par_diff(line) elif res is not None: res.append(line) diff --git a/man/tests/parser_standard_options_test.py b/man/tests/parser_standard_options_test.py new file mode 100644 index 00000000000..7684b4a874f --- /dev/null +++ b/man/tests/parser_standard_options_test.py @@ -0,0 +1,299 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +"""Tests for the extraction of standard parser options from the C source.""" + +import re +import sys +from pathlib import Path + +import pytest + +MAN_DIR = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(MAN_DIR)) + +import parser_standard_options as pso # noqa: E402 + +STANDARD_OPTIONS_SOURCE = MAN_DIR.parent / "lib" / "gis" / "parser_standard_options.c" + +# Excerpt of lib/gis/parser_standard_options.c with the constructs the parser +# has to handle: a dynamic answer, a description split into two string +# literals, translation macros, and a multiline function call. +C_SOURCE = """\ + /* raster maps */ + case G_OPT_MEMORYMB: + Opt->key = "memory"; + Opt->type = TYPE_INTEGER; + Opt->key_desc = "memory in MB"; + Opt->required = NO; + Opt->multiple = NO; + Opt->answer = "300"; + /* start dynamic answer */ + /* check MEMORYMB in GISRC, set with g.gisenv */ + memstr = G_store(G_getenv_nofatal("MEMORYMB")); + if (memstr && *memstr) + Opt->answer = memstr; + /* end dynamic answer */ + Opt->label = _("Maximum memory to be used (in MB)"); + Opt->description = _("Cache size for raster rows"); + break; + case G_OPT_R_TYPE: + Opt->key = "type"; + Opt->type = TYPE_STRING; + Opt->required = YES; + Opt->multiple = NO; + Opt->label = _("Type of raster map to be created"); + Opt->description = _("Storage type for resultant raster map"); + Opt->options = "CELL,FCELL,DCELL"; + G_asprintf((char **)&(Opt->descriptions), "CELL;%s;FCELL;%s;DCELL;%s", + _("Integer"), _("Single precision floating point"), + _("Double precision floating point")); + break; + case G_OPT_M_NPROCS: + Opt->key = "nprocs"; + Opt->type = TYPE_INTEGER; + Opt->required = NO; + Opt->multiple = NO; + Opt->answer = "0"; + Opt->label = _("Number of threads for parallel computing"); + Opt->description = _("0: use OpenMP default; >0: use nprocs; " + "<0: use MAX-nprocs"); + break; +""" + + +def parse_c_source(): + return dict(pso.parse_options(C_SOURCE.splitlines())) + + +def test_parse_options_names_and_order(): + """Check that each case label is an option, in the order of the source""" + names = [name for name, _ in pso.parse_options(C_SOURCE.splitlines())] + assert names == ["G_OPT_MEMORYMB", "G_OPT_R_TYPE", "G_OPT_M_NPROCS"] + + +def test_parse_options_values_are_cleaned(): + """Check that quotes, translation macros and semicolons are removed""" + option = parse_c_source()["G_OPT_R_TYPE"] + assert option == { + "key": "type", + "type": "TYPE_STRING", + "required": "YES", + "multiple": "NO", + "label": "Type of raster map to be created", + "description": "Storage type for resultant raster map", + "options": "CELL,FCELL,DCELL", + } + + +def test_parse_options_ignores_dynamic_answer(): + """Check that the static answer is used instead of the computed one""" + option = parse_c_source()["G_OPT_MEMORYMB"] + assert option["answer"] == "300" + assert option["key_desc"] == "memory in MB" + + +def test_parse_options_joins_multiline_strings(): + """Check that a description split into several literals is one string""" + option = parse_c_source()["G_OPT_M_NPROCS"] + assert option["description"] == ( + "0: use OpenMP default; >0: use nprocs; <0: use MAX-nprocs" + ) + + +def test_parse_options_skips_multiline_function_calls(): + """Check that the arguments of a multiline call are not taken as options""" + option = parse_c_source()["G_OPT_R_TYPE"] + assert "descriptions" not in option + assert not any("Integer" in value for value in option.values()) + + +def test_parse_options_continues_after_multiline_function_call(): + """Check that an assignment following a multiline call is still extracted""" + lines = [ + "case G_OPT_AFTER_CALL:", + 'Opt->key = "after";', + "G_asprintf(", + '(char **)&(Opt->descriptions), "a;%s",', + '_("A"));', + 'Opt->description = _("After the call");', + "break;", + ] + assert pso.parse_options(lines) == [ + ("G_OPT_AFTER_CALL", {"key": "after", "description": "After the call"}) + ] + + +def test_parse_options_ignores_lines_outside_cases(): + """Check that nothing is extracted before the first case or without a body""" + assert pso.parse_options([]) == [] + assert pso.parse_options(['Opt->key = "stray";', "break;"]) == [] + assert pso.parse_options(["case G_OPT_EMPTY:", "break;"]) == [] + + +def test_parse_options_unescapes_quotes(): + """Check that escaped double quotes in a string become plain quotes""" + lines = [ + "case G_OPT_QUOTED:", + r'Opt->description = _("Use \"auto\" for default");', + "break;", + ] + assert pso.parse_options(lines) == [ + ("G_OPT_QUOTED", {"description": 'Use "auto" for default'}) + ] + + +def test_parse_options_all_standard_options(): + """Check that every standard option in the real source file is extracted""" + source = STANDARD_OPTIONS_SOURCE.read_text(encoding="utf-8") + expected = re.findall(r"^\s*case (G_OPT_\w+):", source, flags=re.MULTILINE) + options = pso.parse_options(source.splitlines()) + assert [name for name, _ in options] == expected + + +def test_parse_options_real_raster_input(): + """Check a well known option from the real source file""" + source = STANDARD_OPTIONS_SOURCE.read_text(encoding="utf-8") + option = dict(pso.parse_options(source.splitlines()))["G_OPT_R_INPUT"] + assert option["key"] == "input" + assert option["required"] == "YES" + assert option["gisprompt"] == "old,cell,raster" + + +@pytest.mark.parametrize( + ("name", "heading"), + [ + ("G_OPT_R_INPUT", "Raster"), + ("G_OPT_R3_INPUT", "Raster 3D"), + ("G_OPT_V_OUTPUT", "Vector"), + ("G_OPT_V3_TYPE", "Vector"), + ("G_OPT_STRDS_INPUT", "Temporal"), + ("G_OPT_CN", "Color"), + ("G_OPT_MEMORYMB", "Other"), + ("G_OPT_UNKNOWN_PREFIX", "Other"), + ], +) +def test_get_heading(name, heading): + """Check that the heading is selected by the part after G_OPT_""" + assert pso.get_heading(name) == heading + + +def test_get_heading_invalid_identifier(): + """Check that an identifier not starting with G_OPT_ is reported""" + with pytest.raises(ValueError, match="Strangely formatted"): + pso.get_heading("R_INPUT") + + +def test_write_markdown_option_string_option(): + """Check the usage examples generated for an option of type string""" + md = [] + option = { + "key": "input", + "type": "TYPE_STRING", + "key_desc": "name", + "required": "YES", + "multiple": "NO", + } + pso.write_markdown_option( + md, "G_OPT_R_INPUT", option, ["key", "type", "required", "multiple"] + ) + text = "\n".join(md) + assert "| key | input |" in md + assert "| type | TYPE_STRING (`type: string`) |" in md + assert "| required | YES (`required: yes`) |" in md + assert "| multiple | NO (`multiple: no`) |" in md + assert "# %option G_OPT_R_INPUT" in text + assert 'input = options["input"]' in text + # A string is quoted and the key description is the example without answer. + assert 'input="name"' in md + + +def test_write_markdown_option_answer_example(): + """Check that the answer is the example unless it is a constant""" + md = [] + pso.write_markdown_option( + md, + "G_OPT_M_NPROCS", + {"key": "nprocs", "type": "TYPE_INTEGER", "answer": "0"}, + ["key", "answer"], + ) + assert "nprocs=0" in md + + md = [] + pso.write_markdown_option( + md, + "G_OPT_C", + { + "key": "color", + "type": "TYPE_STRING", + "answer": "DEFAULT_FG_COLOR", + "key_desc": "name", + }, + ["key", "answer"], + ) + assert 'color="name"' in md + + +def test_write_markdown_option_dynamic_value(): + """Check that a computed value is described as such""" + md = [] + pso.write_markdown_option( + md, + "G_OPT_EXAMPLE", + {"key": "example", "answer": "G_getenv_nofatal(NAME)"}, + ["key", "answer"], + ) + assert "| answer | Computed dynamically by *G_getenv_nofatal(NAME)* |" in md + + +def test_write_markdown_option_without_key(): + """Check that no Python usage examples are generated without a key""" + md = [] + pso.write_markdown_option(md, "G_OPT_NOKEY", {"type": "TYPE_STRING"}, ["type"]) + assert "To use the value in Python:" not in "\n".join(md) + + +def test_option_table_columns_are_sorted_union(): + """Check that the columns are all keys used by any option, sorted""" + table = pso.OptTable([("A", {"type": "t", "key": "k"}), ("B", {"required": "YES"})]) + assert table.columns == ["key", "required", "type"] + + +def test_option_table_csv_rows(): + """Check that missing values are empty and the delimiter is configurable""" + table = pso.OptTable([("A", {"key": "k", "type": "t"}), ("B", {"key": "l"})]) + rows = table.csv(delimiter="|").splitlines()[1:] + assert rows == ["A|k|t", "B|l|"] + + +def test_option_table_html(): + """Check the table structure, the header and the empty cells""" + table = pso.OptTable([("A", {"key": "k"}), ("B", {})]) + html = table.html(toptions=None).splitlines() + assert html[0] == "" + assert " " in html + assert " " in html + assert " " in html + assert " " in html + assert " " in html + assert html[-1] == "
optionkeyAk
" + assert pso.OptTable([]).html().splitlines()[0] == "" + + +def test_option_table_markdown_groups_options_by_heading(): + """Check that options are listed in a section of their heading""" + table = pso.OptTable( + [ + ("G_OPT_V_INPUT", {"key": "map"}), + ("G_OPT_R_INPUT", {"key": "input"}), + ("G_OPT_MEMORYMB", {"key": "memory"}), + ] + ) + lines = table.markdown().splitlines() + assert pso.MARKDOWN_INTRO_TEXT.splitlines()[0] in lines + raster = lines.index("## Raster") + vector = lines.index("## Vector") + other = lines.index("## Other") + assert lines.index("### G_OPT_R_INPUT") > raster + assert lines.index("### G_OPT_V_INPUT") > vector + assert lines.index("### G_OPT_MEMORYMB") > other + # Headings are sorted, with the catch-all one last. + assert raster < vector < other diff --git a/python/grass/script/tests/utils_test.py b/python/grass/script/tests/utils_test.py index 153a0daba5c..568218b25ce 100644 --- a/python/grass/script/tests/utils_test.py +++ b/python/grass/script/tests/utils_test.py @@ -1,6 +1,9 @@ """Test functions in grass.script.utils""" import multiprocessing +import operator +import os +import re import sys import pytest @@ -103,3 +106,262 @@ def test_get_multiprocessing_context(): assert ctx.get_start_method() == "fork" else: assert ctx.get_start_method() == multiprocessing.get_start_method() + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("elevation", "elevation"), + ("with space-and.dot", "with_space_and_dot"), + ("1st", "x1st"), + ("_hidden", "x_hidden"), + # Only ASCII letters are legal, so accented letters are replaced. + ("na\u00efve", "na_ve"), + # SQL keywords get a suffix to be usable as column names. + ("and", "and_"), + ("or", "or_"), + ("not", "not_"), + ], +) +def test_legalize_vector_name(name, expected): + """Check that invalid characters, leading digits and keywords are fixed""" + assert gutils.legalize_vector_name(name) == expected + + +def test_legalize_vector_name_fallback_prefix(): + """Check that the prefix is configurable and can be empty""" + assert gutils.legalize_vector_name("1st", fallback_prefix="map") == "map1st" + # An empty prefix is for names used as a suffix of an already valid name. + assert gutils.legalize_vector_name("1st", fallback_prefix="") == "1st" + + +def test_legalize_vector_name_invalid_arguments(): + """Check that an empty name and a prefix not starting with a letter fail""" + with pytest.raises(ValueError, match="cannot be empty"): + gutils.legalize_vector_name("") + with pytest.raises(ValueError, match="must start with an ASCII letter"): + gutils.legalize_vector_name("1st", fallback_prefix="1") + + +def test_append_random_suffix_length(): + """Check that the suffix has the requested length and legal characters""" + result = gutils.append_random("tmp", suffix_length=8) + assert re.fullmatch(r"tmp_[a-z0-9]{8}", result) + + +def test_append_random_total_length(): + """Check that the total length includes the name and the underscore""" + result = gutils.append_random("tmp", total_length=16) + assert len(result) == 16 + assert re.fullmatch(r"tmp_[a-z0-9]{12}", result) + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({}, "has to be provided"), + ({"suffix_length": 4, "total_length": 16}, "not both"), + # No room for the underscore plus at least one random character. + ({"total_length": 4}, "No characters left"), + ], +) +def test_append_random_invalid_arguments(kwargs, message): + """Check that inconsistent length arguments are rejected""" + with pytest.raises(ValueError, match=message): + gutils.append_random("tmp", **kwargs) + + +def test_append_uuid(): + """Check that a full UUID4 hex string is appended and differs between calls""" + first = gutils.append_uuid("tmp") + assert re.fullmatch(r"tmp_[0-9a-f]{32}", first) + assert first != gutils.append_uuid("tmp") + + +def test_append_node_pid(): + """Check that the name is followed by a legal node name and the process ID""" + result = gutils.append_node_pid("tmp") + assert re.fullmatch(rf"tmp_[A-Za-z0-9_]+_{os.getpid()}", result) + assert gutils.legalize_vector_name(result) == result + + +def test_append_node_pid_legalizes_node_name(monkeypatch): + """Check that characters of the host name which are not legal are replaced""" + monkeypatch.setattr(gutils.platform, "node", lambda: "my-host.example.org") + assert gutils.append_node_pid("tmp") == f"tmp_my_host_example_org_{os.getpid()}" + + +@pytest.mark.parametrize( + ("path", "ext", "expected"), + [ + ("/data/map.txt", None, "map.txt"), + ("/data/map.txt", "txt", "map"), + # The extension is compared ignoring the case of the path. + ("/data/map.TXT", "txt", "map"), + # Other extensions are kept, and only the last extension is removed. + ("/data/map.txt", "csv", "map.txt"), + ("/data/map.tar.gz", "gz", "map.tar"), + ("/data/noext", "txt", "noext"), + ], +) +def test_basename(path, ext, expected): + """Check that directories and a matching extension are removed""" + assert gutils.basename(path, ext) == expected + + +def test_try_remove(tmp_path): + """Check that a file is removed and that failures are silent""" + path = tmp_path / "file.txt" + path.write_text("content") + gutils.try_remove(path) + assert not path.exists() + # Already removed, no exception expected. + gutils.try_remove(path) + # A directory cannot be removed by os.remove(), no exception expected. + gutils.try_remove(tmp_path) + assert tmp_path.is_dir() + + +def test_try_rmdir(tmp_path): + """Check that empty and non-empty directories are removed silently""" + empty = tmp_path / "empty" + empty.mkdir() + gutils.try_rmdir(empty) + assert not empty.exists() + + full = tmp_path / "full" + (full / "nested").mkdir(parents=True) + (full / "nested" / "file.txt").write_text("content") + gutils.try_rmdir(full) + assert not full.exists() + + # Missing directory, no exception expected. + gutils.try_rmdir(full) + + +def test_diff_files(tmp_path): + """Check that the differ output marks changed and unchanged lines""" + first = tmp_path / "first.txt" + second = tmp_path / "second.txt" + first.write_text("same\nold\n") + second.write_text("same\nnew\n") + assert gutils.diff_files(first, second) == [ + " same\n", + "- old\n", + "+ new\n", + ] + assert gutils.diff_files(first, first) == [" same\n", " old\n"] + + +def test_split_keeps_quoted_values_together(): + """Check that a quoted value with spaces is a single list item""" + assert gutils.split('d.vect map=roads where="cat < 10"') == [ + "d.vect", + "map=roads", + "where=cat < 10", + ] + + +def test_naturally_sorted(): + """Check that numbers are compared by value and letters ignoring case""" + items = ["map10", "map2", "Map1", "map1a"] + assert gutils.naturally_sorted(items) == ["Map1", "map1a", "map2", "map10"] + # The input is left untouched, unlike with naturally_sort(). + assert items == ["map10", "map2", "Map1", "map1a"] + gutils.naturally_sort(items) + assert items == ["Map1", "map1a", "map2", "map10"] + + +def test_naturally_sorted_key(): + """Check that the key function selects the text used for sorting""" + items = [("b", "file10"), ("a", "file9")] + assert gutils.naturally_sorted(items, key=operator.itemgetter(1)) == [ + ("a", "file9"), + ("b", "file10"), + ] + + +@pytest.mark.parametrize( + ("number", "max_number", "expected"), + [(10, 1000, "0010"), (10, 10, "10"), (0, 9, "0"), (7, 100, "007")], +) +def test_get_num_suffix(number, max_number, expected): + """Check that the padding depends on the digits of the maximum number""" + assert gutils.get_num_suffix(number, max_number) == expected + + +def test_parse_key_val_bytes_and_separators(): + """Check parsing of bytes input and custom separators""" + assert gutils.parse_key_val(b"min=20\nmax=50") == {"min": "20", "max": "50"} + assert gutils.parse_key_val("min:20|max:50", sep=":", vsep="|") == { + "min": "20", + "max": "50", + } + + +def test_parse_key_val_missing_value_uses_default(): + """Check that keys without a separator get the default value""" + assert gutils.parse_key_val("a=1\nb", dflt="none") == {"a": "1", "b": "none"} + assert gutils.parse_key_val("a=1\nb", val_type=int, dflt=0) == {"a": 1, "b": 0} + assert gutils.parse_key_val("a=1\nb") == {"a": "1", "b": None} + + +def test_parse_key_val_splits_on_first_separator_only(): + """Check that a value may contain the separator itself""" + assert gutils.parse_key_val("expr=a=b") == {"expr": "a=b"} + assert gutils.parse_key_val("expr=a=b", val_type=str.upper) == {"expr": "A=B"} + + +def test_parse_key_val_empty_input(): + """Check that empty input results in an empty dictionary""" + assert gutils.parse_key_val("") == {} + assert gutils.parse_key_val(b"") == {} + + +def test_get_lib_path_from_grass_etc(monkeypatch, tmp_path): + """Check that the module directory in GRASS_ETCDIR is preferred""" + (tmp_path / "etc" / "mymod").mkdir(parents=True) + (tmp_path / "addon" / "etc" / "mymod").mkdir(parents=True) + monkeypatch.setenv("GRASS_ETCDIR", str(tmp_path / "etc")) + monkeypatch.setenv("GRASS_ADDON_BASE", str(tmp_path / "addon")) + assert gutils.get_lib_path("mymod") == str(tmp_path / "etc" / "mymod") + + +def test_get_lib_path_from_addon_base(monkeypatch, tmp_path): + """Check the addon locations used when the module is not in GRASS_ETCDIR""" + (tmp_path / "etc").mkdir() + monkeypatch.setenv("GRASS_ETCDIR", str(tmp_path / "etc")) + monkeypatch.setenv("GRASS_ADDON_BASE", str(tmp_path / "addon")) + + (tmp_path / "addon" / "etc" / "mymod").mkdir(parents=True) + assert gutils.get_lib_path("mymod") == str(tmp_path / "addon" / "etc" / "mymod") + + # Without the etc directory, the module directory inside the addon is used. + (tmp_path / "addon" / "etc" / "mymod").rmdir() + (tmp_path / "addon" / "mymod" / "mymod").mkdir(parents=True) + assert gutils.get_lib_path("mymod") == str(tmp_path / "addon" / "mymod" / "mymod") + + +def test_get_lib_path_not_found(monkeypatch, tmp_path): + """Check that None is returned when no location and no build directory match""" + monkeypatch.setenv("GRASS_ETCDIR", str(tmp_path)) + monkeypatch.delenv("GRASS_ADDON_BASE", raising=False) + monkeypatch.chdir(tmp_path) + assert gutils.get_lib_path("module_which_does_not_exist") is None + + +def test_set_path_local_directory(monkeypatch, tmp_path): + """Check that an existing local directory makes its parent importable""" + (tmp_path / "libmymod").mkdir() + monkeypatch.setattr(sys, "path", list(sys.path)) + gutils.set_path("mymod", "libmymod", str(tmp_path)) + assert sys.path[-1] == str(tmp_path) + + +def test_set_path_not_found(monkeypatch, tmp_path): + """Check that a library found neither locally nor in GRASS fails to import""" + monkeypatch.setenv("GRASS_ETCDIR", str(tmp_path)) + monkeypatch.delenv("GRASS_ADDON_BASE", raising=False) + monkeypatch.chdir(tmp_path) + with pytest.raises(ImportError, match="Not able to find"): + gutils.set_path("module_which_does_not_exist", "lib")