diff --git a/src/skillspector/nodes/analyzers/common.py b/src/skillspector/nodes/analyzers/common.py index cb38d6890..f8f7d0300 100644 --- a/src/skillspector/nodes/analyzers/common.py +++ b/src/skillspector/nodes/analyzers/common.py @@ -26,7 +26,7 @@ # Keep the analyzer and runner fence walkers lexically aligned without sharing # their state machines, since they consume different coordinate systems. -MARKDOWN_FENCE_OPEN = re.compile(r"^[ ]{0,3}(`{3,}|~{3,})[^\r\n]*$") +MARKDOWN_FENCE_OPEN = re.compile(r"^[ ]{0,3}(`{3,}(?=[^`\r\n]*$)|~{3,})[^\r\n]*$") MARKDOWN_FENCE_CLOSE = re.compile(r"^[ ]{0,3}(`{3,}|~{3,})[ \t]*$") LOGICAL_LINE_BREAK = re.compile(r"\r\n|[\r\n\v\f\x1c-\x1e\x85\u2028\u2029]") LINE_BREAK_CHARS = "\r\n\v\f\x1c\x1d\x1e\x85\u2028\u2029" diff --git a/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py b/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py index ce6a4bb50..edc6f0ab5 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py +++ b/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py @@ -31,10 +31,17 @@ from skillspector.logging_config import get_logger from skillspector.models import AnalyzerFinding, Location, Severity +from skillspector.security_reconstruction import validated_json_string_spans from skillspector.state import AnalyzerNodeResponse, SkillspectorState from . import static_runner -from .common import get_context, get_line_number +from .common import ( + LINE_BREAK_CHARS, + MARKDOWN_FENCE_CLOSE, + MARKDOWN_FENCE_OPEN, + get_context, + get_line_number, +) from .pattern_defaults import PatternCategory logger = get_logger(__name__) @@ -1450,6 +1457,9 @@ def _destructive_command_words(content: str) -> Iterator[tuple[int, int]]: def _has_shell_command_word_exhaustion( content: str, check_runtime: Callable[[], None], + *, + structural_quote_closers: set[int] | None = None, + structural_quote_openers: set[int] | None = None, ) -> bool: """Find candidate command words whose deterministic parse hit a safety bound.""" parsed_through = 0 @@ -1459,7 +1469,14 @@ def _has_shell_command_word_exhaustion( for candidate in _SHELL_COMMAND_WORD_START_RE.finditer(content): check_runtime() start = candidate.start() - if start < parsed_through or not _is_shell_command_word_start(content, start): + if structural_quote_closers is not None and start in structural_quote_closers: + continue + json_string_start = structural_quote_openers is not None and ( + start in structural_quote_openers or start - 1 in structural_quote_openers + ) + if start < parsed_through or ( + not json_string_start and not _is_shell_command_word_start(content, start) + ): continue if _has_quoted_assignment_prefix(content, start): continue @@ -2115,12 +2132,375 @@ def _tm1_candidates( yield command_start, command_end, command, 0.9 +def _markdown_block_separator(line: str, check_runtime: Callable[[], None]) -> bool: + """Recognize Setext underlines/thematic breaks with bounded, linear work.""" + line = line.strip(" \t") + marker = line[:1] + if marker not in {"=", "-", "*", "_"}: + return False + count = 0 + internal_gap = False + for index, character in enumerate(line): + if index % 256 == 0: + check_runtime() + if character == marker: + count += 1 + elif character in " \t" and marker != "=": + internal_gap = True + else: + return False + return count >= (3 if internal_gap or marker in {"*", "_"} else 1) + + +def _markdown_table_cells( + line: str, start: int, check_runtime: Callable[[], None] +) -> list[tuple[int, int]]: + """Locate GFM cells without copying content or changing source offsets.""" + end = len(line) + while start < end and line[start] in " \t": + if start % 256 == 0: + check_runtime() + start += 1 + while end > start and line[end - 1] in " \t": + if end % 256 == 0: + check_runtime() + end -= 1 + if start == end: + return [] + # Edge pipes are optional. In cmark-gfm a backslash immediately before a + # pipe escapes it even when that backslash follows another backslash. + if line[start] == "|": + start += 1 + if line[end - 1] == "|" and (end == 1 or line[end - 2] != "\\"): + end -= 1 + if start > end: + return [] + cells: list[tuple[int, int]] = [] + cell_start = start + for cursor in range(start, end): + if (cursor - start) % 256 == 0: + check_runtime() + if line[cursor] == "|" and (cursor == start or line[cursor - 1] != "\\"): + cells.append((cell_start, cursor)) + cell_start = cursor + 1 + cells.append((cell_start, end)) + return cells + + +def _markdown_table_delimiter_columns( + line: str, minimum_indent: int, check_runtime: Callable[[], None] +) -> int | None: + """Prove a compatible delimiter row under the existing container scope.""" + line = line.rstrip(LINE_BREAK_CHARS) + prefix = 0 + column = 0 + while prefix < len(line) and line[prefix] in " \t": + if prefix % 256 == 0: + check_runtime() + column += 4 - column % 4 if line[prefix] == "\t" else 1 + prefix += 1 + if column < minimum_indent or column >= 4 or prefix == len(line): + return None + if line[prefix] not in "|:-": + return None + # A new list item or a bare Setext/thematic underline takes precedence + # over table recognition, including a pipe-bearing preceding header. + if ( + line[prefix] == "-" and prefix + 1 < len(line) and line[prefix + 1] in " \t" + ) or _markdown_block_separator(line, check_runtime): + return None + cells = _markdown_table_cells(line, prefix, check_runtime) + if not cells: + return None + for start, end in cells: + check_runtime() + while start < end and line[start] in " \t": + if start % 256 == 0: + check_runtime() + start += 1 + while end > start and line[end - 1] in " \t": + if end % 256 == 0: + check_runtime() + end -= 1 + if start < end and line[start] == ":": + start += 1 + hyphen_start = start + while start < end and line[start] == "-": + if (start - hyphen_start) % 256 == 0: + check_runtime() + start += 1 + if start == hyphen_start: + return None + if start < end and line[start] == ":": + start += 1 + if start != end: + return None + return len(cells) + + +def _markdown_shell_text( + content: str, check_runtime: Callable[[], None], *, complete_context: bool = True +) -> str: + """Mask Markdown delimiters while retaining code and exact source offsets. + + Inline code delimiters are not legacy shell substitutions. Fenced and + indented code stays literal; longer inline delimiters preserve backticks + inside their bodies. Pair equal-length runs in linear time. + """ + output = list(content) + runs: list[tuple[int, int]] = [] + list_marker = re.compile(r"(?:[-+*]|[0-9]{1,9}[.)])(?=[ \t])") + backtick_runs = re.compile(r"`+") + + def mask_inline_delimiters() -> None: + if not complete_context: + runs.clear() + return + next_by_length: dict[int, int] = {} + closing: dict[int, int] = {} + for index in range(len(runs) - 1, -1, -1): + check_runtime() + start, end = runs[index] + length = end - start + if length in next_by_length: + closing[index] = next_by_length[length] + next_by_length[length] = index + index = 0 + while index < len(runs): + check_runtime() + start, end = runs[index] + escape_start = start + while escape_start > 0 and content[escape_start - 1] == "\\": + escape_start -= 1 + close_index = closing.get(index) + if (start - escape_start) % 2 or close_index is None: + index += 1 + continue + close_start, close_end = runs[close_index] + output[start:end] = " " * (end - start) + output[close_start:close_end] = " " * (close_end - close_start) + index = close_index + 1 + runs.clear() + + # This is a conservative projection, not a general Markdown renderer. + # Container/HTML bodies with uncertain inline ownership remain literal. + fence: tuple[str, int, int] | None = None + quoted_block = False + html_end: str | None = None + paragraph_open = False + paragraph_in_list = False + paragraph_list_indent = 0 + table_columns: int | None = None + table_minimum_indent = 0 + table_delimiter_line = -1 + offset = 0 + lines = content.splitlines(keepends=True) + for line_index, line in enumerate(lines): + check_runtime() + stripped = line.rstrip(LINE_BREAK_CHARS) + leading = stripped.lstrip(" \t") + indentation = len(stripped[: len(stripped) - len(leading)].expandtabs(4)) + # Interpret list padding in columns, preserving the original offsets. + # More than four columns after a marker can introduce indented code. + prefix = len(stripped) - len(leading) + column = indentation + list_indented = False + has_list_marker = False + list_markers = 0 + if indentation < 4: + while marker := list_marker.match(stripped, prefix): + check_runtime() + if ( + not has_list_marker + and paragraph_open + and not paragraph_in_list + and marker[0][0].isdigit() + and int(marker[0][:-1]) != 1 + ): + # Only a list starting at 1 can interrupt a paragraph. + # Other numbers may be literal text within an inline span. + break + has_list_marker = True + list_markers += 1 + column += marker.end() - prefix + prefix = marker.end() + padding_start = column + while prefix < len(stripped) and stripped[prefix] in " \t": + check_runtime() + column += 4 - column % 4 if stripped[prefix] == "\t" else 1 + prefix += 1 + if column - padding_start > 4: + list_indented = True + break + leading = stripped[prefix:] + quote_start = indentation < 4 and leading.startswith(">") + heading = indentation < 4 and re.match(r"#{1,6}(?:[ \t]|$)", leading) is not None + separator = indentation < 4 and _markdown_block_separator(stripped, check_runtime) + setext_only = separator and (leading.startswith("=") or leading.rstrip(" \t") == "--") + empty_list_item = ( + not paragraph_open and re.fullmatch(r"(?:[-+*]|[0-9]{1,9}[.)])", leading) is not None + ) + if table_columns is not None and setext_only: + # A table row has no paragraph for a Setext underline to close. + # Single '-' still starts an empty item; '---' remains thematic. + separator = False + html_open = re.match(r"<(?:[A-Za-z][A-Za-z0-9-]*(?=[\s/>]|$)|[!?/])", leading) + continuing_paragraph = False + if table_columns is not None and ( + html_end is not None + or fence is not None + or quote_start + or quoted_block + or html_open + or not leading + or indentation >= 4 + or indentation < table_minimum_indent + or list_indented + or has_list_marker + or empty_list_item + or heading + or separator + ): + table_columns = None + if html_end is not None: + mask_inline_delimiters() + if (html_end and html_end in leading.lower()) or (not html_end and not leading): + html_end = None + elif fence is not None: + fence_body = stripped.lstrip(" \t") + closing_fence = MARKDOWN_FENCE_CLOSE.fullmatch(fence_body) + if ( + closing_fence + and 0 <= indentation - fence[2] <= 3 + and closing_fence[1][0] == fence[0] + and len(closing_fence[1]) >= fence[1] + ): + begin, end = closing_fence.span(1) + body_start = offset + len(stripped) - len(fence_body) + output[body_start + begin : body_start + end] = " " * (end - begin) + fence = None + elif quote_start or quoted_block: + mask_inline_delimiters() + quoted_block = bool(leading) + elif html_open: + mask_inline_delimiters() + raw_tag = re.match(r"<(pre|script|style|textarea)(?=[\s/>]|$)", leading, re.I) + terminator = ( + f"" + if raw_tag + else "-->" + if leading.startswith("