diff --git a/pyrit/converter/puzzled/keyword_masker.py b/pyrit/converter/puzzled/keyword_masker.py index a14fd5a557..a8bf310e96 100644 --- a/pyrit/converter/puzzled/keyword_masker.py +++ b/pyrit/converter/puzzled/keyword_masker.py @@ -248,7 +248,13 @@ def _get_nlp() -> Any: def _pos_lookup(prompt: str) -> dict[str, str]: """ - Map each alphabetic noun/verb in the prompt (by ``_word_key``) to its clue label. + Map each noun/verb in the prompt (by ``_word_key``) to its clue label. + + The prompt is normalized before tagging because the tagger is sensitive to the spelling + it receives: composed "café" and decomposed "cafe\u0301" are given different parts of + speech, which would make the same prompt select a different word. Words are accepted with + ``_MASKABLE_WORD_PATTERN``, the same test ``_iter_words`` uses, so a word carrying a + combining mark keeps its tag; ``token.is_alpha`` is false for all of them. Returns an empty mapping when spaCy is unavailable. @@ -262,8 +268,8 @@ def _pos_lookup(prompt: str) -> dict[str, str]: if nlp is None: return {} lookup: dict[str, str] = {} - for token in nlp(prompt): - if token.is_alpha and token.pos_ in _MASKABLE_POS: + for token in nlp(unicodedata.normalize("NFC", prompt)): + if _MASKABLE_WORD_PATTERN.fullmatch(token.text) and token.pos_ in _MASKABLE_POS: # Keep the first tag seen for a given surface form. lookup.setdefault(_word_key(token.text), _MASKABLE_POS[token.pos_]) return lookup diff --git a/tests/unit/converter/test_puzzled_keyword_masker.py b/tests/unit/converter/test_puzzled_keyword_masker.py index bbbaf4d824..fc0bfc2530 100644 --- a/tests/unit/converter/test_puzzled_keyword_masker.py +++ b/tests/unit/converter/test_puzzled_keyword_masker.py @@ -45,6 +45,22 @@ def __call__(self, text: str) -> list[_FakeToken]: return self._tokens +class _RecordingNLP: + """Records the text it is given and tags by exact spelling, the way the real tagger does. + + ``is_alpha`` is derived with ``str.isalpha`` exactly as spaCy derives it, so a word + carrying a combining mark reports False here too. + """ + + def __init__(self, tags: dict[str, str]): + self.seen: list[str] = [] + self._tags = tags + + def __call__(self, text: str) -> list[_FakeToken]: + self.seen.append(text) + return [_FakeToken(word, self._tags.get(word, "ADV"), is_alpha=word.isalpha()) for word in text.split()] + + # --- mask count rule ------------------------------------------------------- @@ -147,6 +163,38 @@ def test_mask_prompt_prefers_pos_tagged_words_over_plain_words(): assert {w.text for w in result.masked_words} == {"steal", "documents"} +def test_pos_lookup_tags_words_with_combining_marks(): + # is_alpha comes from str.isalpha, which is False once a combining mark is present, so + # the tag the pipeline produced would otherwise be dropped. Built at runtime so an editor + # that normalizes the file to NFC cannot silently change the test. + nfd = unicodedata.normalize("NFD", "café") + fake = _FakeNLP([_FakeToken(nfd, "NOUN", is_alpha=False)]) + with patch.object(keyword_masker, "_get_nlp", lambda: fake): + assert keyword_masker._pos_lookup(nfd) == {keyword_masker._word_key(nfd): "noun"} + + +def test_pos_lookup_normalizes_before_tagging(): + nfd = unicodedata.normalize("NFD", "café") + fake = _RecordingNLP({}) + with patch.object(keyword_masker, "_get_nlp", lambda: fake): + keyword_masker._pos_lookup(nfd) + assert fake.seen == [unicodedata.normalize("NFC", nfd)] + + +def test_mask_prompt_selection_is_stable_across_normalization_forms(): + # The fake tags "café" only in its composed spelling, which is how the real tagger + # behaves: it gives the decomposed spelling a different part of speech, and the tier in + # _rank_candidates then picks a different word. + selections = {} + for form in ("NFC", "NFD"): + fake = _RecordingNLP({"café": "NOUN"}) + with patch.object(keyword_masker, "_get_nlp", lambda nlp=fake: nlp): + result = mask_prompt(unicodedata.normalize(form, "surprisingly café"), num_to_mask=1) + word = result.masked_words[0] + selections[form] = (unicodedata.normalize("NFC", word.text), word.clue) + assert selections["NFC"] == selections["NFD"] == ("café", "4-letter noun") + + @_no_spacy def test_mask_prompt_prefers_longer_words_without_hints(): result = mask_prompt("cat elephant dog", num_to_mask=1)