From f5db4e2ea457913c8b5aa5e8cbb787c5b35303c1 Mon Sep 17 00:00:00 2001 From: lovit Date: Wed, 11 Mar 2026 02:07:25 +0900 Subject: [PATCH] =?UTF-8?q?fix(tokenizer):=20tokenize=20offset=20=EA=B3=84?= =?UTF-8?q?=EC=82=B0=EC=9D=B4=20=EB=8B=A4=EC=A4=91=20=EA=B3=B5=EB=B0=B1?= =?UTF-8?q?=EC=97=90=EC=84=9C=20=ED=8B=80=EB=A6=AC=EB=8A=94=20=EB=B2=84?= =?UTF-8?q?=EA=B7=B8=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sentence.split() + offset += len + 1 방식은 연속 공백 시 Token position 오류. re.finditer(r'\S+', sentence)로 교체하여 각 eojeol의 실제 시작 위치를 직접 추출. 단위 테스트 추가. Co-Authored-By: Claude Sonnet 4.6 --- soynlp/tokenizer/tokenizer.py | 6 ++---- tests/unit/test_tokenizers.py | 9 +++++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/soynlp/tokenizer/tokenizer.py b/soynlp/tokenizer/tokenizer.py index f48deb9..4094724 100644 --- a/soynlp/tokenizer/tokenizer.py +++ b/soynlp/tokenizer/tokenizer.py @@ -108,11 +108,9 @@ def tokenize(self, sentence, return_words=True): Token(hank`s, score=1, position=(27, 33), eojeol_id=2), Token(report, score=1, position=(34, 40), eojeol_id=3)] """ - offset = 0 tokens = [] - for eojeol_id, token in enumerate(sentence.split()): - tokens += self._tokenize(token, offset, eojeol_id) - offset += len(token) + 1 + for eojeol_id, m in enumerate(re.finditer(r"\S+", sentence)): + tokens += self._tokenize(m.group(), m.start(), eojeol_id) if return_words: tokens = [token.word for token in tokens] return tokens diff --git a/tests/unit/test_tokenizers.py b/tests/unit/test_tokenizers.py index fefe765..b2b45fc 100644 --- a/tests/unit/test_tokenizers.py +++ b/tests/unit/test_tokenizers.py @@ -4,6 +4,7 @@ from soynlp.tokenizer.tokenizer_builder import EojeolPatternTrainer +<<<<<<< HEAD def test_regex_tokenizer_empty_string(): tokenizer = RegexTokenizer() assert tokenizer.tokenize("") == [] @@ -19,6 +20,14 @@ def test_regex_tokenizer_version_number(): assert tokenizer.tokenize("42") == ["42"] +def test_regex_tokenizer_multi_space_offset(): + # 연속 공백이 있을 때 Token의 position이 정확해야 한다 (issue #280) + tokenizer = RegexTokenizer() + tokens = tokenizer.tokenize("hello world", return_words=False) + assert tokens[0].word == "hello" and tokens[0].begin == 0 and tokens[0].end == 5 + assert tokens[1].word == "world" and tokens[1].begin == 7 and tokens[1].end == 12 + + def test_regex_tokenizer(): sentence = "abc123가나다 alphabet!!3.14한글 hank`s report" expected_words = ["abc", "123", "가나다", "alphabet", "!!", "3.14", "한글", "hank`s", "report"]