From 1895229e200bd4aa9a594319664f4b3259a1d10b Mon Sep 17 00:00:00 2001 From: Klaas Hoekema Date: Mon, 9 Oct 2017 15:57:30 -0400 Subject: [PATCH 1/3] Use separator and remove special status of hyphen Though the separator can be overridden, this has been using the default separator ('-') behind the scenes then replacing it with the chosen separator as the last step. This means, among other things, that hyphens would always be replaced with the separator, even if you overrode the allowed characters regex to try to allow them. This changes `slugify` to use the provided `separator` throughout, which required some changes to the allowed-characters and multiple-separators logic. --- slugify/slugify.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/slugify/slugify.py b/slugify/slugify.py index af0c609..306f308 100644 --- a/slugify/slugify.py +++ b/slugify/slugify.py @@ -23,10 +23,9 @@ DECIMAL_PATTERN = re.compile('&#(\d+);') HEX_PATTERN = re.compile('&#x([\da-fA-F]+);') QUOTE_PATTERN = re.compile(r'[\']+') -ALLOWED_CHARS_PATTERN = re.compile(r'[^-a-z0-9]+') -DUPLICATE_DASH_PATTERN = re.compile('-{2,}') NUMBERS_PATTERN = re.compile('(?<=\d),(?=\d)') DEFAULT_SEPARATOR = '-' +ALLOWED_CHARS = 'a-z0-9' def smart_truncate(string, max_length=0, word_boundaries=False, separator=' ', save_order=False): @@ -93,7 +92,7 @@ def slugify(text, entities=True, decimal=True, hexadecimal=True, max_length=0, w text = _unicode(text, 'utf-8', 'ignore') # replace quotes with dashes - pre-process - text = QUOTE_PATTERN.sub(DEFAULT_SEPARATOR, text) + text = QUOTE_PATTERN.sub(separator, text) # decode unicode text = unidecode.unidecode(text) @@ -135,24 +134,21 @@ def slugify(text, entities=True, decimal=True, hexadecimal=True, max_length=0, w text = NUMBERS_PATTERN.sub('', text) # replace all other unwanted characters - pattern = regex_pattern or ALLOWED_CHARS_PATTERN - text = re.sub(pattern, DEFAULT_SEPARATOR, text) + pattern = regex_pattern or re.compile(r'[^{0}{1}]+'.format(separator, ALLOWED_CHARS)) + text = re.sub(pattern, separator, text) - # remove redundant - text = DUPLICATE_DASH_PATTERN.sub(DEFAULT_SEPARATOR, text).strip(DEFAULT_SEPARATOR) + # collapse multiple separators + text = re.sub(re.escape(separator) + r'{2,}', separator, text).strip(separator) # remove stopwords if stopwords: stopwords_lower = [s.lower() for s in stopwords] - words = [w for w in text.split(DEFAULT_SEPARATOR) if w not in stopwords_lower] - text = DEFAULT_SEPARATOR.join(words) + words = [w for w in text.split(separator) if w not in stopwords_lower] + text = separator.join(words) # smart truncate if requested if max_length > 0: - text = smart_truncate(text, max_length, word_boundary, DEFAULT_SEPARATOR, save_order) - - if separator != DEFAULT_SEPARATOR: - text = text.replace(DEFAULT_SEPARATOR, separator) + text = smart_truncate(text, max_length, word_boundary, separator, save_order) return text From 1f6fa9b16f76707793e72056d2a1a557f013431f Mon Sep 17 00:00:00 2001 From: Klaas Hoekema Date: Mon, 9 Oct 2017 16:02:25 -0400 Subject: [PATCH 2/3] Adjust existing tests for new separator behavior Since `slugify` is no longer using '-' as a separator behind the scenes and switching at the last minute, the "collapse multiple separators" and "truncate" steps are now operating on the text with the chosen separator rather than on one with hyphens. Which changes the expected output of a couple tests. - test_multi_character_separator is different because the multi-character separator now counts against the length calculation, so it had to drop one piece. - test_regex_pattern_keep_underscore_with_underscore_as_separator now strips leading and trailing underscores, rather than just collapsing them. To me the new behavior seems more correct. --- test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.py b/test.py index 6730756..fe87507 100644 --- a/test.py +++ b/test.py @@ -89,7 +89,7 @@ def test_custom_separator(self): def test_multi_character_separator(self): txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=20, word_boundary=True, separator="ZZZZZZ") - self.assertEqual(r, "jajaZZZZZZlolZZZZZZmememeooZZZZZZa") + self.assertEqual(r, "jajaZZZZZZlolZZZZZZa") def test_save_order(self): txt = 'one two three four five' @@ -182,7 +182,7 @@ def test_regex_pattern_keep_underscore_with_underscore_as_separator(self): txt = "___This is a test___" regex_pattern = r'[^-a-z0-9_]+' r = slugify(txt, separator='_', regex_pattern=regex_pattern) - self.assertNotEqual(r, "_this_is_a_test_") + self.assertEqual(r, "this_is_a_test") class TestUtils(unittest.TestCase): From 0bff01fffc7d9f6c26b67f1c96e449ec0820c4c8 Mon Sep 17 00:00:00 2001 From: Klaas Hoekema Date: Mon, 9 Oct 2017 16:07:09 -0400 Subject: [PATCH 3/3] Add tests for revised hyphen behavior Adds two tests, both with an overridden separator, to exercise the new behavior around hyphens. 'test_separator_remove_hyphens' will behave the same as it would have before, but for different reasons. 'test_regex_pattern_allow_hyphens' will now pass, when it would have failed (replaced hyphens even though they're supposed to be allowed) before. --- test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test.py b/test.py index fe87507..ce26842 100644 --- a/test.py +++ b/test.py @@ -184,6 +184,19 @@ def test_regex_pattern_keep_underscore_with_underscore_as_separator(self): r = slugify(txt, separator='_', regex_pattern=regex_pattern) self.assertEqual(r, "this_is_a_test") + def test_separator_remove_hyphens(self): + """With an overridden separator, hyphens should be stripped.""" + txt = "___This is a hyphen-containing test." + r = slugify(txt, separator='_') + self.assertEqual(r, "this_is_a_hyphen_containing_test") + + def test_regex_pattern_allow_hyphens(self): + """With an overridden separator and a regex, it should be possible to preserve hyphens.""" + txt = "___This is a hyphen-containing test." + regex_pattern = r'[^-a-z0-9_]+' + r = slugify(txt, separator='_', regex_pattern=regex_pattern) + self.assertEqual(r, "this_is_a_hyphen-containing_test") + class TestUtils(unittest.TestCase):