diff --git a/slugify/slugify.py b/slugify/slugify.py index af0c609..306f308 100644 --- a/slugify/slugify.py +++ b/slugify/slugify.py @@ -23,10 +23,9 @@ DECIMAL_PATTERN = re.compile('&#(\d+);') HEX_PATTERN = re.compile('&#x([\da-fA-F]+);') QUOTE_PATTERN = re.compile(r'[\']+') -ALLOWED_CHARS_PATTERN = re.compile(r'[^-a-z0-9]+') -DUPLICATE_DASH_PATTERN = re.compile('-{2,}') NUMBERS_PATTERN = re.compile('(?<=\d),(?=\d)') DEFAULT_SEPARATOR = '-' +ALLOWED_CHARS = 'a-z0-9' def smart_truncate(string, max_length=0, word_boundaries=False, separator=' ', save_order=False): @@ -93,7 +92,7 @@ def slugify(text, entities=True, decimal=True, hexadecimal=True, max_length=0, w text = _unicode(text, 'utf-8', 'ignore') # replace quotes with dashes - pre-process - text = QUOTE_PATTERN.sub(DEFAULT_SEPARATOR, text) + text = QUOTE_PATTERN.sub(separator, text) # decode unicode text = unidecode.unidecode(text) @@ -135,24 +134,21 @@ def slugify(text, entities=True, decimal=True, hexadecimal=True, max_length=0, w text = NUMBERS_PATTERN.sub('', text) # replace all other unwanted characters - pattern = regex_pattern or ALLOWED_CHARS_PATTERN - text = re.sub(pattern, DEFAULT_SEPARATOR, text) + pattern = regex_pattern or re.compile(r'[^{0}{1}]+'.format(separator, ALLOWED_CHARS)) + text = re.sub(pattern, separator, text) - # remove redundant - text = DUPLICATE_DASH_PATTERN.sub(DEFAULT_SEPARATOR, text).strip(DEFAULT_SEPARATOR) + # collapse multiple separators + text = re.sub(re.escape(separator) + r'{2,}', separator, text).strip(separator) # remove stopwords if stopwords: stopwords_lower = [s.lower() for s in stopwords] - words = [w for w in text.split(DEFAULT_SEPARATOR) if w not in stopwords_lower] - text = DEFAULT_SEPARATOR.join(words) + words = [w for w in text.split(separator) if w not in stopwords_lower] + text = separator.join(words) # smart truncate if requested if max_length > 0: - text = smart_truncate(text, max_length, word_boundary, DEFAULT_SEPARATOR, save_order) - - if separator != DEFAULT_SEPARATOR: - text = text.replace(DEFAULT_SEPARATOR, separator) + text = smart_truncate(text, max_length, word_boundary, separator, save_order) return text diff --git a/test.py b/test.py index 6730756..ce26842 100644 --- a/test.py +++ b/test.py @@ -89,7 +89,7 @@ def test_custom_separator(self): def test_multi_character_separator(self): txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=20, word_boundary=True, separator="ZZZZZZ") - self.assertEqual(r, "jajaZZZZZZlolZZZZZZmememeooZZZZZZa") + self.assertEqual(r, "jajaZZZZZZlolZZZZZZa") def test_save_order(self): txt = 'one two three four five' @@ -182,7 +182,20 @@ def test_regex_pattern_keep_underscore_with_underscore_as_separator(self): txt = "___This is a test___" regex_pattern = r'[^-a-z0-9_]+' r = slugify(txt, separator='_', regex_pattern=regex_pattern) - self.assertNotEqual(r, "_this_is_a_test_") + self.assertEqual(r, "this_is_a_test") + + def test_separator_remove_hyphens(self): + """With an overridden separator, hyphens should be stripped.""" + txt = "___This is a hyphen-containing test." + r = slugify(txt, separator='_') + self.assertEqual(r, "this_is_a_hyphen_containing_test") + + def test_regex_pattern_allow_hyphens(self): + """With an overridden separator and a regex, it should be possible to preserve hyphens.""" + txt = "___This is a hyphen-containing test." + regex_pattern = r'[^-a-z0-9_]+' + r = slugify(txt, separator='_', regex_pattern=regex_pattern) + self.assertEqual(r, "this_is_a_hyphen-containing_test") class TestUtils(unittest.TestCase):