diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py index 279c27708..306f7f86c 100644 --- a/crawl4ai/utils.py +++ b/crawl4ai/utils.py @@ -2552,12 +2552,18 @@ def is_external_url(url: str, base_domain: str) -> bool: if not parsed.netloc: # Relative URL return False - # Strip port and 'www.' from both domains for comparison - url_domain = parsed.netloc.lower().split(":")[0].replace("www.", "") - base = base_domain.lower().split(":")[0].replace("www.", "") - - # Check if URL domain ends with base domain - return not url_domain.endswith(base) + # Strip port and a leading 'www.' from both domains for comparison. + # removeprefix (not replace) so a host containing 'www.' somewhere + # other than the front is left untouched. + url_domain = parsed.netloc.lower().split(":")[0].removeprefix("www.") + base = base_domain.lower().split(":")[0].removeprefix("www.") + + # Internal means the base domain itself or one of its subdomains. A + # bare endswith(base) also matches look-alike domains that merely end + # with the base string (notexample.com for example.com), which are + # genuinely different sites an attacker can register. + is_internal = url_domain == base or url_domain.endswith(f".{base}") + return not is_internal except Exception: return False diff --git a/tests/regression/test_reg_utils.py b/tests/regression/test_reg_utils.py index dfc63c42d..f637689b4 100644 --- a/tests/regression/test_reg_utils.py +++ b/tests/regression/test_reg_utils.py @@ -15,6 +15,7 @@ efficient_normalize_url_for_deep_crawl, sanitize_input_encode, generate_content_hash, + is_external_url, ) from crawl4ai.cache_context import CacheContext, CacheMode @@ -498,3 +499,48 @@ def test_image_description_threshold_exists(self): """IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD should exist.""" from crawl4ai.config import IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD assert isinstance(IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD, (int, float)) + + +# =================================================================== +# is_external_url +# =================================================================== + +class TestIsExternalUrl: + """Verify is_external_url only treats the base domain and its subdomains + as internal, and does not mistake look-alike domains for internal ones.""" + + def test_base_domain_is_internal(self): + """The base domain itself is internal.""" + assert is_external_url("http://example.com", "example.com") is False + + def test_subdomain_is_internal(self): + """A real subdomain of the base domain is internal.""" + assert is_external_url("http://blog.example.com", "example.com") is False + + def test_www_is_internal(self): + """A leading www. is stripped, so www.example.com is internal.""" + assert is_external_url("http://www.example.com", "example.com") is False + + def test_prefixed_lookalike_is_external(self): + """A host that merely ends with the base string is a different site.""" + assert is_external_url("http://malicious-example.com", "example.com") is True + + def test_bare_lookalike_is_external(self): + """notexample.com is not a subdomain of example.com.""" + assert is_external_url("http://notexample.com/path", "example.com") is True + + def test_base_as_prefix_is_external(self): + """The base domain appearing as a prefix is a different site.""" + assert is_external_url("http://example.com.evil.com", "example.com") is True + + def test_unrelated_domain_is_external(self): + """An unrelated domain is external.""" + assert is_external_url("http://other.org", "example.com") is True + + def test_relative_url_is_internal(self): + """A relative URL has no netloc and is treated as internal.""" + assert is_external_url("/path/page", "example.com") is False + + def test_mailto_is_external(self): + """Non-http schemes such as mailto: are external.""" + assert is_external_url("mailto:hi@example.com", "example.com") is True