Skip to content

Commit c2d2956

Browse files
ericapisaniericapisani
authored
feat(stdlib): Support data_collection filtering for URL query params (#7291)
Previously the stdlib httplib integration only gated url.full, url.query and url.fragment on send_default_pii, so with data_collection configured the URL data was dropped entirely and no allow/denylist filtering was ever applied. The data_collection experiment's url_query_params behaviour is now applied to span streaming spans and breadcrumbs, matching httpx and pyreqwest. url.full is now reassembled with the filtered query and fragment, and empty url.query/url.fragment attributes are no longer emitted. The legacy (non span-streaming) path is left unchanged. Fixes PY-2744 Fixes #7279
1 parent 3826c55 commit c2d2956

8 files changed

Lines changed: 303 additions & 207 deletions

File tree

‎sentry_sdk/integrations/boto3.py‎

Lines changed: 2 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -3,22 +3,18 @@
33

44
import sentry_sdk
55
from sentry_sdk.consts import OP, SPANDATA
6-
from sentry_sdk.data_collection import (
7-
_apply_data_collection_filtering_to_query_string,
8-
)
96
from sentry_sdk.integrations import DidNotEnable, Integration, _check_minimum_version
10-
from sentry_sdk.scope import should_send_default_pii
117
from sentry_sdk.traces import StreamedSpan
128
from sentry_sdk.tracing import BAGGAGE_HEADER_NAME, Span
139
from sentry_sdk.tracing_utils import (
1410
add_http_breadcrumb,
1511
add_sentry_baggage_to_headers,
12+
get_url_attributes,
1613
has_span_streaming_enabled,
1714
should_propagate_trace,
1815
)
1916
from sentry_sdk.utils import (
2017
capture_internal_exceptions,
21-
has_data_collection_enabled,
2218
parse_url,
2319
parse_version,
2420
)
@@ -28,9 +24,6 @@
2824

2925
from botocore.model import ServiceId
3026

31-
from sentry_sdk._types import Attributes
32-
from sentry_sdk.client import BaseClient as SentryClient
33-
from sentry_sdk.utils import ParsedUrl
3427

3528
try:
3629
from botocore import __version__ as BOTOCORE_VERSION
@@ -70,40 +63,6 @@ def sentry_patched_init(
7063
BaseClient.__init__ = sentry_patched_init # type: ignore
7164

7265

73-
def _get_url_attributes(
74-
client: "SentryClient", parsed_url: "Optional[ParsedUrl]"
75-
) -> "Attributes":
76-
attributes: "Attributes" = {}
77-
if parsed_url is None:
78-
return attributes
79-
80-
query: "Optional[str]"
81-
if has_data_collection_enabled(client.options):
82-
query = None
83-
if parsed_url.query:
84-
query = _apply_data_collection_filtering_to_query_string(
85-
query_string=parsed_url.query,
86-
behaviour=client.options["data_collection"]["url_query_params"],
87-
)
88-
elif should_send_default_pii():
89-
query = parsed_url.query
90-
else:
91-
return attributes
92-
93-
url_full = parsed_url.url
94-
if query:
95-
attributes[SPANDATA.URL_QUERY] = query
96-
url_full += "?" + query
97-
98-
if parsed_url.fragment:
99-
attributes[SPANDATA.URL_FRAGMENT] = parsed_url.fragment
100-
url_full += "#" + parsed_url.fragment
101-
102-
attributes[SPANDATA.URL_FULL] = url_full
103-
104-
return attributes
105-
106-
10766
def _sentry_request_created(
10867
service_id: "ServiceId", request: "AWSRequest", operation_name: str, **kwargs: "Any"
10968
) -> None:
@@ -123,7 +82,7 @@ def _sentry_request_created(
12382
is_span_streaming_enabled = has_span_streaming_enabled(client.options)
12483
span: "Union[Span, StreamedSpan, None]" = None
12584
if is_span_streaming_enabled:
126-
url_attributes = _get_url_attributes(client, parsed_url)
85+
url_attributes = get_url_attributes(client, parsed_url)
12786
breadcrumb.update(url_attributes)
12887

12988
if request.method is not None:

‎sentry_sdk/integrations/httpx.py‎

Lines changed: 4 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -2,31 +2,25 @@
22

33
import sentry_sdk
44
from sentry_sdk.consts import OP, SPANDATA
5-
from sentry_sdk.data_collection import (
6-
_apply_data_collection_filtering_to_query_string,
7-
)
85
from sentry_sdk.integrations import DidNotEnable, Integration
9-
from sentry_sdk.scope import should_send_default_pii
106
from sentry_sdk.tracing_utils import (
117
add_http_breadcrumb,
128
add_http_request_source,
9+
get_url_attributes,
1310
has_span_streaming_enabled,
1411
propagate_trace_headers,
1512
)
1613
from sentry_sdk.utils import (
1714
SENSITIVE_DATA_SUBSTITUTE,
1815
capture_internal_exceptions,
1916
ensure_integration_enabled,
20-
has_data_collection_enabled,
2117
parse_url,
2218
)
2319

2420
if TYPE_CHECKING:
25-
from typing import Any, Optional
21+
from typing import Any
2622

2723
from sentry_sdk._types import Attributes
28-
from sentry_sdk.client import BaseClient
29-
from sentry_sdk.utils import ParsedUrl
3024

3125

3226
try:
@@ -51,45 +45,6 @@ def setup_once() -> None:
5145
_install_httpx_async_client()
5246

5347

54-
def _get_url_attributes(
55-
client: "BaseClient", parsed_url: "Optional[ParsedUrl]"
56-
) -> "Attributes":
57-
attributes: "Attributes" = {}
58-
if parsed_url is None:
59-
return attributes
60-
61-
url_full = parsed_url.url
62-
63-
if has_data_collection_enabled(client.options):
64-
if parsed_url.query:
65-
filtered_query = _apply_data_collection_filtering_to_query_string(
66-
query_string=parsed_url.query,
67-
behaviour=client.options["data_collection"]["url_query_params"],
68-
)
69-
if filtered_query:
70-
attributes["url.query"] = filtered_query
71-
url_full += "?" + filtered_query
72-
73-
if parsed_url.fragment:
74-
attributes["url.fragment"] = parsed_url.fragment
75-
url_full += "#" + parsed_url.fragment
76-
77-
attributes["url.full"] = url_full
78-
79-
elif should_send_default_pii():
80-
if parsed_url.query:
81-
attributes["url.query"] = parsed_url.query
82-
url_full += "?" + parsed_url.query
83-
84-
if parsed_url.fragment:
85-
attributes["url.fragment"] = parsed_url.fragment
86-
url_full += "#" + parsed_url.fragment
87-
88-
attributes["url.full"] = url_full
89-
90-
return attributes
91-
92-
9348
def _install_httpx_client() -> None:
9449
real_send = Client.send
9550

@@ -109,7 +64,7 @@ def send(self: "Client", request: "Request", **kwargs: "Any") -> "Response":
10964
propagate_trace_headers(client, request)
11065
return real_send(self, request, **kwargs)
11166

112-
url_attributes = _get_url_attributes(client, parsed_url)
67+
url_attributes = get_url_attributes(client, parsed_url)
11368

11469
with sentry_sdk.traces.start_span(
11570
name="%s %s"
@@ -218,7 +173,7 @@ async def send(
218173
propagate_trace_headers(client, request)
219174
return await real_send(self, request, **kwargs)
220175

221-
url_attributes = _get_url_attributes(client, parsed_url)
176+
url_attributes = get_url_attributes(client, parsed_url)
222177

223178
with sentry_sdk.traces.start_span(
224179
name="%s %s"

‎sentry_sdk/integrations/httpx2.py‎

Lines changed: 4 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -2,31 +2,25 @@
22

33
import sentry_sdk
44
from sentry_sdk.consts import OP, SPANDATA
5-
from sentry_sdk.data_collection import (
6-
_apply_data_collection_filtering_to_query_string,
7-
)
85
from sentry_sdk.integrations import DidNotEnable, Integration
9-
from sentry_sdk.scope import should_send_default_pii
106
from sentry_sdk.tracing_utils import (
117
add_http_breadcrumb,
128
add_http_request_source,
9+
get_url_attributes,
1310
has_span_streaming_enabled,
1411
propagate_trace_headers,
1512
)
1613
from sentry_sdk.utils import (
1714
SENSITIVE_DATA_SUBSTITUTE,
1815
capture_internal_exceptions,
1916
ensure_integration_enabled,
20-
has_data_collection_enabled,
2117
parse_url,
2218
)
2319

2420
if TYPE_CHECKING:
25-
from typing import Any, Optional
21+
from typing import Any
2622

2723
from sentry_sdk._types import Attributes
28-
from sentry_sdk.client import BaseClient
29-
from sentry_sdk.utils import ParsedUrl
3024

3125

3226
try:
@@ -51,45 +45,6 @@ def setup_once() -> None:
5145
_install_httpx2_async_client()
5246

5347

54-
def _get_url_attributes(
55-
client: "BaseClient", parsed_url: "Optional[ParsedUrl]"
56-
) -> "Attributes":
57-
attributes: "Attributes" = {}
58-
if parsed_url is None:
59-
return attributes
60-
61-
url_full = parsed_url.url
62-
63-
if has_data_collection_enabled(client.options):
64-
if parsed_url.query:
65-
filtered_query = _apply_data_collection_filtering_to_query_string(
66-
query_string=parsed_url.query,
67-
behaviour=client.options["data_collection"]["url_query_params"],
68-
)
69-
if filtered_query:
70-
attributes["url.query"] = filtered_query
71-
url_full += "?" + filtered_query
72-
73-
if parsed_url.fragment:
74-
attributes["url.fragment"] = parsed_url.fragment
75-
url_full += "#" + parsed_url.fragment
76-
77-
attributes["url.full"] = url_full
78-
79-
elif should_send_default_pii():
80-
if parsed_url.query:
81-
attributes["url.query"] = parsed_url.query
82-
url_full += "?" + parsed_url.query
83-
84-
if parsed_url.fragment:
85-
attributes["url.fragment"] = parsed_url.fragment
86-
url_full += "#" + parsed_url.fragment
87-
88-
attributes["url.full"] = url_full
89-
90-
return attributes
91-
92-
9348
def _install_httpx2_client() -> None:
9449
real_send = Client.send
9550

@@ -109,7 +64,7 @@ def send(self: "Client", request: "Request", **kwargs: "Any") -> "Response":
10964
propagate_trace_headers(client, request)
11065
return real_send(self, request, **kwargs)
11166

112-
url_attributes = _get_url_attributes(client, parsed_url)
67+
url_attributes = get_url_attributes(client, parsed_url)
11368

11469
with sentry_sdk.traces.start_span(
11570
name="%s %s"
@@ -218,7 +173,7 @@ async def send(
218173
propagate_trace_headers(client, request)
219174
return await real_send(self, request, **kwargs)
220175

221-
url_attributes = _get_url_attributes(client, parsed_url)
176+
url_attributes = get_url_attributes(client, parsed_url)
222177

223178
with sentry_sdk.traces.start_span(
224179
name="%s %s"

‎sentry_sdk/integrations/pyreqwest.py‎

Lines changed: 3 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -4,25 +4,21 @@
44
import sentry_sdk
55
from sentry_sdk import start_span
66
from sentry_sdk.consts import OP, SPANDATA
7-
from sentry_sdk.data_collection import (
8-
_apply_data_collection_filtering_to_query_string,
9-
)
107
from sentry_sdk.integrations import DidNotEnable, Integration
11-
from sentry_sdk.scope import should_send_default_pii
128
from sentry_sdk.traces import StreamedSpan
139
from sentry_sdk.tracing import BAGGAGE_HEADER_NAME
1410
from sentry_sdk.tracing_utils import (
1511
add_http_breadcrumb,
1612
add_http_request_source,
1713
add_sentry_baggage_to_headers,
14+
get_url_attributes,
1815
has_span_streaming_enabled,
1916
propagate_trace_headers,
2017
should_propagate_trace,
2118
)
2219
from sentry_sdk.utils import (
2320
SENSITIVE_DATA_SUBSTITUTE,
2421
capture_internal_exceptions,
25-
has_data_collection_enabled,
2622
logger,
2723
parse_url,
2824
)
@@ -31,7 +27,6 @@
3127
from typing import Optional
3228

3329
from sentry_sdk._types import Attributes
34-
from sentry_sdk.client import BaseClient
3530
from sentry_sdk.utils import ParsedUrl
3631

3732
try:
@@ -97,40 +92,6 @@ def sentry_patched_method(self: "Any", *args: "Any", **kwargs: "Any") -> "Any":
9792
setattr(cls, method_name, sentry_patched_method)
9893

9994

100-
def _get_url_attributes(
101-
client: "BaseClient", parsed_url: "Optional[ParsedUrl]"
102-
) -> "Attributes":
103-
attributes: "Attributes" = {}
104-
if parsed_url is None:
105-
return attributes
106-
107-
query: "Optional[str]"
108-
if has_data_collection_enabled(client.options):
109-
query = None
110-
if parsed_url.query:
111-
query = _apply_data_collection_filtering_to_query_string(
112-
query_string=parsed_url.query,
113-
behaviour=client.options["data_collection"]["url_query_params"],
114-
)
115-
elif should_send_default_pii():
116-
query = parsed_url.query
117-
else:
118-
return attributes
119-
120-
url_full = parsed_url.url
121-
if query:
122-
attributes[SPANDATA.URL_QUERY] = query
123-
url_full += "?" + query
124-
125-
if parsed_url.fragment:
126-
attributes[SPANDATA.URL_FRAGMENT] = parsed_url.fragment
127-
url_full += "#" + parsed_url.fragment
128-
129-
attributes[SPANDATA.URL_FULL] = url_full
130-
131-
return attributes
132-
133-
13495
def _get_breadcrumb_url_data(
13596
parsed_url: "Optional[ParsedUrl]", url_attributes: "Attributes"
13697
) -> "dict[str, Any]":
@@ -233,7 +194,7 @@ async def sentry_async_middleware(
233194
# after the request has been sent
234195
parsed_url = parse_url(str(request.url), sanitize=False)
235196

236-
url_attributes = _get_url_attributes(sentry_sdk.get_client(), parsed_url)
197+
url_attributes = get_url_attributes(sentry_sdk.get_client(), parsed_url)
237198

238199
response = None
239200
with _sentry_pyreqwest_span(request, url_attributes) as span:
@@ -273,7 +234,7 @@ def sentry_sync_middleware(
273234
# after the request has been sent
274235
parsed_url = parse_url(str(request.url), sanitize=False)
275236

276-
url_attributes = _get_url_attributes(sentry_sdk.get_client(), parsed_url)
237+
url_attributes = get_url_attributes(sentry_sdk.get_client(), parsed_url)
277238

278239
response = None
279240
with _sentry_pyreqwest_span(request, url_attributes) as span:

0 commit comments

Comments
 (0)