diff --git a/bugbug/bug_features.py b/bugbug/bug_features.py index c494540ba9..747fa3c4f8 100644 --- a/bugbug/bug_features.py +++ b/bugbug/bug_features.py @@ -620,6 +620,54 @@ def get_author_ids(): return author_ids +PATH_RE = re.compile(r"(?:[\w.-]+/)+[\w.-]+\.[a-zA-Z0-9]+") + + +def find_component(path, component_mapping): + # A path mentioned in a comment might not match a repository path exactly + # (e.g. it could have extra leading directories, like a local checkout + # path), so try the path itself and then progressively shorter suffixes + # of it, keeping the longest one that matches. + parts = path.split(b"/") + for i in range(len(parts)): + candidate = b"/".join(parts[i:]) + if candidate in component_mapping: + return component_mapping[candidate].tobytes().decode("utf-8") + + return None + + +def extract_path_components(comments): + paths = [ + match.group().encode("utf-8") + for comment in comments + for match in PATH_RE.finditer(comment["text"]) + ] + component_mapping = repository.get_component_mapping() + components = (find_component(path, component_mapping) for path in paths) + return [component for component in components if component is not None] + + +# Components of paths mentioned in bug comments. +# In case of stack traces, the position is important (e.g. component of the first path is more indicative than component of the last path, +# which tends to be a "main"-like entry point common to most stack traces). We weight each mention by the reciprocal of its rank among all +# paths mentioned in the comments, and sum the weights of repeated mentions of the same component. + + +class CommentFirstPathComponent(SingleBugFeature): + def __call__(self, bug, **kwargs): + components = extract_path_components(bug["comments"]) + return components[0] if components else None + + +class CommentPathsComponents(SingleBugFeature): + def __call__(self, bug, **kwargs): + weights = {} + for i, component in enumerate(extract_path_components(bug["comments"])): + weights[component] = weights.get(component, 0) + 1 / (i + 1) + return weights + + class BugExtractor(BaseEstimator, TransformerMixin): def __init__( self, @@ -680,6 +728,11 @@ def apply_transform(bug): data[sys.intern(f"{item} in {feature_extractor_name}")] = True continue + if isinstance(res, dict): + for item, weight in res.items(): + data[sys.intern(f"{item} in {feature_extractor_name}")] = weight + continue + data[feature_extractor_name] = res reporter_experience_map[bug["creator"]] += 1 diff --git a/bugbug/model.py b/bugbug/model.py index c1fccb768a..e168a2afae 100644 --- a/bugbug/model.py +++ b/bugbug/model.py @@ -7,7 +7,7 @@ import pickle from collections import defaultdict from os import makedirs, path -from typing import Any +from typing import Any, Callable import matplotlib import numpy as np @@ -180,6 +180,8 @@ def __init__(self, lemmatization=False): self.training_dbs: list[str] = [] # DBs and DB support files required at runtime. self.eval_dbs: dict[str, tuple[str, ...]] = {} + # Non-DB data files required for training (e.g. LMDB mappings), downloaded on demand. + self.training_extra_downloads: list[Callable[[], None]] = [] self.le = LabelEncoder() diff --git a/bugbug/models/component.py b/bugbug/models/component.py index a31954ca02..f5ca40e3ef 100644 --- a/bugbug/models/component.py +++ b/bugbug/models/component.py @@ -14,7 +14,7 @@ from sklearn.feature_extraction import DictVectorizer from sklearn.pipeline import Pipeline -from bugbug import bug_features, bugzilla, feature_cleanup, utils +from bugbug import bug_features, bugzilla, feature_cleanup, repository, utils from bugbug.bugzilla import get_product_component_count from bugbug.model import BugModel from bugbug.model_calibration import IsotonicRegressionCalibrator @@ -74,6 +74,8 @@ def __init__(self, calibration=True, lemmatization=False): self.cross_validation_enabled = False self.calculate_importance = False + self.training_extra_downloads = [repository.download_component_mapping] + feature_extractors = [ bug_features.HasSTR(), bug_features.Severity(), @@ -85,6 +87,8 @@ def __init__(self, calibration=True, lemmatization=False): bug_features.Whiteboard(), bug_features.Patches(), bug_features.Landings(), + bug_features.CommentFirstPathComponent(), + bug_features.CommentPathsComponents(), ] cleanup_functions = [ diff --git a/scripts/trainer.py b/scripts/trainer.py index 29df276581..bc627d00cd 100644 --- a/scripts/trainer.py +++ b/scripts/trainer.py @@ -32,6 +32,9 @@ def go(self, args): for required_db in model_obj.training_dbs: assert db.download(required_db) + for extra_download in model_obj.training_extra_downloads: + extra_download() + if args.download_eval: model_obj.download_eval_dbs() else: diff --git a/tests/test_bug_features.py b/tests/test_bug_features.py index f659411157..c43e2435a5 100644 --- a/tests/test_bug_features.py +++ b/tests/test_bug_features.py @@ -8,13 +8,16 @@ import pytest +from bugbug import bug_features from bugbug.bug_features import ( BlockedBugsNumber, BugExtractor, BugReporter, BugTypes, CommentCount, + CommentFirstPathComponent, CommentLength, + CommentPathsComponents, Component, DeltaNightlyRequestMerge, HasCrashSignature, @@ -32,6 +35,8 @@ Product, Severity, Whiteboard, + extract_path_components, + find_component, ) from bugbug.feature_cleanup import fileref, url @@ -187,3 +192,107 @@ def test_BugTypes(read) -> None: BugTypes, [["performance"], ["memory"], ["power"], ["security"], ["crash"]], ) + + +@pytest.fixture +def mock_component_mapping(monkeypatch): + # Mimics repository.get_component_mapping(), which is backed by an + # LMDBDict with bytes keys and memoryview values. + mapping = { + b"dom/base/nsGlobalWindow.cpp": memoryview(b"Core::DOM"), + b"layout/generic/nsFrame.cpp": memoryview(b"Core::Layout"), + } + monkeypatch.setattr( + bug_features.repository, "get_component_mapping", lambda: mapping + ) + return mapping + + +def test_extract_path_components(mock_component_mapping): + comments = [ + {"text": "The crash happens in dom/base/nsGlobalWindow.cpp around line 42."}, + { + "text": "It might also be related to layout/generic/nsFrame.cpp and " + "some/unknown/path.cpp." + }, + ] + + assert extract_path_components(comments) == ["Core::DOM", "Core::Layout"] + + +def test_extract_path_components_no_match(mock_component_mapping): + comments = [{"text": "Nothing looks like a source path here."}] + + assert extract_path_components(comments) == [] + + +def test_find_component_exact_match(mock_component_mapping): + assert ( + find_component(b"dom/base/nsGlobalWindow.cpp", mock_component_mapping) + == "Core::DOM" + ) + + +def test_find_component_longest_suffix_match(mock_component_mapping): + # The path has an extra leading directory (e.g. a local checkout root), + # so it doesn't match a mapping key exactly, but its longest suffix does. + assert ( + find_component( + b"mozilla-central/dom/base/nsGlobalWindow.cpp", mock_component_mapping + ) + == "Core::DOM" + ) + + +def test_find_component_no_match(mock_component_mapping): + assert find_component(b"some/unknown/path.cpp", mock_component_mapping) is None + + +def test_extract_path_components_extra_prefix(mock_component_mapping): + comments = [ + {"text": "Reproduced with src/mozilla-central/dom/base/nsGlobalWindow.cpp."} + ] + + assert extract_path_components(comments) == ["Core::DOM"] + + +def test_comment_first_path_component(mock_component_mapping): + bug = { + "comments": [ + {"text": "See layout/generic/nsFrame.cpp and dom/base/nsGlobalWindow.cpp."} + ] + } + + assert CommentFirstPathComponent()(bug) == "Core::Layout" + + +def test_comment_first_path_component_no_match(mock_component_mapping): + bug = {"comments": [{"text": "Nothing looks like a source path here."}]} + + assert CommentFirstPathComponent()(bug) is None + + +def test_comment_paths_components(mock_component_mapping): + bug = { + "comments": [ + {"text": "See layout/generic/nsFrame.cpp and dom/base/nsGlobalWindow.cpp."} + ] + } + + assert CommentPathsComponents()(bug) == {"Core::Layout": 1, "Core::DOM": 1 / 2} + + +def test_comment_paths_components_repeated_mentions(mock_component_mapping): + bug = { + "comments": [ + { + "text": "See layout/generic/nsFrame.cpp, then " + "dom/base/nsGlobalWindow.cpp, then layout/generic/nsFrame.cpp again." + } + ] + } + + assert CommentPathsComponents()(bug) == { + "Core::Layout": 1 + 1 / 3, + "Core::DOM": 1 / 2, + }