From 542cd6cf17fa322916b7ac37e9a12a820a4171df Mon Sep 17 00:00:00 2001 From: YeDonS <120633988+YeDonS@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:16:04 -0400 Subject: [PATCH 1/7] eval: freeze first 25 model predictions and verify patch identity in harness results --- .github/workflows/ci.yml | 4 +- reports/model-batches/batch-01/manifest.json | 138 ++++++++++++++++++ .../model-batches/batch-01/predictions.jsonl | 25 ++++ .../batch-01/swebench-predictions.jsonl | 25 ++++ scripts/aggregate_patch_results.py | 18 +++ scripts/evaluate_patches.py | 11 +- tests/test_evaluation_runs.py | 8 +- 7 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 reports/model-batches/batch-01/manifest.json create mode 100644 reports/model-batches/batch-01/predictions.jsonl create mode 100644 reports/model-batches/batch-01/swebench-predictions.jsonl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8659203..8462ddc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,8 +17,8 @@ jobs: enable-cache: true python-version: ${{ matrix.python }} - run: uv sync --locked --all-extras - - run: uv run ruff check src tests - - run: uv run ruff format --check src tests + - run: uv run ruff check src tests scripts + - run: uv run ruff format --check src tests scripts - run: uv run pytest --cov=code_assistant --cov-report=xml - name: Run real offline reproduction run: | diff --git a/reports/model-batches/batch-01/manifest.json b/reports/model-batches/batch-01/manifest.json new file mode 100644 index 0000000..5751efe --- /dev/null +++ b/reports/model-batches/batch-01/manifest.json @@ -0,0 +1,138 @@ +{ + "kind": "frozen_inference_batch", + "batch": 1, + "count": 25, + "parent_manifest_sha256": "84c775f6f20462427a272c70a47e09a23c92a2ad6c5234949aa3cdd706c0f07b", + "predictions_sha256": "0347de1a434264eb5ed67729ac0d39185d7385e1637d581ea8298d48807dc4c1", + "parent_configuration": { + "mode": "model", + "count": 100, + "task_ids": [ + "astropy__astropy-12907", + "django__django-16255", + "matplotlib__matplotlib-23964", + "mwaskom__seaborn-2848", + "pallets__flask-4992", + "psf__requests-2317", + "pydata__xarray-4094", + "pylint-dev__pylint-7993", + "pytest-dev__pytest-5692", + "scikit-learn__scikit-learn-13584", + "sphinx-doc__sphinx-8713", + "sympy__sympy-12419", + "astropy__astropy-14365", + "django__django-11742", + "matplotlib__matplotlib-26011", + "mwaskom__seaborn-3407", + "pallets__flask-4045", + "psf__requests-2148", + "pydata__xarray-3364", + "pylint-dev__pylint-5859", + "pytest-dev__pytest-5227", + "scikit-learn__scikit-learn-11040", + "sphinx-doc__sphinx-7975", + "sympy__sympy-13437", + "astropy__astropy-7746", + "django__django-13757", + "matplotlib__matplotlib-25498", + "mwaskom__seaborn-3190", + "pallets__flask-5063", + "psf__requests-3362", + "pydata__xarray-4493", + "pylint-dev__pylint-7080", + "pytest-dev__pytest-5103", + "scikit-learn__scikit-learn-14894", + "sphinx-doc__sphinx-8801", + "sympy__sympy-11870", + "astropy__astropy-14995", + "django__django-10914", + "matplotlib__matplotlib-23563", + "mwaskom__seaborn-3010", + "psf__requests-1963", + "pydata__xarray-4248", + "pylint-dev__pylint-7114", + "pytest-dev__pytest-8906", + "scikit-learn__scikit-learn-15512", + "sphinx-doc__sphinx-8506", + "sympy__sympy-20590", + "astropy__astropy-14182", + "django__django-13447", + "matplotlib__matplotlib-23314", + "psf__requests-2674", + "pydata__xarray-5131", + "pylint-dev__pylint-7228", + "pytest-dev__pytest-5413", + "scikit-learn__scikit-learn-14087", + "sphinx-doc__sphinx-8595", + "sympy__sympy-18087", + "astropy__astropy-6938", + "django__django-12184", + "matplotlib__matplotlib-24265", + "psf__requests-863", + "pylint-dev__pylint-6506", + "pytest-dev__pytest-7373", + "scikit-learn__scikit-learn-14983", + "sphinx-doc__sphinx-8282", + "sympy__sympy-13647", + "django__django-13448", + "matplotlib__matplotlib-18869", + "pytest-dev__pytest-5495", + "scikit-learn__scikit-learn-14092", + "sphinx-doc__sphinx-7686", + "sympy__sympy-20154", + "django__django-11001", + "matplotlib__matplotlib-24149", + "pytest-dev__pytest-11143", + "scikit-learn__scikit-learn-13497", + "sphinx-doc__sphinx-7738", + "sympy__sympy-13480", + "django__django-12983", + "matplotlib__matplotlib-25332", + "pytest-dev__pytest-7168", + "scikit-learn__scikit-learn-25500", + "sphinx-doc__sphinx-8721", + "sympy__sympy-19487", + "django__django-14997", + "matplotlib__matplotlib-24334", + "pytest-dev__pytest-11148", + "scikit-learn__scikit-learn-25747", + "sphinx-doc__sphinx-8435", + "sympy__sympy-13895", + "django__django-12497", + "matplotlib__matplotlib-26020", + "pytest-dev__pytest-7490", + "scikit-learn__scikit-learn-25638", + "sphinx-doc__sphinx-8273", + "sympy__sympy-14308", + "django__django-13401", + "matplotlib__matplotlib-25079", + "pytest-dev__pytest-8365", + "scikit-learn__scikit-learn-13142" + ], + "tasks_sha256": "933b5103f3a746835a41a71422d84487b504291ca577b5a450f9bf280b458a18", + "model": "codex-cli:gpt-6.1-sol", + "seed": 2026, + "created_at": "2026-10-08T18:05:29.872171+00:00", + "test_executor": "disabled", + "max_tool_calls": 18, + "max_search_passes": 2, + "max_agent_steps": 5, + "model_timeout": 120, + "codex_reasoning_effort": "medium", + "codex_transport": "https", + "workers": 6, + "environment": { + "python": "3.12.13", + "platform": "macOS-15.4.1-arm64-arm-64bit", + "source_tree_sha256": "d4e5b6d72a4d66ac37e10f45ae0c900d24a439de2ee2ff1d5c6ca68fd5f1ddf1", + "packages": { + "langchain": "1.4.3", + "langgraph": "1.2.14", + "fastapi": "0.143.0", + "numpy": "2.5.3", + "rank-bm25": "0.2.2" + } + }, + "codex_cli_version": "codex-cli 0.162.0-alpha.2" + } +} diff --git a/reports/model-batches/batch-01/predictions.jsonl b/reports/model-batches/batch-01/predictions.jsonl new file mode 100644 index 0000000..ec3c26e --- /dev/null +++ b/reports/model-batches/batch-01/predictions.jsonl @@ -0,0 +1,25 @@ +{"instance_id": "astropy__astropy-12907", "mode": "model", "status": "diagnosed", "predicted_files": ["astropy/modeling/separable.py"], "root_cause": "_separable recursively computes dependency matrices for compound children and passes them to _cstack for '&'. When the right operand is a matrix, _cstack fills its entire block with ones instead of copying that matrix, destroying the nested model's independent dependencies.", "model_patch": "--- a/astropy/modeling/separable.py\n+++ b/astropy/modeling/separable.py\n@@ -242,7 +242,7 @@\n cright = _coord_matrix(right, 'right', noutp)\n else:\n cright = np.zeros((noutp, right.shape[1]))\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\n \n return np.hstack([cleft, cright])\n \n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "django__django-16255", "mode": "model", "status": "diagnosed", "predicted_files": ["django/contrib/sitemaps/__init__.py"], "root_cause": "Sitemap.get_latest_lastmod() calls max() on an empty list when items() is empty and lastmod is callable. This raises ValueError, but the method only catches TypeError. The exception propagates through the sitemap index view.", "model_patch": "--- a/django/contrib/sitemaps/__init__.py\n+++ b/django/contrib/sitemaps/__init__.py\n@@ -168,7 +168,7 @@\n if callable(self.lastmod):\n try:\n return max([self.lastmod(item) for item in self.items()])\n- except TypeError:\n+ except (TypeError, ValueError):\n return None\n else:\n return self.lastmod\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "matplotlib__matplotlib-23964", "mode": "model", "status": "diagnosed", "predicted_files": ["lib/matplotlib/backends/backend_ps.py"], "root_cause": "RendererPS.draw_text initializes curr_stream to None. For an empty line, _text_helpers.layout yields no glyphs, but the TrueType branch unconditionally appends curr_stream to stream. The subsequent loop attempts to unpack None, raising TypeError. Text.draw forwards empty lines to the backend.", "model_patch": "--- a/lib/matplotlib/backends/backend_ps.py\n+++ b/lib/matplotlib/backends/backend_ps.py\n@@ -668,4 +668,5 @@\n # append the last entry\n- stream.append(curr_stream)\n+ if curr_stream:\n+ stream.append(curr_stream)\n \n self.set_color(*gc.get_rgb())\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "mwaskom__seaborn-2848", "mode": "model", "status": "diagnosed", "predicted_files": ["seaborn/_oldcore.py", "tests/test_core.py"], "root_cause": "PairGrid passes excluded hue values to scatterplot. Their categorical color lookup fails, but HueMapping incorrectly attempts numeric interpolation with norm=None. The subsequent np.isnan call on an excluded string raises the reported TypeError.", "model_patch": "--- a/seaborn/_oldcore.py\n+++ b/seaborn/_oldcore.py\n@@ -148,6 +148,9 @@\n try:\n # Use a value that's in the original data vector\n value = self.lookup_table[key]\n except KeyError:\n+ if self.norm is None:\n+ return (0, 0, 0, 0)\n+\n # Use the colormap to interpolate between existing datapoints\n # (e.g. in the context of making a continuous legend)\n--- a/tests/test_core.py\n+++ b/tests/test_core.py\n@@ -157,6 +157,7 @@\n # Test hue order\n hue_order = [\"a\", \"c\", \"d\"]\n m = HueMapping(p, order=hue_order)\n assert m.levels == hue_order\n+ assert m(\"b\") == (0, 0, 0, 0)\n \n # Test long data\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pallets__flask-4992", "mode": "model", "status": "diagnosed", "predicted_files": ["src/flask/config.py"], "root_cause": "Config.from_file unconditionally calls open(filename), producing a text stream, and provides no way to select binary mode. This conflicts with tomllib.load's binary-stream requirement.", "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "psf__requests-2317", "mode": "model", "status": "diagnosed", "predicted_files": ["requests/sessions.py"], "root_cause": "Session.request converts the method using builtin_str, which is str on Python 3. Thus b'GET' becomes \"b'GET'\" and uppercasing produces \"B'GET'\", not the valid method GET. Subsequent preparation only uppercases and does not repair the conversion.", "model_patch": "--- a/requests/sessions.py\n+++ b/requests/sessions.py\n@@ -428,5 +428,5 @@\n- method = builtin_str(method)\n+ method = to_native_string(method)\n \n # Create the Request.\n req = Request(\n method = method.upper(),\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pydata__xarray-4094", "mode": "model", "status": "diagnosed", "predicted_files": ["xarray/core/dataarray.py", "xarray/tests/test_dataset.py"], "root_cause": "When all variable dimensions are sample dimensions, stacking produces a one-level MultiIndex. Selecting each variable in to_unstacked_dataset removes the stacked dimension but retains its scalar coordinate, with different values for each variable. squeeze(drop=True) does not remove an already-scalar coordinate, so Dataset construction encounters conflicting coordinates.", "model_patch": "--- a/xarray/core/dataarray.py\n+++ b/xarray/core/dataarray.py\n@@ -1963,7 +1963,7 @@\n # pull variables out of datarray\n data_dict = {}\n for k in variables:\n- data_dict[k] = self.sel({variable_dim: k}).squeeze(drop=True)\n+ data_dict[k] = self.sel({variable_dim: k}, drop=True).squeeze(drop=True)\n \n # unstacked dataset\n return Dataset(data_dict)\n--- a/xarray/tests/test_dataset.py\n+++ b/xarray/tests/test_dataset.py\n@@ -3033,3 +3033,10 @@\n+ def test_to_stacked_array_to_unstacked_dataset_single_dimension(self):\n+ arr = xr.DataArray(np.arange(3), coords=[(\"x\", [0, 1, 2])])\n+ data = xr.Dataset({\"a\": arr, \"b\": arr})\n+ stacked = data.to_stacked_array(\"y\", sample_dims=[\"x\"])\n+ unstacked = stacked.to_unstacked_dataset(\"y\")\n+ assert_identical(data, unstacked)\n+\n def test_to_stacked_array_to_unstacked_dataset(self):\n # make a two dimensional dataset\n a, b = create_test_stacked_array()\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pylint-dev__pylint-7993", "mode": "model", "status": "diagnosed", "predicted_files": ["pylint/reporters/text.py", "tests/reporters/unittest_reporting.py"], "root_cause": "TextReporter validates placeholders using a regex that does not respect escaped braces ({{ and }}). It mistakes the literal JSON opening for an unsupported field and removes part of the template before str.format renders it.", "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pytest-dev__pytest-5692", "mode": "model", "status": "diagnosed", "predicted_files": ["src/_pytest/junitxml.py"], "root_cause": "LogXML.pytest_sessionfinish constructs the testsuite element without hostname or timestamp attributes. The available JUnit options provide no switch to enable them; custom suite properties produce child elements rather than attributes.", "model_patch": "--- a/src/_pytest/junitxml.py\n+++ b/src/_pytest/junitxml.py\n@@ -11,6 +11,8 @@\n+import datetime\n import functools\n import os\n import re\n+import socket\n import sys\n import time\n \n@@ -665,6 +667,10 @@\n failures=self.stats[\"failure\"],\n skipped=self.stats[\"skipped\"],\n tests=numtests,\n time=\"%.3f\" % suite_time_delta,\n+ timestamp=datetime.datetime.fromtimestamp(\n+ self.suite_start_time\n+ ).isoformat(),\n+ hostname=socket.gethostname(),\n )\n logfile.write(Junit.testsuites([suite_node]).unicode(indent=0))\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "scikit-learn__scikit-learn-13584", "mode": "model", "status": "diagnosed", "predicted_files": ["sklearn/utils/_pprint.py"], "root_cause": "_changed_params evaluates `v != init_params[k]` as a boolean. LogisticRegressionCV stores the supplied Cs array unchanged, while its default is 10. Comparing the array with that scalar produces a boolean array, whose truth evaluation raises the reported ValueError. Both changed-only rendering paths use this helper.", "model_patch": "--- a/sklearn/utils/_pprint.py\n+++ b/sklearn/utils/_pprint.py\n@@ -95,7 +95,7 @@\n init_params = signature(init_func).parameters\n init_params = {name: param.default for name, param in init_params.items()}\n for k, v in params.items():\n- if (v != init_params[k] and\n+ if (repr(v) != repr(init_params[k]) and\n not (is_scalar_nan(init_params[k]) and is_scalar_nan(v))):\n filtered_params[k] = v\n return filtered_params\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "sphinx-doc__sphinx-8713", "mode": "model", "status": "diagnosed", "predicted_files": ["sphinx/ext/napoleon/docstring.py"], "root_cause": "The Other Parameters handler unconditionally calls _format_fields, ignoring napoleon_use_param. The Parameters handler checks that setting and uses docutils parameter fields when enabled.", "model_patch": "--- a/sphinx/ext/napoleon/docstring.py\n+++ b/sphinx/ext/napoleon/docstring.py\n@@ -684,6 +684,10 @@\n def _parse_other_parameters_section(self, section: str) -> List[str]:\n- return self._format_fields(_('Other Parameters'), self._consume_fields())\n+ if self._config.napoleon_use_param:\n+ fields = self._consume_fields(multiple=True)\n+ return self._format_docutils_params(fields)\n+ else:\n+ return self._format_fields(_('Other Parameters'), self._consume_fields())\n \n def _parse_parameters_section(self, section: str) -> List[str]:\n if self._config.napoleon_use_param:\n # Allow to declare multiple parameters at once (ex: x, y: int)\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "sympy__sympy-12419", "mode": "model", "status": "diagnosed", "predicted_files": ["sympy/matrices/expressions/matexpr.py"], "root_cause": "Identity._entry uses structural equality to decide whether indices coincide. Distinct symbolic indices i and j therefore produce zero immediately, even though they can take equal values during summation. The summand is already zero before Sum evaluates it. Symbolic identity entries must instead retain a KroneckerDelta.", "model_patch": "--- a/sympy/matrices/expressions/matexpr.py\n+++ b/sympy/matrices/expressions/matexpr.py\n@@ -478,8 +478,6 @@\n def _entry(self, i, j):\n- if i == j:\n- return S.One\n- else:\n- return S.Zero\n+ from sympy import KroneckerDelta\n+ return KroneckerDelta(i, j)\n \n def _eval_determinant(self):\n return S.One\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "astropy__astropy-14365", "mode": "model", "status": "diagnosed", "predicted_files": ["astropy/io/ascii/qdp.py"], "root_cause": "_line_type() matches READ SERR/TERR commands using a case-sensitive regular expression, so lowercase or mixed-case commands raise ValueError before error-column parsing. Downstream parsing already normalizes the error type with .lower().", "model_patch": "--- a/astropy/io/ascii/qdp.py\n+++ b/astropy/io/ascii/qdp.py\n@@ -62,3 +62,3 @@\n _decimal_re = r\"[+-]?(\\d+(\\.\\d*)?|\\.\\d+)([eE][+-]?\\d+)?\"\n- _command_re = r\"READ [TS]ERR(\\s+[0-9]+)+\"\n+ _command_re = r\"(?i:READ [TS]ERR(\\s+[0-9]+)+)\"\n \n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "django__django-11742", "mode": "model", "status": "diagnosed", "predicted_files": ["django/db/models/fields/__init__.py"], "root_cause": "Field._check_choices() validates choice structure but never compares stored choice values with max_length. Consequently, structurally valid choices pass checks even when their string values exceed the field's capacity.", "model_patch": "--- a/django/db/models/fields/__init__.py\n+++ b/django/db/models/fields/__init__.py\n@@ -260,6 +260,7 @@\n # Expect [group_name, [value, display]]\n+ choice_max_length = 0\n for choices_group in self.choices:\n try:\n group_name, group_choices = choices_group\n except (TypeError, ValueError):\n # Containing non-pairs\n@@ -267,12 +268,18 @@\n try:\n- if not all(\n- is_value(value) and is_value(human_name)\n- for value, human_name in group_choices\n- ):\n+ valid_group = True\n+ for value, human_name in group_choices:\n+ if not is_value(value) or not is_value(human_name):\n+ valid_group = False\n+ break\n+ if isinstance(value, str):\n+ choice_max_length = max(choice_max_length, len(value))\n+ if not valid_group:\n break\n except (TypeError, ValueError):\n # No groups, choices in the form [value, display]\n value, human_name = group_name, group_choices\n if not is_value(value) or not is_value(human_name):\n break\n+ if isinstance(value, str):\n+ choice_max_length = max(choice_max_length, len(value))\n \n@@ -279,6 +286,15 @@\n # Special case: choices=['ab']\n if isinstance(choices_group, str):\n break\n else:\n+ if type(self.max_length) is int and self.max_length > 0 and choice_max_length > self.max_length:\n+ return [\n+ checks.Error(\n+ \"'max_length' is too small to fit the longest value \"\n+ \"in 'choices' (%s characters).\" % choice_max_length,\n+ obj=self,\n+ id='fields.E009',\n+ )\n+ ]\n return []\n \n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "matplotlib__matplotlib-26011", "mode": "model", "status": "diagnosed", "predicted_files": ["lib/matplotlib/axis.py"], "root_cause": "Axes.set_xlim delegates to Axis._set_lim, which gates callback dispatch and shared-axis propagation behind emit. Shared siblings are updated with emit=False, so their limits change without their xlim_changed callbacks being dispatched.", "model_patch": "--- a/lib/matplotlib/axis.py\n+++ b/lib/matplotlib/axis.py\n@@ -1243,6 +1243,7 @@\n for other in self._get_shared_axes():\n if other is not self.axes:\n other._axis_map[name]._set_lim(\n v0, v1, emit=False, auto=auto)\n+ other.callbacks.process(f\"{name}lim_changed\", other)\n if other.figure != self.figure:\n other.figure.canvas.draw_idle()\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "mwaskom__seaborn-3407", "mode": "model", "status": "diagnosed", "predicted_files": ["seaborn/axisgrid.py"], "root_cause": "PairGrid.map_diag converts tuple column labels into a two-dimensional NumPy object array. Iteration then yields arrays rather than tuple keys, causing pandas to interpret self.data[var] as list-like column selection and raise KeyError. Both diagonal mapping branches use these corrupted keys.", "model_patch": "--- a/seaborn/axisgrid.py\n+++ b/seaborn/axisgrid.py\n@@ -1472,7 +1472,7 @@\n for ax in diag_axes[1:]:\n share_axis(diag_axes[0], ax, \"y\")\n \n- self.diag_vars = np.array(diag_vars, np.object_)\n+ self.diag_vars = diag_vars\n self.diag_axes = np.array(diag_axes, np.object_)\n \n if \"hue\" not in signature(func).parameters:\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pallets__flask-4045", "mode": "model", "status": "diagnosed", "predicted_files": ["src/flask/blueprints.py", "tests/test_blueprints.py"], "root_cause": "Blueprint.__init__ assigns name without rejecting dots. Nested blueprint registration and endpoint generation use dots as namespace separators, so dotted blueprint names are ambiguous. Endpoint names already have dot validation, but blueprint names do not.", "model_patch": "--- a/src/flask/blueprints.py\n+++ b/src/flask/blueprints.py\n@@ -188,6 +188,9 @@\n template_folder=template_folder,\n root_path=root_path,\n )\n+ if \".\" in name:\n+ raise ValueError(\"'name' may not contain a dot '.' character.\")\n+\n self.name = name\n self.url_prefix = url_prefix\n self.subdomain = subdomain\n--- a/tests/test_blueprints.py\n+++ b/tests/test_blueprints.py\n@@ -7,6 +7,12 @@\n import flask\n \n \n+@pytest.mark.parametrize(\"name\", [\".name\", \"na.me\", \"name.\"])\n+def test_blueprint_name_with_dot(name):\n+ with pytest.raises(ValueError, match=\"may not contain a dot\"):\n+ flask.Blueprint(name, __name__)\n+\n+\n def test_blueprint_specific_error_handling(app, client):\n frontend = flask.Blueprint(\"frontend\", __name__)\n backend = flask.Blueprint(\"backend\", __name__)\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "psf__requests-2148", "mode": "model", "status": "diagnosed", "predicted_files": ["requests/packages/urllib3/response.py", "requests/models.py"], "root_cause": "Response-body exception translation is incomplete. urllib3 HTTPResponse.read() does not catch ordinary socket.error, so connection resets escape unchanged. Requests Response.iter_content() also does not translate urllib3 ProtocolError; .content consumes this iterator, exposing the error through .text.", "model_patch": "--- a/requests/packages/urllib3/response.py\n+++ b/requests/packages/urllib3/response.py\n@@ -1,6 +1,6 @@\n import zlib\n import io\n-from socket import timeout as SocketTimeout\n+from socket import error as SocketError, timeout as SocketTimeout\n \n from ._collections import HTTPHeaderDict\n from .exceptions import ProtocolError, DecodeError, ReadTimeoutError\n@@ -209,3 +209,3 @@\n- except HTTPException as e:\n+ except (HTTPException, SocketError) as e:\n # This includes IncompleteRead.\n raise ProtocolError('Connection broken: %r' % e, e)\n--- a/requests/models.py\n+++ b/requests/models.py\n@@ -19,7 +19,7 @@\n from .packages.urllib3.fields import RequestField\n from .packages.urllib3.filepost import encode_multipart_formdata\n from .packages.urllib3.util import parse_url\n-from .packages.urllib3.exceptions import DecodeError\n+from .packages.urllib3.exceptions import DecodeError, ProtocolError\n from .exceptions import (\n HTTPError, RequestException, MissingSchema, InvalidURL,\n ChunkedEncodingError, ContentDecodingError)\n@@ -637,5 +637,5 @@\n for chunk in self.raw.stream(chunk_size, decode_content=True):\n yield chunk\n- except IncompleteRead as e:\n+ except (IncompleteRead, ProtocolError) as e:\n raise ChunkedEncodingError(e)\n except DecodeError as e:\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pydata__xarray-3364", "mode": "model", "status": "diagnosed", "predicted_files": ["xarray/core/concat.py"], "root_cause": "Dataset concatenation requires matching variable sets instead of constructing missing-value blocks. Variable selection and output iteration rely on the first dataset, comparisons directly access absent variables, and both merging and concatenation reject missing variables. join='outer' aligns indexes, not variable names.", "model_patch": "--- a/xarray/core/concat.py\n+++ b/xarray/core/concat.py\n@@ -191,6 +191,15 @@\n # all nonindexes that are not the same in each dataset\n- for k in getattr(datasets[0], subset):\n+ names = dict.fromkeys(\n+ k for ds in datasets for k in getattr(ds, subset)\n+ ) if subset == \"data_vars\" else getattr(datasets[0], subset)\n+ for k in names:\n if k not in concat_over:\n+ if subset == \"data_vars\" and any(\n+ k not in ds.variables for ds in datasets\n+ ):\n+ concat_over.add(k)\n+ equals[k] = False\n+ continue\n # Compare the variable of all datasets vs. the one\n # of the first dataset. Perform the minimum amount of\n # loads in order to avoid multiple loads from disk\n@@ -217,5 +227,6 @@\n elif opt == \"all\":\n concat_over.update(\n- set(getattr(datasets[0], subset)) - set(datasets[0].dims)\n+ k for ds in (datasets if subset == \"data_vars\" else datasets[:1])\n+ for k in getattr(ds, subset) if k not in ds.dims\n )\n elif opt == \"minimal\":\n@@ -225,3 +236,7 @@\n else:\n- invalid_vars = [k for k in opt if k not in getattr(datasets[0], subset)]\n+ available = set(getattr(datasets[0], subset))\n+ if subset == \"data_vars\":\n+ for ds in datasets[1:]:\n+ available.update(ds.data_vars)\n+ invalid_vars = [k for k in opt if k not in available]\n if invalid_vars:\n@@ -235,4 +250,4 @@\n \"some variables in data_vars are not data variables \"\n- \"on the first dataset: %s\" % (invalid_vars,)\n+ \"on any dataset: %s\" % (invalid_vars,)\n )\n concat_over.update(opt)\n@@ -315,3 +330,3 @@\n- absent_merge_vars = variables_to_merge - set(ds.variables)\n+ absent_merge_vars = (variables_to_merge & coord_names) - set(ds.variables)\n if absent_merge_vars:\n raise ValueError(\n@@ -323,3 +338,4 @@\n for var in variables_to_merge:\n- to_merge[var].append(ds.variables[var])\n+ if var in ds.variables:\n+ to_merge[var].append(ds.variables[var])\n \n@@ -359,7 +375,27 @@\n- for k in datasets[0].variables:\n+ variable_names = dict.fromkeys(k for ds in datasets for k in ds.variables)\n+ for k in variable_names:\n if k in concat_over:\n- try:\n- vars = ensure_common_dims([ds.variables[k] for ds in datasets])\n- except KeyError:\n+ if k in coord_names and any(k not in ds.variables for ds in datasets):\n raise ValueError(\"%r is not present in all datasets.\" % k)\n+ template = next(ds.variables[k] for ds in datasets if k in ds.variables)\n+ vars = []\n+ for ds, dim_len in zip(datasets, concat_dim_lengths):\n+ if k in ds.variables:\n+ vars.append(ds.variables[k])\n+ continue\n+ missing_dims = template.dims\n+ if dim not in missing_dims:\n+ missing_dims = (dim,) + missing_dims\n+ shape = tuple(\n+ dim_len if d == dim else dims_sizes[d] for d in missing_dims\n+ )\n+ size = 1\n+ for length in shape:\n+ size *= length\n+ missing = pd.Series([], dtype=template.dtype).reindex(\n+ pd.RangeIndex(size),\n+ fill_value=None if fill_value is dtypes.NA else fill_value,\n+ ).values.reshape(shape)\n+ vars.append(Variable(missing_dims, missing, attrs=template.attrs))\n+ vars = ensure_common_dims(vars)\n combined = concat_vars(vars, dim, positions)\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pylint-dev__pylint-5859", "mode": "model", "status": "diagnosed", "predicted_files": ["pylint/checkers/misc.py"], "root_cause": "EncodingChecker.open() appends a word boundary (\\b) to note patterns. In '# ???: no', the final '?' and following ':' are both non-word characters, so no boundary exists and process_tokens() never emits W0511. Alphabetic tags such as 'YES:' do have that boundary.", "model_patch": "--- a/pylint/checkers/misc.py\n+++ b/pylint/checkers/misc.py\n@@ -122,7 +122,7 @@\n notes = \"|\".join(re.escape(note) for note in self.config.notes)\n if self.config.notes_rgx:\n- regex_string = rf\"#\\s*({notes}|{self.config.notes_rgx})\\b\"\n+ regex_string = rf\"#\\s*({notes}|{self.config.notes_rgx})(?=[:\\s]|$)\"\n else:\n- regex_string = rf\"#\\s*({notes})\\b\"\n+ regex_string = rf\"#\\s*({notes})(?=[:\\s]|$)\"\n \n self._fixme_pattern = re.compile(regex_string, re.I)\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pytest-dev__pytest-5227", "mode": "model", "status": "diagnosed", "predicted_files": ["src/_pytest/logging.py", "testing/logging/test_reporting.py"], "root_cause": "DEFAULT_LOG_FORMAT omits the logger name and displays the filename first. Captured logging uses this default; file and live logging also fall back to it unless overridden. The name field identifies the logger, not necessarily the module.", "model_patch": "--- a/src/_pytest/logging.py\n+++ b/src/_pytest/logging.py\n@@ -16,4 +16,4 @@\n from _pytest.pathlib import Path\n \n-DEFAULT_LOG_FORMAT = \"%(filename)-25s %(lineno)4d %(levelname)-8s %(message)s\"\n+DEFAULT_LOG_FORMAT = \"%(levelname)-8s %(name)s:%(filename)s:%(lineno)d %(message)s\"\n DEFAULT_LOG_DATE_FORMAT = \"%H:%M:%S\"\n--- a/testing/logging/test_reporting.py\n+++ b/testing/logging/test_reporting.py\n@@ -282,6 +282,6 @@\n result.stdout.fnmatch_lines(\n [\n \"test_log_cli_default_level.py::test_log_cli \",\n- \"test_log_cli_default_level.py*WARNING message will be shown*\",\n+ \"*WARNING*catchlog:test_log_cli_default_level.py:*WARNING message will be shown*\",\n ]\n )\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "scikit-learn__scikit-learn-11040", "mode": "model", "status": "diagnosed", "predicted_files": ["sklearn/neighbors/base.py"], "root_cause": "NeighborsBase._fit checks positivity but not integer type, allowing n_neighbors=3.0 through. Its validation also occurs after sparse/tree-input early returns. KNeighborsMixin.kneighbors does not validate the resolved parameter or explicit override, so floats reach integer-only NumPy operations or tree queries.", "model_patch": "--- a/sklearn/neighbors/base.py\n+++ b/sklearn/neighbors/base.py\n@@ -9,6 +9,7 @@\n import warnings\n from abc import ABCMeta, abstractmethod\n+from numbers import Integral\n \n import numpy as np\n from scipy.sparse import csr_matrix, issparse\n \n@@ -157,3 +158,12 @@\n def _fit(self, X):\n+ if self.n_neighbors is not None:\n+ if not isinstance(self.n_neighbors, Integral):\n+ raise ValueError(\"n_neighbors must be a positive integer; \"\n+ \"got %r\" % self.n_neighbors)\n+ if self.n_neighbors <= 0:\n+ raise ValueError(\n+ \"Expected n_neighbors > 0. Got %d\" %\n+ self.n_neighbors)\n+\n self._check_algorithm_metric()\n if self.metric_params is None:\n@@ -255,10 +265,3 @@\n- if self.n_neighbors is not None:\n- if self.n_neighbors <= 0:\n- raise ValueError(\n- \"Expected n_neighbors > 0. Got %d\" %\n- self.n_neighbors\n- )\n-\n return self\n \n @property\n@@ -329,6 +332,14 @@\n if n_neighbors is None:\n n_neighbors = self.n_neighbors\n \n+ if not isinstance(n_neighbors, Integral):\n+ raise ValueError(\"n_neighbors must be a positive integer; \"\n+ \"got %r\" % n_neighbors)\n+ if n_neighbors <= 0:\n+ raise ValueError(\n+ \"Expected n_neighbors > 0. Got %d\" %\n+ n_neighbors)\n+\n if X is not None:\n query_is_train = False\n X = check_array(X, accept_sparse='csr')\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "sphinx-doc__sphinx-7975", "mode": "model", "status": "diagnosed", "predicted_files": ["sphinx/environment/adapters/indexentries.py"], "root_cause": "The index sorter prefixes alphabetic entries with chr(127), but leaves symbols unchanged. Consequently, @ sorts before letters while £ and ← sort after them. The subsequent groupby() only combines adjacent entries, producing two separate groups named Symbols. The subentry sorter uses the same flawed ordering.", "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "sympy__sympy-13437", "mode": "model", "status": "diagnosed", "predicted_files": ["sympy/functions/combinatorial/numbers.py", "sympy/functions/combinatorial/tests/test_comb_numbers.py"], "root_cause": "bell.eval handles only nonnegative integer indices and has no positive-infinity case, so bell(oo) remains unevaluated. Fibonacci and Lucas already handle this case explicitly.", "model_patch": "--- a/sympy/functions/combinatorial/numbers.py\n+++ b/sympy/functions/combinatorial/numbers.py\n@@ -434,5 +434,8 @@\n @classmethod\n def eval(cls, n, k_sym=None, symbols=None):\n+ if n is S.Infinity and k_sym is None and symbols is None:\n+ return S.Infinity\n+\n if n.is_Integer and n.is_nonnegative:\n if k_sym is None:\n return Integer(cls._bell(int(n)))\n--- a/sympy/functions/combinatorial/tests/test_comb_numbers.py\n+++ b/sympy/functions/combinatorial/tests/test_comb_numbers.py\n@@ -80,4 +80,8 @@\n def test_bell():\n assert [bell(n) for n in range(8)] == [1, 1, 2, 5, 15, 52, 203, 877]\n \n+ assert bell(S.Infinity) == S.Infinity\n+ n = Dummy('n')\n+ assert bell(n).limit(n, S.Infinity) == S.Infinity\n+\n assert bell(0, x) == 1\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "astropy__astropy-7746", "mode": "model", "status": "diagnosed", "predicted_files": ["astropy/wcs/wcs.py"], "root_cause": "The shared _array_converter unconditionally invokes the transformation backend for empty coordinates. Separate empty axes become a zero-row coordinate array passed to self.wcs.p2s, matching the reported wcslib error. The single-array branch also lacks empty-input handling.", "model_patch": "--- a/astropy/wcs/wcs.py\n+++ b/astropy/wcs/wcs.py\n@@ -1222,7 +1222,7 @@\n \n if ra_dec_order and sky == 'input':\n xy = self._denormalize_sky(xy)\n- output = func(xy, origin)\n+ output = xy if xy.size == 0 else func(xy, origin)\n if ra_dec_order and sky == 'output':\n output = self._normalize_sky(output)\n return (output[:, 0].reshape(axes[0].shape),\n@@ -1239,7 +1239,7 @@\n \"of shape (N, {0})\".format(self.naxis))\n if ra_dec_order and sky == 'input':\n xy = self._denormalize_sky(xy)\n- result = func(xy, origin)\n+ result = xy if xy.size == 0 else func(xy, origin)\n if ra_dec_order and sky == 'output':\n result = self._normalize_sky(result)\n return result\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} diff --git a/reports/model-batches/batch-01/swebench-predictions.jsonl b/reports/model-batches/batch-01/swebench-predictions.jsonl new file mode 100644 index 0000000..792f298 --- /dev/null +++ b/reports/model-batches/batch-01/swebench-predictions.jsonl @@ -0,0 +1,25 @@ +{"instance_id": "astropy__astropy-12907", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/astropy/modeling/separable.py\n+++ b/astropy/modeling/separable.py\n@@ -242,7 +242,7 @@\n cright = _coord_matrix(right, 'right', noutp)\n else:\n cright = np.zeros((noutp, right.shape[1]))\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\n \n return np.hstack([cleft, cright])\n \n"} +{"instance_id": "django__django-16255", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/django/contrib/sitemaps/__init__.py\n+++ b/django/contrib/sitemaps/__init__.py\n@@ -168,7 +168,7 @@\n if callable(self.lastmod):\n try:\n return max([self.lastmod(item) for item in self.items()])\n- except TypeError:\n+ except (TypeError, ValueError):\n return None\n else:\n return self.lastmod\n"} +{"instance_id": "matplotlib__matplotlib-23964", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/lib/matplotlib/backends/backend_ps.py\n+++ b/lib/matplotlib/backends/backend_ps.py\n@@ -668,4 +668,5 @@\n # append the last entry\n- stream.append(curr_stream)\n+ if curr_stream:\n+ stream.append(curr_stream)\n \n self.set_color(*gc.get_rgb())\n"} +{"instance_id": "mwaskom__seaborn-2848", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/seaborn/_oldcore.py\n+++ b/seaborn/_oldcore.py\n@@ -148,6 +148,9 @@\n try:\n # Use a value that's in the original data vector\n value = self.lookup_table[key]\n except KeyError:\n+ if self.norm is None:\n+ return (0, 0, 0, 0)\n+\n # Use the colormap to interpolate between existing datapoints\n # (e.g. in the context of making a continuous legend)\n--- a/tests/test_core.py\n+++ b/tests/test_core.py\n@@ -157,6 +157,7 @@\n # Test hue order\n hue_order = [\"a\", \"c\", \"d\"]\n m = HueMapping(p, order=hue_order)\n assert m.levels == hue_order\n+ assert m(\"b\") == (0, 0, 0, 0)\n \n # Test long data\n"} +{"instance_id": "pallets__flask-4992", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": ""} +{"instance_id": "psf__requests-2317", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/requests/sessions.py\n+++ b/requests/sessions.py\n@@ -428,5 +428,5 @@\n- method = builtin_str(method)\n+ method = to_native_string(method)\n \n # Create the Request.\n req = Request(\n method = method.upper(),\n"} +{"instance_id": "pydata__xarray-4094", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/xarray/core/dataarray.py\n+++ b/xarray/core/dataarray.py\n@@ -1963,7 +1963,7 @@\n # pull variables out of datarray\n data_dict = {}\n for k in variables:\n- data_dict[k] = self.sel({variable_dim: k}).squeeze(drop=True)\n+ data_dict[k] = self.sel({variable_dim: k}, drop=True).squeeze(drop=True)\n \n # unstacked dataset\n return Dataset(data_dict)\n--- a/xarray/tests/test_dataset.py\n+++ b/xarray/tests/test_dataset.py\n@@ -3033,3 +3033,10 @@\n+ def test_to_stacked_array_to_unstacked_dataset_single_dimension(self):\n+ arr = xr.DataArray(np.arange(3), coords=[(\"x\", [0, 1, 2])])\n+ data = xr.Dataset({\"a\": arr, \"b\": arr})\n+ stacked = data.to_stacked_array(\"y\", sample_dims=[\"x\"])\n+ unstacked = stacked.to_unstacked_dataset(\"y\")\n+ assert_identical(data, unstacked)\n+\n def test_to_stacked_array_to_unstacked_dataset(self):\n # make a two dimensional dataset\n a, b = create_test_stacked_array()\n"} +{"instance_id": "pylint-dev__pylint-7993", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": ""} +{"instance_id": "pytest-dev__pytest-5692", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/src/_pytest/junitxml.py\n+++ b/src/_pytest/junitxml.py\n@@ -11,6 +11,8 @@\n+import datetime\n import functools\n import os\n import re\n+import socket\n import sys\n import time\n \n@@ -665,6 +667,10 @@\n failures=self.stats[\"failure\"],\n skipped=self.stats[\"skipped\"],\n tests=numtests,\n time=\"%.3f\" % suite_time_delta,\n+ timestamp=datetime.datetime.fromtimestamp(\n+ self.suite_start_time\n+ ).isoformat(),\n+ hostname=socket.gethostname(),\n )\n logfile.write(Junit.testsuites([suite_node]).unicode(indent=0))\n"} +{"instance_id": "scikit-learn__scikit-learn-13584", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/sklearn/utils/_pprint.py\n+++ b/sklearn/utils/_pprint.py\n@@ -95,7 +95,7 @@\n init_params = signature(init_func).parameters\n init_params = {name: param.default for name, param in init_params.items()}\n for k, v in params.items():\n- if (v != init_params[k] and\n+ if (repr(v) != repr(init_params[k]) and\n not (is_scalar_nan(init_params[k]) and is_scalar_nan(v))):\n filtered_params[k] = v\n return filtered_params\n"} +{"instance_id": "sphinx-doc__sphinx-8713", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/sphinx/ext/napoleon/docstring.py\n+++ b/sphinx/ext/napoleon/docstring.py\n@@ -684,6 +684,10 @@\n def _parse_other_parameters_section(self, section: str) -> List[str]:\n- return self._format_fields(_('Other Parameters'), self._consume_fields())\n+ if self._config.napoleon_use_param:\n+ fields = self._consume_fields(multiple=True)\n+ return self._format_docutils_params(fields)\n+ else:\n+ return self._format_fields(_('Other Parameters'), self._consume_fields())\n \n def _parse_parameters_section(self, section: str) -> List[str]:\n if self._config.napoleon_use_param:\n # Allow to declare multiple parameters at once (ex: x, y: int)\n"} +{"instance_id": "sympy__sympy-12419", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/sympy/matrices/expressions/matexpr.py\n+++ b/sympy/matrices/expressions/matexpr.py\n@@ -478,8 +478,6 @@\n def _entry(self, i, j):\n- if i == j:\n- return S.One\n- else:\n- return S.Zero\n+ from sympy import KroneckerDelta\n+ return KroneckerDelta(i, j)\n \n def _eval_determinant(self):\n return S.One\n"} +{"instance_id": "astropy__astropy-14365", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/astropy/io/ascii/qdp.py\n+++ b/astropy/io/ascii/qdp.py\n@@ -62,3 +62,3 @@\n _decimal_re = r\"[+-]?(\\d+(\\.\\d*)?|\\.\\d+)([eE][+-]?\\d+)?\"\n- _command_re = r\"READ [TS]ERR(\\s+[0-9]+)+\"\n+ _command_re = r\"(?i:READ [TS]ERR(\\s+[0-9]+)+)\"\n \n"} +{"instance_id": "django__django-11742", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/django/db/models/fields/__init__.py\n+++ b/django/db/models/fields/__init__.py\n@@ -260,6 +260,7 @@\n # Expect [group_name, [value, display]]\n+ choice_max_length = 0\n for choices_group in self.choices:\n try:\n group_name, group_choices = choices_group\n except (TypeError, ValueError):\n # Containing non-pairs\n@@ -267,12 +268,18 @@\n try:\n- if not all(\n- is_value(value) and is_value(human_name)\n- for value, human_name in group_choices\n- ):\n+ valid_group = True\n+ for value, human_name in group_choices:\n+ if not is_value(value) or not is_value(human_name):\n+ valid_group = False\n+ break\n+ if isinstance(value, str):\n+ choice_max_length = max(choice_max_length, len(value))\n+ if not valid_group:\n break\n except (TypeError, ValueError):\n # No groups, choices in the form [value, display]\n value, human_name = group_name, group_choices\n if not is_value(value) or not is_value(human_name):\n break\n+ if isinstance(value, str):\n+ choice_max_length = max(choice_max_length, len(value))\n \n@@ -279,6 +286,15 @@\n # Special case: choices=['ab']\n if isinstance(choices_group, str):\n break\n else:\n+ if type(self.max_length) is int and self.max_length > 0 and choice_max_length > self.max_length:\n+ return [\n+ checks.Error(\n+ \"'max_length' is too small to fit the longest value \"\n+ \"in 'choices' (%s characters).\" % choice_max_length,\n+ obj=self,\n+ id='fields.E009',\n+ )\n+ ]\n return []\n \n"} +{"instance_id": "matplotlib__matplotlib-26011", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/lib/matplotlib/axis.py\n+++ b/lib/matplotlib/axis.py\n@@ -1243,6 +1243,7 @@\n for other in self._get_shared_axes():\n if other is not self.axes:\n other._axis_map[name]._set_lim(\n v0, v1, emit=False, auto=auto)\n+ other.callbacks.process(f\"{name}lim_changed\", other)\n if other.figure != self.figure:\n other.figure.canvas.draw_idle()\n"} +{"instance_id": "mwaskom__seaborn-3407", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/seaborn/axisgrid.py\n+++ b/seaborn/axisgrid.py\n@@ -1472,7 +1472,7 @@\n for ax in diag_axes[1:]:\n share_axis(diag_axes[0], ax, \"y\")\n \n- self.diag_vars = np.array(diag_vars, np.object_)\n+ self.diag_vars = diag_vars\n self.diag_axes = np.array(diag_axes, np.object_)\n \n if \"hue\" not in signature(func).parameters:\n"} +{"instance_id": "pallets__flask-4045", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/src/flask/blueprints.py\n+++ b/src/flask/blueprints.py\n@@ -188,6 +188,9 @@\n template_folder=template_folder,\n root_path=root_path,\n )\n+ if \".\" in name:\n+ raise ValueError(\"'name' may not contain a dot '.' character.\")\n+\n self.name = name\n self.url_prefix = url_prefix\n self.subdomain = subdomain\n--- a/tests/test_blueprints.py\n+++ b/tests/test_blueprints.py\n@@ -7,6 +7,12 @@\n import flask\n \n \n+@pytest.mark.parametrize(\"name\", [\".name\", \"na.me\", \"name.\"])\n+def test_blueprint_name_with_dot(name):\n+ with pytest.raises(ValueError, match=\"may not contain a dot\"):\n+ flask.Blueprint(name, __name__)\n+\n+\n def test_blueprint_specific_error_handling(app, client):\n frontend = flask.Blueprint(\"frontend\", __name__)\n backend = flask.Blueprint(\"backend\", __name__)\n"} +{"instance_id": "psf__requests-2148", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/requests/packages/urllib3/response.py\n+++ b/requests/packages/urllib3/response.py\n@@ -1,6 +1,6 @@\n import zlib\n import io\n-from socket import timeout as SocketTimeout\n+from socket import error as SocketError, timeout as SocketTimeout\n \n from ._collections import HTTPHeaderDict\n from .exceptions import ProtocolError, DecodeError, ReadTimeoutError\n@@ -209,3 +209,3 @@\n- except HTTPException as e:\n+ except (HTTPException, SocketError) as e:\n # This includes IncompleteRead.\n raise ProtocolError('Connection broken: %r' % e, e)\n--- a/requests/models.py\n+++ b/requests/models.py\n@@ -19,7 +19,7 @@\n from .packages.urllib3.fields import RequestField\n from .packages.urllib3.filepost import encode_multipart_formdata\n from .packages.urllib3.util import parse_url\n-from .packages.urllib3.exceptions import DecodeError\n+from .packages.urllib3.exceptions import DecodeError, ProtocolError\n from .exceptions import (\n HTTPError, RequestException, MissingSchema, InvalidURL,\n ChunkedEncodingError, ContentDecodingError)\n@@ -637,5 +637,5 @@\n for chunk in self.raw.stream(chunk_size, decode_content=True):\n yield chunk\n- except IncompleteRead as e:\n+ except (IncompleteRead, ProtocolError) as e:\n raise ChunkedEncodingError(e)\n except DecodeError as e:\n"} +{"instance_id": "pydata__xarray-3364", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/xarray/core/concat.py\n+++ b/xarray/core/concat.py\n@@ -191,6 +191,15 @@\n # all nonindexes that are not the same in each dataset\n- for k in getattr(datasets[0], subset):\n+ names = dict.fromkeys(\n+ k for ds in datasets for k in getattr(ds, subset)\n+ ) if subset == \"data_vars\" else getattr(datasets[0], subset)\n+ for k in names:\n if k not in concat_over:\n+ if subset == \"data_vars\" and any(\n+ k not in ds.variables for ds in datasets\n+ ):\n+ concat_over.add(k)\n+ equals[k] = False\n+ continue\n # Compare the variable of all datasets vs. the one\n # of the first dataset. Perform the minimum amount of\n # loads in order to avoid multiple loads from disk\n@@ -217,5 +227,6 @@\n elif opt == \"all\":\n concat_over.update(\n- set(getattr(datasets[0], subset)) - set(datasets[0].dims)\n+ k for ds in (datasets if subset == \"data_vars\" else datasets[:1])\n+ for k in getattr(ds, subset) if k not in ds.dims\n )\n elif opt == \"minimal\":\n@@ -225,3 +236,7 @@\n else:\n- invalid_vars = [k for k in opt if k not in getattr(datasets[0], subset)]\n+ available = set(getattr(datasets[0], subset))\n+ if subset == \"data_vars\":\n+ for ds in datasets[1:]:\n+ available.update(ds.data_vars)\n+ invalid_vars = [k for k in opt if k not in available]\n if invalid_vars:\n@@ -235,4 +250,4 @@\n \"some variables in data_vars are not data variables \"\n- \"on the first dataset: %s\" % (invalid_vars,)\n+ \"on any dataset: %s\" % (invalid_vars,)\n )\n concat_over.update(opt)\n@@ -315,3 +330,3 @@\n- absent_merge_vars = variables_to_merge - set(ds.variables)\n+ absent_merge_vars = (variables_to_merge & coord_names) - set(ds.variables)\n if absent_merge_vars:\n raise ValueError(\n@@ -323,3 +338,4 @@\n for var in variables_to_merge:\n- to_merge[var].append(ds.variables[var])\n+ if var in ds.variables:\n+ to_merge[var].append(ds.variables[var])\n \n@@ -359,7 +375,27 @@\n- for k in datasets[0].variables:\n+ variable_names = dict.fromkeys(k for ds in datasets for k in ds.variables)\n+ for k in variable_names:\n if k in concat_over:\n- try:\n- vars = ensure_common_dims([ds.variables[k] for ds in datasets])\n- except KeyError:\n+ if k in coord_names and any(k not in ds.variables for ds in datasets):\n raise ValueError(\"%r is not present in all datasets.\" % k)\n+ template = next(ds.variables[k] for ds in datasets if k in ds.variables)\n+ vars = []\n+ for ds, dim_len in zip(datasets, concat_dim_lengths):\n+ if k in ds.variables:\n+ vars.append(ds.variables[k])\n+ continue\n+ missing_dims = template.dims\n+ if dim not in missing_dims:\n+ missing_dims = (dim,) + missing_dims\n+ shape = tuple(\n+ dim_len if d == dim else dims_sizes[d] for d in missing_dims\n+ )\n+ size = 1\n+ for length in shape:\n+ size *= length\n+ missing = pd.Series([], dtype=template.dtype).reindex(\n+ pd.RangeIndex(size),\n+ fill_value=None if fill_value is dtypes.NA else fill_value,\n+ ).values.reshape(shape)\n+ vars.append(Variable(missing_dims, missing, attrs=template.attrs))\n+ vars = ensure_common_dims(vars)\n combined = concat_vars(vars, dim, positions)\n"} +{"instance_id": "pylint-dev__pylint-5859", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/pylint/checkers/misc.py\n+++ b/pylint/checkers/misc.py\n@@ -122,7 +122,7 @@\n notes = \"|\".join(re.escape(note) for note in self.config.notes)\n if self.config.notes_rgx:\n- regex_string = rf\"#\\s*({notes}|{self.config.notes_rgx})\\b\"\n+ regex_string = rf\"#\\s*({notes}|{self.config.notes_rgx})(?=[:\\s]|$)\"\n else:\n- regex_string = rf\"#\\s*({notes})\\b\"\n+ regex_string = rf\"#\\s*({notes})(?=[:\\s]|$)\"\n \n self._fixme_pattern = re.compile(regex_string, re.I)\n"} +{"instance_id": "pytest-dev__pytest-5227", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/src/_pytest/logging.py\n+++ b/src/_pytest/logging.py\n@@ -16,4 +16,4 @@\n from _pytest.pathlib import Path\n \n-DEFAULT_LOG_FORMAT = \"%(filename)-25s %(lineno)4d %(levelname)-8s %(message)s\"\n+DEFAULT_LOG_FORMAT = \"%(levelname)-8s %(name)s:%(filename)s:%(lineno)d %(message)s\"\n DEFAULT_LOG_DATE_FORMAT = \"%H:%M:%S\"\n--- a/testing/logging/test_reporting.py\n+++ b/testing/logging/test_reporting.py\n@@ -282,6 +282,6 @@\n result.stdout.fnmatch_lines(\n [\n \"test_log_cli_default_level.py::test_log_cli \",\n- \"test_log_cli_default_level.py*WARNING message will be shown*\",\n+ \"*WARNING*catchlog:test_log_cli_default_level.py:*WARNING message will be shown*\",\n ]\n )\n"} +{"instance_id": "scikit-learn__scikit-learn-11040", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/sklearn/neighbors/base.py\n+++ b/sklearn/neighbors/base.py\n@@ -9,6 +9,7 @@\n import warnings\n from abc import ABCMeta, abstractmethod\n+from numbers import Integral\n \n import numpy as np\n from scipy.sparse import csr_matrix, issparse\n \n@@ -157,3 +158,12 @@\n def _fit(self, X):\n+ if self.n_neighbors is not None:\n+ if not isinstance(self.n_neighbors, Integral):\n+ raise ValueError(\"n_neighbors must be a positive integer; \"\n+ \"got %r\" % self.n_neighbors)\n+ if self.n_neighbors <= 0:\n+ raise ValueError(\n+ \"Expected n_neighbors > 0. Got %d\" %\n+ self.n_neighbors)\n+\n self._check_algorithm_metric()\n if self.metric_params is None:\n@@ -255,10 +265,3 @@\n- if self.n_neighbors is not None:\n- if self.n_neighbors <= 0:\n- raise ValueError(\n- \"Expected n_neighbors > 0. Got %d\" %\n- self.n_neighbors\n- )\n-\n return self\n \n @property\n@@ -329,6 +332,14 @@\n if n_neighbors is None:\n n_neighbors = self.n_neighbors\n \n+ if not isinstance(n_neighbors, Integral):\n+ raise ValueError(\"n_neighbors must be a positive integer; \"\n+ \"got %r\" % n_neighbors)\n+ if n_neighbors <= 0:\n+ raise ValueError(\n+ \"Expected n_neighbors > 0. Got %d\" %\n+ n_neighbors)\n+\n if X is not None:\n query_is_train = False\n X = check_array(X, accept_sparse='csr')\n"} +{"instance_id": "sphinx-doc__sphinx-7975", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": ""} +{"instance_id": "sympy__sympy-13437", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/sympy/functions/combinatorial/numbers.py\n+++ b/sympy/functions/combinatorial/numbers.py\n@@ -434,5 +434,8 @@\n @classmethod\n def eval(cls, n, k_sym=None, symbols=None):\n+ if n is S.Infinity and k_sym is None and symbols is None:\n+ return S.Infinity\n+\n if n.is_Integer and n.is_nonnegative:\n if k_sym is None:\n return Integer(cls._bell(int(n)))\n--- a/sympy/functions/combinatorial/tests/test_comb_numbers.py\n+++ b/sympy/functions/combinatorial/tests/test_comb_numbers.py\n@@ -80,4 +80,8 @@\n def test_bell():\n assert [bell(n) for n in range(8)] == [1, 1, 2, 5, 15, 52, 203, 877]\n \n+ assert bell(S.Infinity) == S.Infinity\n+ n = Dummy('n')\n+ assert bell(n).limit(n, S.Infinity) == S.Infinity\n+\n assert bell(0, x) == 1\n"} +{"instance_id": "astropy__astropy-7746", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/astropy/wcs/wcs.py\n+++ b/astropy/wcs/wcs.py\n@@ -1222,7 +1222,7 @@\n \n if ra_dec_order and sky == 'input':\n xy = self._denormalize_sky(xy)\n- output = func(xy, origin)\n+ output = xy if xy.size == 0 else func(xy, origin)\n if ra_dec_order and sky == 'output':\n output = self._normalize_sky(output)\n return (output[:, 0].reshape(axes[0].shape),\n@@ -1239,7 +1239,7 @@\n \"of shape (N, {0})\".format(self.naxis))\n if ra_dec_order and sky == 'input':\n xy = self._denormalize_sky(xy)\n- result = func(xy, origin)\n+ result = xy if xy.size == 0 else func(xy, origin)\n if ra_dec_order and sky == 'output':\n result = self._normalize_sky(result)\n return result\n"} diff --git a/scripts/aggregate_patch_results.py b/scripts/aggregate_patch_results.py index 6587069..9aeb3f8 100644 --- a/scripts/aggregate_patch_results.py +++ b/scripts/aggregate_patch_results.py @@ -18,6 +18,24 @@ def aggregate(predictions: Path, artifacts: Path) -> dict: ids = [row["instance_id"] for row in rows] if len(ids) != len(set(ids)) or set(ids) - expected: raise ValueError("duplicate or unknown shard outcomes") + prediction_map = {p["instance_id"]: p for p in submitted} + for row in rows: + if row["status"] not in { + "resolved", + "unresolved", + "empty_patch", + "infrastructure_error", + "evaluation_error", + } or row.get("resolved") is not (row["status"] == "resolved"): + raise ValueError("invalid or inconsistent evaluation outcome") + patch = prediction_map[row["instance_id"]].get("model_patch", "") + if ( + "patch_sha256" in row + and row["patch_sha256"] != hashlib.sha256(patch.encode()).hexdigest() + ): + raise ValueError("evaluation patch hash does not match frozen prediction") + if row["resolved"] and not patch: + raise ValueError("cannot resolve an empty patch") missing = sorted(expected - set(ids)) resolved = sorted( r["instance_id"] for r in rows if r["status"] == "resolved" and r["resolved"] is True diff --git a/scripts/evaluate_patches.py b/scripts/evaluate_patches.py index 3b444fc..6af6656 100644 --- a/scripts/evaluate_patches.py +++ b/scripts/evaluate_patches.py @@ -89,8 +89,16 @@ def main() -> None: raise ValueError("invalid instance ID") case = args.output / identifier case.mkdir() + patch_sha256 = hashlib.sha256(prediction.get("model_patch", "").encode()).hexdigest() if not prediction.get("model_patch", "").strip(): - outcomes.append({"instance_id": identifier, "status": "empty_patch", "resolved": False}) + outcomes.append( + { + "instance_id": identifier, + "status": "empty_patch", + "resolved": False, + "patch_sha256": patch_sha256, + } + ) write_jsonl(args.output / "outcomes.jsonl", outcomes) continue row = reference[identifier] @@ -102,6 +110,7 @@ def main() -> None: "status": "infrastructure_error", "resolved": False, "image": image, + "patch_sha256": patch_sha256, } try: with (case / "image-pull.log").open("w") as stream: diff --git a/tests/test_evaluation_runs.py b/tests/test_evaluation_runs.py index 0a5c963..f939301 100644 --- a/tests/test_evaluation_runs.py +++ b/tests/test_evaluation_runs.py @@ -67,7 +67,13 @@ def load_aggregator(): def test_patch_aggregation_keeps_empty_error_and_missing_in_denominator(tmp_path): predictions = tmp_path / "predictions.jsonl" - write_jsonl(predictions, [{"instance_id": name} for name in ("a", "b", "c", "d")]) + write_jsonl( + predictions, + [ + {"instance_id": name, "model_patch": "patch" if name == "a" else ""} + for name in ("a", "b", "c", "d") + ], + ) artifacts = tmp_path / "artifacts" write_jsonl( artifacts / "one/outcomes.jsonl", From 93e32c246c0f6c90a67c8acfabe285ddbe0ea97b Mon Sep 17 00:00:00 2001 From: YeDonS <120633988+YeDonS@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:21:02 -0400 Subject: [PATCH 2/7] eval: freeze predictions 26 through 50 for parallel official testing --- reports/model-batches/batch-02/manifest.json | 140 ++++++++++++++++++ .../model-batches/batch-02/predictions.jsonl | 25 ++++ .../batch-02/swebench-predictions.jsonl | 25 ++++ scripts/freeze_batch.py | 56 +++++++ 4 files changed, 246 insertions(+) create mode 100644 reports/model-batches/batch-02/manifest.json create mode 100644 reports/model-batches/batch-02/predictions.jsonl create mode 100644 reports/model-batches/batch-02/swebench-predictions.jsonl create mode 100644 scripts/freeze_batch.py diff --git a/reports/model-batches/batch-02/manifest.json b/reports/model-batches/batch-02/manifest.json new file mode 100644 index 0000000..9afabe0 --- /dev/null +++ b/reports/model-batches/batch-02/manifest.json @@ -0,0 +1,140 @@ +{ + "kind": "frozen_inference_batch", + "batch": 2, + "start": 25, + "stop": 50, + "count": 25, + "parent_manifest_sha256": "84c775f6f20462427a272c70a47e09a23c92a2ad6c5234949aa3cdd706c0f07b", + "predictions_sha256": "3e3e3741403ad4b82f806bb9169c142b4a37de98874d352486b10a0a8d7be4d6", + "parent_configuration": { + "mode": "model", + "count": 100, + "task_ids": [ + "astropy__astropy-12907", + "django__django-16255", + "matplotlib__matplotlib-23964", + "mwaskom__seaborn-2848", + "pallets__flask-4992", + "psf__requests-2317", + "pydata__xarray-4094", + "pylint-dev__pylint-7993", + "pytest-dev__pytest-5692", + "scikit-learn__scikit-learn-13584", + "sphinx-doc__sphinx-8713", + "sympy__sympy-12419", + "astropy__astropy-14365", + "django__django-11742", + "matplotlib__matplotlib-26011", + "mwaskom__seaborn-3407", + "pallets__flask-4045", + "psf__requests-2148", + "pydata__xarray-3364", + "pylint-dev__pylint-5859", + "pytest-dev__pytest-5227", + "scikit-learn__scikit-learn-11040", + "sphinx-doc__sphinx-7975", + "sympy__sympy-13437", + "astropy__astropy-7746", + "django__django-13757", + "matplotlib__matplotlib-25498", + "mwaskom__seaborn-3190", + "pallets__flask-5063", + "psf__requests-3362", + "pydata__xarray-4493", + "pylint-dev__pylint-7080", + "pytest-dev__pytest-5103", + "scikit-learn__scikit-learn-14894", + "sphinx-doc__sphinx-8801", + "sympy__sympy-11870", + "astropy__astropy-14995", + "django__django-10914", + "matplotlib__matplotlib-23563", + "mwaskom__seaborn-3010", + "psf__requests-1963", + "pydata__xarray-4248", + "pylint-dev__pylint-7114", + "pytest-dev__pytest-8906", + "scikit-learn__scikit-learn-15512", + "sphinx-doc__sphinx-8506", + "sympy__sympy-20590", + "astropy__astropy-14182", + "django__django-13447", + "matplotlib__matplotlib-23314", + "psf__requests-2674", + "pydata__xarray-5131", + "pylint-dev__pylint-7228", + "pytest-dev__pytest-5413", + "scikit-learn__scikit-learn-14087", + "sphinx-doc__sphinx-8595", + "sympy__sympy-18087", + "astropy__astropy-6938", + "django__django-12184", + "matplotlib__matplotlib-24265", + "psf__requests-863", + "pylint-dev__pylint-6506", + "pytest-dev__pytest-7373", + "scikit-learn__scikit-learn-14983", + "sphinx-doc__sphinx-8282", + "sympy__sympy-13647", + "django__django-13448", + "matplotlib__matplotlib-18869", + "pytest-dev__pytest-5495", + "scikit-learn__scikit-learn-14092", + "sphinx-doc__sphinx-7686", + "sympy__sympy-20154", + "django__django-11001", + "matplotlib__matplotlib-24149", + "pytest-dev__pytest-11143", + "scikit-learn__scikit-learn-13497", + "sphinx-doc__sphinx-7738", + "sympy__sympy-13480", + "django__django-12983", + "matplotlib__matplotlib-25332", + "pytest-dev__pytest-7168", + "scikit-learn__scikit-learn-25500", + "sphinx-doc__sphinx-8721", + "sympy__sympy-19487", + "django__django-14997", + "matplotlib__matplotlib-24334", + "pytest-dev__pytest-11148", + "scikit-learn__scikit-learn-25747", + "sphinx-doc__sphinx-8435", + "sympy__sympy-13895", + "django__django-12497", + "matplotlib__matplotlib-26020", + "pytest-dev__pytest-7490", + "scikit-learn__scikit-learn-25638", + "sphinx-doc__sphinx-8273", + "sympy__sympy-14308", + "django__django-13401", + "matplotlib__matplotlib-25079", + "pytest-dev__pytest-8365", + "scikit-learn__scikit-learn-13142" + ], + "tasks_sha256": "933b5103f3a746835a41a71422d84487b504291ca577b5a450f9bf280b458a18", + "model": "codex-cli:gpt-6.1-sol", + "seed": 2026, + "created_at": "2026-10-08T18:05:29.872171+00:00", + "test_executor": "disabled", + "max_tool_calls": 18, + "max_search_passes": 2, + "max_agent_steps": 5, + "model_timeout": 120, + "codex_reasoning_effort": "medium", + "codex_transport": "https", + "workers": 6, + "environment": { + "python": "3.12.13", + "platform": "macOS-15.4.1-arm64-arm-64bit", + "source_tree_sha256": "d4e5b6d72a4d66ac37e10f45ae0c900d24a439de2ee2ff1d5c6ca68fd5f1ddf1", + "packages": { + "langchain": "1.4.3", + "langgraph": "1.2.14", + "fastapi": "0.143.0", + "numpy": "2.5.3", + "rank-bm25": "0.2.2" + } + }, + "codex_cli_version": "codex-cli 0.162.0-alpha.2" + } +} diff --git a/reports/model-batches/batch-02/predictions.jsonl b/reports/model-batches/batch-02/predictions.jsonl new file mode 100644 index 0000000..6ea7818 --- /dev/null +++ b/reports/model-batches/batch-02/predictions.jsonl @@ -0,0 +1,25 @@ +{"instance_id": "django__django-13757", "mode": "model", "status": "diagnosed", "predicted_files": ["django/db/models/fields/json.py", "tests/model_fields/test_jsonfield.py"], "root_cause": "For isnull=True, KeyTransformIsNull delegates to generic IsNull on SQLite and Oracle. This tests whether the extracted value is SQL NULL, conflating an absent key with a present key containing JSON null. The existing regression test uses a non-null-valued key and misses this distinction.", "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "matplotlib__matplotlib-25498", "mode": "model", "status": "diagnosed", "predicted_files": ["lib/matplotlib/colorbar.py"], "root_cause": "Replacing the mappable's norm triggers a colorbar update before the new norm is autoscaled from the image data. Colorbar._process_values instead assigns an unscaled norm limits of 0 and 1, which are invalid for LogNorm. Initial colorbar construction autoscales the mappable, but the update path lacks that step.", "model_patch": "--- a/lib/matplotlib/colorbar.py\n+++ b/lib/matplotlib/colorbar.py\n@@ -1104,6 +1104,9 @@\n b = np.hstack((b, b[-1] + 1))\n \n # transform from 0-1 to vmin-vmax:\n+ if self.mappable.get_array() is not None:\n+ with self.norm.callbacks.blocked():\n+ self.mappable.autoscale_None()\n if not self.norm.scaled():\n self.norm.vmin = 0\n self.norm.vmax = 1\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "mwaskom__seaborn-3190", "mode": "model", "status": "diagnosed", "predicted_files": ["seaborn/_core/scales.py"], "root_cause": "Boolean color data selects Continuous through Property.default_scale. Color requires normalization, but ContinuousBase._setup subtracts boolean minimum and maximum values without numeric coercion, causing NumPy's boolean-subtraction TypeError.", "model_patch": "--- a/seaborn/_core/scales.py\n+++ b/seaborn/_core/scales.py\n@@ -349,3 +349,3 @@\n- vmin, vmax = axis.convert_units((vmin, vmax))\n+ vmin, vmax = map(float, axis.convert_units((vmin, vmax)))\n a = forward(vmin)\n b = forward(vmax) - forward(vmin)\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pallets__flask-5063", "mode": "model", "status": "diagnosed", "predicted_files": ["src/flask/cli.py"], "root_cause": "The routes CLI reads registered URL rules but displays only Endpoint, Methods, and Rule, omitting subdomain metadata. Blueprint registration already forwards the configured subdomain to app.add_url_rule; this is a presentation gap.", "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "psf__requests-3362", "mode": "model", "status": "diagnosed", "predicted_files": ["requests/utils.py", "requests/models.py", "requests/adapters.py"], "root_cause": "In this snapshot, application/json without a charset leaves r.encoding as None. iter_content(decode_unicode=True) then yields bytes unchanged, whereas r.text falls back to apparent_encoding and decodes. The APIs therefore differ when no encoding is set.", "model_patch": "--- a/requests/utils.py\n+++ b/requests/utils.py\n@@ -354,6 +354,9 @@\n \n if 'text' in content_type:\n return 'ISO-8859-1'\n \n+ if content_type.lower() == 'application/json':\n+ return 'utf-8'\n+\n \n def stream_decode_response_unicode(iterator, r):\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pydata__xarray-4493", "mode": "model", "status": "diagnosed", "predicted_files": ["xarray/core/variable.py"], "root_cause": "Tuple-wrapped DataArrays bypass the top-level DataArray unwrapping in as_variable and are passed directly to Variable. Its as_compatible_data conversion accesses the DataArray's .values, eagerly materializing its dask backing array. This shared conversion explains the reported behavior in both Dataset construction and update.", "model_patch": "--- a/xarray/core/variable.py\n+++ b/xarray/core/variable.py\n@@ -198,10 +198,12 @@\n Finally, wrap it up with an adapter if necessary.\n \"\"\"\n+ from .dataarray import DataArray\n+\n if fastpath and getattr(data, \"ndim\", 0) > 0:\n # can't use fastpath (yet) for scalars\n return _maybe_wrap_data(data)\n \n- if isinstance(data, Variable):\n+ if isinstance(data, (Variable, DataArray)):\n return data.data\n \n if isinstance(data, NON_NUMPY_SUPPORTED_ARRAY_TYPES):\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pylint-dev__pylint-7080", "mode": "model", "status": "insufficient_evidence", "predicted_files": [], "root_cause": null, "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pytest-dev__pytest-5103", "mode": "model", "status": "insufficient_evidence", "predicted_files": [], "root_cause": null, "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "scikit-learn__scikit-learn-14894", "mode": "model", "status": "diagnosed", "predicted_files": ["sklearn/svm/base.py"], "root_cause": "When support_vectors_ is empty, _sparse_fit passes a zero step to np.arange while constructing CSR row pointers. For SVR, n_class is 1; the failure is caused by the zero step, not a zero class count.", "model_patch": "--- a/sklearn/svm/base.py\n+++ b/sklearn/svm/base.py\n@@ -289,6 +289,9 @@\n dual_coef_indices = np.tile(np.arange(n_SV), n_class)\n- dual_coef_indptr = np.arange(0, dual_coef_indices.size + 1,\n- dual_coef_indices.size / n_class)\n+ if n_SV == 0:\n+ dual_coef_indptr = np.zeros(n_class + 1, dtype=np.int32)\n+ else:\n+ dual_coef_indptr = np.arange(0, dual_coef_indices.size + 1,\n+ dual_coef_indices.size / n_class)\n self.dual_coef_ = sp.csr_matrix(\n (dual_coef_data, dual_coef_indices, dual_coef_indptr),\n (n_class, n_SV))\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "sphinx-doc__sphinx-8801", "mode": "model", "status": "diagnosed", "predicted_files": ["sphinx/ext/autodoc/importer.py"], "root_cause": "get_class_members() registers annotation-only attributes before processing analyzer documentation. Its subsequent `name not in members` condition prevents attaching that documentation to the existing ObjectMember. For inherited attributes, the subclass namespace lookup cannot recover the superclass documentation, so filter_members() excludes them unless :undoc-members: is enabled.", "model_patch": "--- a/sphinx/ext/autodoc/importer.py\n+++ b/sphinx/ext/autodoc/importer.py\n@@ -307,8 +307,11 @@\n analyzer = ModuleAnalyzer.for_module(modname)\n analyzer.analyze()\n for (ns, name), docstring in analyzer.attr_docs.items():\n- if ns == qualname and name not in members:\n- members[name] = ObjectMember(name, INSTANCEATTR, class_=cls,\n- docstring='\\n'.join(docstring))\n+ if ns == qualname:\n+ if name not in members:\n+ members[name] = ObjectMember(name, INSTANCEATTR, class_=cls,\n+ docstring='\\n'.join(docstring))\n+ elif members[name].class_ is cls:\n+ members[name].docstring = '\\n'.join(docstring)\n except (AttributeError, PycodeError):\n pass\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "sympy__sympy-11870", "mode": "model", "status": "diagnosed", "predicted_files": ["sympy/simplify/trigsimp.py"], "root_cause": "The default trigsimp path calls futrig, whose helper immediately returns expressions without TrigonometricFunction instances. The supplied expression contains exponentials instead, so it bypasses those transformations. A separate API, exptrigsimp, handles exponential-to-trigonometric rewriting. Automatic conversion to sinc is not implemented in the inspected transformations.", "model_patch": "--- a/sympy/simplify/trigsimp.py\n+++ b/sympy/simplify/trigsimp.py\n@@ -443,6 +443,9 @@\n collection of trigonometric transformations described by Fu, et al.\n (see the `fu` docstring).\n \n+ For expressions involving exponentials, use ``exptrigsimp`` to simplify\n+ exponential / trigonometric / hyperbolic identities.\n+\n \n Examples\n ========\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "astropy__astropy-14995", "mode": "model", "status": "diagnosed", "predicted_files": ["astropy/nddata/mixins/ndarithmetic.py"], "root_cause": "_arithmetic_mask checks whether operand is None but not whether operand.mask is None when the left operand has a mask. A maskless operand therefore reaches handle_mask(self.mask, None), causing np.bitwise_or to fail. Scalar operands also take this path because they are wrapped as NDData-like objects.", "model_patch": "--- a/astropy/nddata/mixins/ndarithmetic.py\n+++ b/astropy/nddata/mixins/ndarithmetic.py\n@@ -522,7 +522,7 @@\n # Make a copy so there is no reference in the result.\n return deepcopy(operand.mask)\n- elif operand is None:\n+ elif operand is None or operand.mask is None:\n return deepcopy(self.mask)\n else:\n # Now lets calculate the resulting mask (operation enforces copy)\n return handle_mask(self.mask, operand.mask, **kwds)\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "django__django-10914", "mode": "model", "status": "diagnosed", "predicted_files": ["django/conf/global_settings.py", "docs/howto/deployment/checklist.txt"], "root_cause": "FILE_UPLOAD_PERMISSIONS defaults to None, so FileSystemStorage skips its final chmod. Streamed memory uploads are created with 0o666 masked by umask, while temporary uploads are moved by rename or metadata-preserving copy, retaining their temporary-file permissions. Consequently, upload paths can produce different final permissions.", "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "matplotlib__matplotlib-23563", "mode": "model", "status": "diagnosed", "predicted_files": ["lib/mpl_toolkits/mplot3d/art3d.py"], "root_cause": "line_2d_to_3d changes an attached line's class to Line3D before initializing _verts3d. If broadcasting zs fails in set_3d_properties, the artist remains attached without _verts3d, and subsequent drawing raises AttributeError. Changing input variables does not repair that retained artist.", "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "mwaskom__seaborn-3010", "mode": "model", "status": "diagnosed", "predicted_files": ["seaborn/_stats/regression.py"], "root_cause": "PolyFit._fit_predict passes missing x/y values directly to np.polyfit. Its x.nunique() check counts valid unique values but does not remove missing rows, and GroupBy.apply does not filter them. Missing coordinates can therefore cause the reported least-squares failure.", "model_patch": "--- a/seaborn/_stats/regression.py\n+++ b/seaborn/_stats/regression.py\n@@ -22,6 +22,7 @@\n def _fit_predict(self, data):\n \n+ data = data.dropna(subset=[\"x\", \"y\"])\n x = data[\"x\"]\n y = data[\"y\"]\n if x.nunique() <= self.order:\n # TODO warn?\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "psf__requests-1963", "mode": "model", "status": "diagnosed", "predicted_files": ["requests/sessions.py"], "root_cause": "resolve_redirects copies req and reads req.method on every iteration without updating req. A 303 changes only prepared_request to GET and clears its body; a subsequent 307 copies the original POST again, restoring its method, body, and Content-Length.", "model_patch": "--- a/requests/sessions.py\n+++ b/requests/sessions.py\n@@ -166,6 +166,8 @@\n # .netrc might have more auth for us.\n new_auth = get_netrc_auth(url) if self.trust_env else None\n if new_auth is not None:\n prepared_request.prepare_auth(new_auth)\n \n+ req = prepared_request\n+\n resp = self.send(\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pydata__xarray-4248", "mode": "model", "status": "insufficient_evidence", "predicted_files": [], "root_cause": null, "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pylint-dev__pylint-7114", "mode": "model", "status": "diagnosed", "predicted_files": ["pylint/lint/expand_modules.py"], "root_cause": "get_python_path() treats an uninitialized directory as an import root instead of using its parent. Prepending a/ to sys.path lets a/a.py shadow namespace package a, explaining the incorrect import resolution. expand_modules() then uses name-based resolution to classify the explicit directory as a non-namespace module and schedules its nonexistent a/__init__.py for parsing.", "model_patch": "--- a/pylint/lint/expand_modules.py\n+++ b/pylint/lint/expand_modules.py\n@@ -32,6 +32,8 @@\n dirname = os.path.realpath(os.path.expanduser(filepath))\n if not os.path.isdir(dirname):\n dirname = os.path.dirname(dirname)\n+ elif not os.path.exists(os.path.join(dirname, \"__init__.py\")):\n+ dirname = os.path.dirname(dirname)\n while True:\n if not os.path.exists(os.path.join(dirname, \"__init__.py\")):\n return dirname\n@@ -121,6 +123,9 @@\n else:\n is_namespace = modutils.is_namespace(spec)\n is_directory = modutils.is_directory(spec)\n+ if os.path.isdir(something) and not os.path.exists(filepath):\n+ is_namespace = True\n+ is_directory = True\n if not is_namespace:\n result.append(\n {\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pytest-dev__pytest-8906", "mode": "model", "status": "insufficient_evidence", "predicted_files": [], "root_cause": null, "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "scikit-learn__scikit-learn-15512", "mode": "model", "status": "diagnosed", "predicted_files": ["sklearn/cluster/_affinity_propagation.py"], "root_cause": "Iteration exhaustion only prints a message; it does not prevent clustering from the final exemplars. Whenever K > 0, ordinary centers and labels are returned despite non-convergence, contradicting the documented contract. The estimator directly exposes those results.", "model_patch": "--- a/sklearn/cluster/_affinity_propagation.py\n+++ b/sklearn/cluster/_affinity_propagation.py\n@@ -156,6 +156,7 @@\n ind = np.arange(n_samples)\n \n+ converged = False\n for it in range(max_iter):\n # tmp = A + S; compute responsibilities\n np.add(A, S, tmp)\n I = np.argmax(tmp, axis=1)\n@@ -195,8 +196,9 @@\n if it >= convergence_iter:\n se = np.sum(e, axis=1)\n unconverged = (np.sum((se == convergence_iter) + (se == 0))\n != n_samples)\n- if (not unconverged and (K > 0)) or (it == max_iter):\n+ if not unconverged and (K > 0):\n+ converged = True\n if verbose:\n print(\"Converged after %d iterations.\" % it)\n break\n@@ -204,6 +206,6 @@\n I = np.flatnonzero(E)\n K = I.size # Identify exemplars\n \n- if K > 0:\n+ if converged and K > 0:\n c = np.argmax(S[:, I], axis=1)\n c[I] = np.arange(K) # Identify clusters\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "sphinx-doc__sphinx-8506", "mode": "model", "status": "diagnosed", "predicted_files": ["sphinx/domains/std.py"], "root_cause": "The option-description regex excludes square brackets from option names, so it cannot match a signature beginning with '[' such as '[enable=]PATTERN'. This restriction explains the reported rejection.", "model_patch": "--- a/sphinx/domains/std.py\n+++ b/sphinx/domains/std.py\n@@ -45,4 +45,4 @@\n # RE for option descriptions\n-option_desc_re = re.compile(r'((?:/|--|-|\\+)?[^\\s=[]+)(=?\\s*.*)')\n+option_desc_re = re.compile(r'((?:/|--|-|\\+)?(?:\\[[^\\]]*\\]|[^\\s=[])+)(=?\\s*.*)')\n # RE for grammar tokens\n token_re = re.compile(r'`(\\w+)`', re.U)\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "sympy__sympy-20590", "mode": "model", "status": "insufficient_evidence", "predicted_files": [], "root_cause": null, "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "astropy__astropy-14182", "mode": "model", "status": "diagnosed", "predicted_files": ["astropy/io/ascii/rst.py"], "root_cause": "RST.__init__ rejects header_rows even though FixedWidth supports it. RST also assumes exactly one header row: write() selects lines[1] as the separator, and reading starts data at the fixed offset 3.", "model_patch": "--- a/astropy/io/ascii/rst.py\n+++ b/astropy/io/ascii/rst.py\n@@ -60,7 +60,11 @@\n- def __init__(self):\n- super().__init__(delimiter_pad=None, bookend=False)\n+ def __init__(self, header_rows=None):\n+ super().__init__(\n+ delimiter_pad=None, bookend=False, header_rows=header_rows\n+ )\n+ self.data.start_line = len(self.header.header_rows) + 2\n \n def write(self, lines):\n lines = super().write(lines)\n- lines = [lines[1]] + lines + [lines[1]]\n+ separator = lines[len(self.header.header_rows)]\n+ lines = [separator] + lines + [separator]\n return lines\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "django__django-13447", "mode": "model", "status": "diagnosed", "predicted_files": ["django/contrib/admin/sites.py", "tests/admin_views/test_adminsite.py"], "root_cause": "AdminSite._build_app_dict() has the model class available but omits it from model_dict, so both admin index contexts expose only model metadata. The dictionary-building method is also private.", "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "matplotlib__matplotlib-23314", "mode": "model", "status": "insufficient_evidence", "predicted_files": [], "root_cause": null, "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} diff --git a/reports/model-batches/batch-02/swebench-predictions.jsonl b/reports/model-batches/batch-02/swebench-predictions.jsonl new file mode 100644 index 0000000..6785717 --- /dev/null +++ b/reports/model-batches/batch-02/swebench-predictions.jsonl @@ -0,0 +1,25 @@ +{"instance_id": "django__django-13757", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": ""} +{"instance_id": "matplotlib__matplotlib-25498", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/lib/matplotlib/colorbar.py\n+++ b/lib/matplotlib/colorbar.py\n@@ -1104,6 +1104,9 @@\n b = np.hstack((b, b[-1] + 1))\n \n # transform from 0-1 to vmin-vmax:\n+ if self.mappable.get_array() is not None:\n+ with self.norm.callbacks.blocked():\n+ self.mappable.autoscale_None()\n if not self.norm.scaled():\n self.norm.vmin = 0\n self.norm.vmax = 1\n"} +{"instance_id": "mwaskom__seaborn-3190", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/seaborn/_core/scales.py\n+++ b/seaborn/_core/scales.py\n@@ -349,3 +349,3 @@\n- vmin, vmax = axis.convert_units((vmin, vmax))\n+ vmin, vmax = map(float, axis.convert_units((vmin, vmax)))\n a = forward(vmin)\n b = forward(vmax) - forward(vmin)\n"} +{"instance_id": "pallets__flask-5063", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": ""} +{"instance_id": "psf__requests-3362", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/requests/utils.py\n+++ b/requests/utils.py\n@@ -354,6 +354,9 @@\n \n if 'text' in content_type:\n return 'ISO-8859-1'\n \n+ if content_type.lower() == 'application/json':\n+ return 'utf-8'\n+\n \n def stream_decode_response_unicode(iterator, r):\n"} +{"instance_id": "pydata__xarray-4493", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/xarray/core/variable.py\n+++ b/xarray/core/variable.py\n@@ -198,10 +198,12 @@\n Finally, wrap it up with an adapter if necessary.\n \"\"\"\n+ from .dataarray import DataArray\n+\n if fastpath and getattr(data, \"ndim\", 0) > 0:\n # can't use fastpath (yet) for scalars\n return _maybe_wrap_data(data)\n \n- if isinstance(data, Variable):\n+ if isinstance(data, (Variable, DataArray)):\n return data.data\n \n if isinstance(data, NON_NUMPY_SUPPORTED_ARRAY_TYPES):\n"} +{"instance_id": "pylint-dev__pylint-7080", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": ""} +{"instance_id": "pytest-dev__pytest-5103", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": ""} +{"instance_id": "scikit-learn__scikit-learn-14894", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/sklearn/svm/base.py\n+++ b/sklearn/svm/base.py\n@@ -289,6 +289,9 @@\n dual_coef_indices = np.tile(np.arange(n_SV), n_class)\n- dual_coef_indptr = np.arange(0, dual_coef_indices.size + 1,\n- dual_coef_indices.size / n_class)\n+ if n_SV == 0:\n+ dual_coef_indptr = np.zeros(n_class + 1, dtype=np.int32)\n+ else:\n+ dual_coef_indptr = np.arange(0, dual_coef_indices.size + 1,\n+ dual_coef_indices.size / n_class)\n self.dual_coef_ = sp.csr_matrix(\n (dual_coef_data, dual_coef_indices, dual_coef_indptr),\n (n_class, n_SV))\n"} +{"instance_id": "sphinx-doc__sphinx-8801", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/sphinx/ext/autodoc/importer.py\n+++ b/sphinx/ext/autodoc/importer.py\n@@ -307,8 +307,11 @@\n analyzer = ModuleAnalyzer.for_module(modname)\n analyzer.analyze()\n for (ns, name), docstring in analyzer.attr_docs.items():\n- if ns == qualname and name not in members:\n- members[name] = ObjectMember(name, INSTANCEATTR, class_=cls,\n- docstring='\\n'.join(docstring))\n+ if ns == qualname:\n+ if name not in members:\n+ members[name] = ObjectMember(name, INSTANCEATTR, class_=cls,\n+ docstring='\\n'.join(docstring))\n+ elif members[name].class_ is cls:\n+ members[name].docstring = '\\n'.join(docstring)\n except (AttributeError, PycodeError):\n pass\n"} +{"instance_id": "sympy__sympy-11870", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/sympy/simplify/trigsimp.py\n+++ b/sympy/simplify/trigsimp.py\n@@ -443,6 +443,9 @@\n collection of trigonometric transformations described by Fu, et al.\n (see the `fu` docstring).\n \n+ For expressions involving exponentials, use ``exptrigsimp`` to simplify\n+ exponential / trigonometric / hyperbolic identities.\n+\n \n Examples\n ========\n"} +{"instance_id": "astropy__astropy-14995", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/astropy/nddata/mixins/ndarithmetic.py\n+++ b/astropy/nddata/mixins/ndarithmetic.py\n@@ -522,7 +522,7 @@\n # Make a copy so there is no reference in the result.\n return deepcopy(operand.mask)\n- elif operand is None:\n+ elif operand is None or operand.mask is None:\n return deepcopy(self.mask)\n else:\n # Now lets calculate the resulting mask (operation enforces copy)\n return handle_mask(self.mask, operand.mask, **kwds)\n"} +{"instance_id": "django__django-10914", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": ""} +{"instance_id": "matplotlib__matplotlib-23563", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": ""} +{"instance_id": "mwaskom__seaborn-3010", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/seaborn/_stats/regression.py\n+++ b/seaborn/_stats/regression.py\n@@ -22,6 +22,7 @@\n def _fit_predict(self, data):\n \n+ data = data.dropna(subset=[\"x\", \"y\"])\n x = data[\"x\"]\n y = data[\"y\"]\n if x.nunique() <= self.order:\n # TODO warn?\n"} +{"instance_id": "psf__requests-1963", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/requests/sessions.py\n+++ b/requests/sessions.py\n@@ -166,6 +166,8 @@\n # .netrc might have more auth for us.\n new_auth = get_netrc_auth(url) if self.trust_env else None\n if new_auth is not None:\n prepared_request.prepare_auth(new_auth)\n \n+ req = prepared_request\n+\n resp = self.send(\n"} +{"instance_id": "pydata__xarray-4248", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": ""} +{"instance_id": "pylint-dev__pylint-7114", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/pylint/lint/expand_modules.py\n+++ b/pylint/lint/expand_modules.py\n@@ -32,6 +32,8 @@\n dirname = os.path.realpath(os.path.expanduser(filepath))\n if not os.path.isdir(dirname):\n dirname = os.path.dirname(dirname)\n+ elif not os.path.exists(os.path.join(dirname, \"__init__.py\")):\n+ dirname = os.path.dirname(dirname)\n while True:\n if not os.path.exists(os.path.join(dirname, \"__init__.py\")):\n return dirname\n@@ -121,6 +123,9 @@\n else:\n is_namespace = modutils.is_namespace(spec)\n is_directory = modutils.is_directory(spec)\n+ if os.path.isdir(something) and not os.path.exists(filepath):\n+ is_namespace = True\n+ is_directory = True\n if not is_namespace:\n result.append(\n {\n"} +{"instance_id": "pytest-dev__pytest-8906", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": ""} +{"instance_id": "scikit-learn__scikit-learn-15512", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/sklearn/cluster/_affinity_propagation.py\n+++ b/sklearn/cluster/_affinity_propagation.py\n@@ -156,6 +156,7 @@\n ind = np.arange(n_samples)\n \n+ converged = False\n for it in range(max_iter):\n # tmp = A + S; compute responsibilities\n np.add(A, S, tmp)\n I = np.argmax(tmp, axis=1)\n@@ -195,8 +196,9 @@\n if it >= convergence_iter:\n se = np.sum(e, axis=1)\n unconverged = (np.sum((se == convergence_iter) + (se == 0))\n != n_samples)\n- if (not unconverged and (K > 0)) or (it == max_iter):\n+ if not unconverged and (K > 0):\n+ converged = True\n if verbose:\n print(\"Converged after %d iterations.\" % it)\n break\n@@ -204,6 +206,6 @@\n I = np.flatnonzero(E)\n K = I.size # Identify exemplars\n \n- if K > 0:\n+ if converged and K > 0:\n c = np.argmax(S[:, I], axis=1)\n c[I] = np.arange(K) # Identify clusters\n"} +{"instance_id": "sphinx-doc__sphinx-8506", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/sphinx/domains/std.py\n+++ b/sphinx/domains/std.py\n@@ -45,4 +45,4 @@\n # RE for option descriptions\n-option_desc_re = re.compile(r'((?:/|--|-|\\+)?[^\\s=[]+)(=?\\s*.*)')\n+option_desc_re = re.compile(r'((?:/|--|-|\\+)?(?:\\[[^\\]]*\\]|[^\\s=[])+)(=?\\s*.*)')\n # RE for grammar tokens\n token_re = re.compile(r'`(\\w+)`', re.U)\n"} +{"instance_id": "sympy__sympy-20590", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": ""} +{"instance_id": "astropy__astropy-14182", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": "--- a/astropy/io/ascii/rst.py\n+++ b/astropy/io/ascii/rst.py\n@@ -60,7 +60,11 @@\n- def __init__(self):\n- super().__init__(delimiter_pad=None, bookend=False)\n+ def __init__(self, header_rows=None):\n+ super().__init__(\n+ delimiter_pad=None, bookend=False, header_rows=header_rows\n+ )\n+ self.data.start_line = len(self.header.header_rows) + 2\n \n def write(self, lines):\n lines = super().write(lines)\n- lines = [lines[1]] + lines + [lines[1]]\n+ separator = lines[len(self.header.header_rows)]\n+ lines = [separator] + lines + [separator]\n return lines\n"} +{"instance_id": "django__django-13447", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": ""} +{"instance_id": "matplotlib__matplotlib-23314", "model_name_or_path": "agentic-code-search-gpt-6.1-sol", "model_patch": ""} diff --git a/scripts/freeze_batch.py b/scripts/freeze_batch.py new file mode 100644 index 0000000..15639de --- /dev/null +++ b/scripts/freeze_batch.py @@ -0,0 +1,56 @@ +"""Freeze a completed task interval for official evaluation while inference continues.""" + +import argparse +import hashlib +import json +from pathlib import Path + +from code_assistant.benchmark import export_swebench, read_jsonl, write_jsonl +from code_assistant.telemetry import write_json + + +def freeze(source: Path, destination: Path, start: int, stop: int, batch: int) -> None: + manifest = json.loads((source / "manifest.json").read_text()) + if destination.exists() or not 0 <= start < stop <= manifest["count"]: + raise ValueError("new destination and valid nonempty task interval required") + predictions = read_jsonl(source / "predictions.jsonl") + rows = {r["instance_id"]: r for r in predictions} + if len(rows) != len(predictions): + raise ValueError("duplicate predictions") + identifiers = manifest["task_ids"][start:stop] + if set(identifiers) - set(rows): + raise ValueError("batch has unfinished tasks; never freeze placeholders") + selected = [rows[identifier] for identifier in identifiers] + destination.mkdir(parents=True) + write_jsonl(destination / "predictions.jsonl", selected) + export_swebench( + selected, destination / "swebench-predictions.jsonl", "agentic-code-search-gpt-6.1-sol" + ) + write_json( + destination / "manifest.json", + { + "kind": "frozen_inference_batch", + "batch": batch, + "start": start, + "stop": stop, + "count": len(selected), + "parent_manifest_sha256": hashlib.sha256( + (source / "manifest.json").read_bytes() + ).hexdigest(), + "predictions_sha256": hashlib.sha256( + (destination / "predictions.jsonl").read_bytes() + ).hexdigest(), + "parent_configuration": manifest, + }, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("destination", type=Path) + parser.add_argument("--start", type=int, required=True) + parser.add_argument("--stop", type=int, required=True) + parser.add_argument("--batch", type=int, required=True) + args = parser.parse_args() + freeze(args.source, args.destination, args.start, args.stop, args.batch) From 5b2efcdd93737fd86e1e1dbd01129652d0a30ad4 Mon Sep 17 00:00:00 2001 From: YeDonS <120633988+YeDonS@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:24:22 -0400 Subject: [PATCH 3/7] eval: freeze predictions 51 through 75 and publish three official patch successes --- reports/model-batches/batch-03/manifest.json | 140 ++++++++++++++++++ .../model-batches/batch-03/predictions.jsonl | 25 ++++ .../batch-03/swebench-predictions.jsonl | 25 ++++ .../automated-review/manifest.json | 16 ++ .../automated-review/reviews.jsonl | 3 + .../automated-review/summary.json | 14 ++ .../raw-log-manifest.json | 10 ++ .../astropy__astropy-12907/report.json | 44 ++++++ .../test-output-excerpt.txt | 67 +++++++++ .../raw-log-manifest.json | 10 ++ .../django__django-16255/report.json | 66 +++++++++ .../test-output-excerpt.txt | 106 +++++++++++++ .../raw-log-manifest.json | 10 ++ .../matplotlib__matplotlib-23964/report.json | 46 ++++++ .../test-output-excerpt.txt | 67 +++++++++ .../official-evaluation/results.json | 69 +++++++++ .../model-checkout-integration-result.json | 105 +++++++++++++ ...model-checkout-integration-validation.json | 14 ++ scripts/freeze_batch.py | 8 +- tests/test_evaluation_runs.py | 22 +++ 20 files changed, 865 insertions(+), 2 deletions(-) create mode 100644 reports/model-batches/batch-03/manifest.json create mode 100644 reports/model-batches/batch-03/predictions.jsonl create mode 100644 reports/model-batches/batch-03/swebench-predictions.jsonl create mode 100644 reports/model-pilot-3/automated-review/manifest.json create mode 100644 reports/model-pilot-3/automated-review/reviews.jsonl create mode 100644 reports/model-pilot-3/automated-review/summary.json create mode 100644 reports/model-pilot-3/official-evaluation/astropy__astropy-12907/raw-log-manifest.json create mode 100644 reports/model-pilot-3/official-evaluation/astropy__astropy-12907/report.json create mode 100644 reports/model-pilot-3/official-evaluation/astropy__astropy-12907/test-output-excerpt.txt create mode 100644 reports/model-pilot-3/official-evaluation/django__django-16255/raw-log-manifest.json create mode 100644 reports/model-pilot-3/official-evaluation/django__django-16255/report.json create mode 100644 reports/model-pilot-3/official-evaluation/django__django-16255/test-output-excerpt.txt create mode 100644 reports/model-pilot-3/official-evaluation/matplotlib__matplotlib-23964/raw-log-manifest.json create mode 100644 reports/model-pilot-3/official-evaluation/matplotlib__matplotlib-23964/report.json create mode 100644 reports/model-pilot-3/official-evaluation/matplotlib__matplotlib-23964/test-output-excerpt.txt create mode 100644 reports/model-pilot-3/official-evaluation/results.json create mode 100644 reports/real-model-checkout/model-checkout-integration-result.json create mode 100644 reports/real-model-checkout/model-checkout-integration-validation.json diff --git a/reports/model-batches/batch-03/manifest.json b/reports/model-batches/batch-03/manifest.json new file mode 100644 index 0000000..a33ead4 --- /dev/null +++ b/reports/model-batches/batch-03/manifest.json @@ -0,0 +1,140 @@ +{ + "kind": "frozen_inference_batch", + "batch": 3, + "start": 50, + "stop": 75, + "count": 25, + "parent_manifest_sha256": "84c775f6f20462427a272c70a47e09a23c92a2ad6c5234949aa3cdd706c0f07b", + "predictions_sha256": "1e22d20ab26e6bd0ec61ae40b6f89c296fd8c4c582f21cfc4d4d85a740bda734", + "parent_configuration": { + "mode": "model", + "count": 100, + "task_ids": [ + "astropy__astropy-12907", + "django__django-16255", + "matplotlib__matplotlib-23964", + "mwaskom__seaborn-2848", + "pallets__flask-4992", + "psf__requests-2317", + "pydata__xarray-4094", + "pylint-dev__pylint-7993", + "pytest-dev__pytest-5692", + "scikit-learn__scikit-learn-13584", + "sphinx-doc__sphinx-8713", + "sympy__sympy-12419", + "astropy__astropy-14365", + "django__django-11742", + "matplotlib__matplotlib-26011", + "mwaskom__seaborn-3407", + "pallets__flask-4045", + "psf__requests-2148", + "pydata__xarray-3364", + "pylint-dev__pylint-5859", + "pytest-dev__pytest-5227", + "scikit-learn__scikit-learn-11040", + "sphinx-doc__sphinx-7975", + "sympy__sympy-13437", + "astropy__astropy-7746", + "django__django-13757", + "matplotlib__matplotlib-25498", + "mwaskom__seaborn-3190", + "pallets__flask-5063", + "psf__requests-3362", + "pydata__xarray-4493", + "pylint-dev__pylint-7080", + "pytest-dev__pytest-5103", + "scikit-learn__scikit-learn-14894", + "sphinx-doc__sphinx-8801", + "sympy__sympy-11870", + "astropy__astropy-14995", + "django__django-10914", + "matplotlib__matplotlib-23563", + "mwaskom__seaborn-3010", + "psf__requests-1963", + "pydata__xarray-4248", + "pylint-dev__pylint-7114", + "pytest-dev__pytest-8906", + "scikit-learn__scikit-learn-15512", + "sphinx-doc__sphinx-8506", + "sympy__sympy-20590", + "astropy__astropy-14182", + "django__django-13447", + "matplotlib__matplotlib-23314", + "psf__requests-2674", + "pydata__xarray-5131", + "pylint-dev__pylint-7228", + "pytest-dev__pytest-5413", + "scikit-learn__scikit-learn-14087", + "sphinx-doc__sphinx-8595", + "sympy__sympy-18087", + "astropy__astropy-6938", + "django__django-12184", + "matplotlib__matplotlib-24265", + "psf__requests-863", + "pylint-dev__pylint-6506", + "pytest-dev__pytest-7373", + "scikit-learn__scikit-learn-14983", + "sphinx-doc__sphinx-8282", + "sympy__sympy-13647", + "django__django-13448", + "matplotlib__matplotlib-18869", + "pytest-dev__pytest-5495", + "scikit-learn__scikit-learn-14092", + "sphinx-doc__sphinx-7686", + "sympy__sympy-20154", + "django__django-11001", + "matplotlib__matplotlib-24149", + "pytest-dev__pytest-11143", + "scikit-learn__scikit-learn-13497", + "sphinx-doc__sphinx-7738", + "sympy__sympy-13480", + "django__django-12983", + "matplotlib__matplotlib-25332", + "pytest-dev__pytest-7168", + "scikit-learn__scikit-learn-25500", + "sphinx-doc__sphinx-8721", + "sympy__sympy-19487", + "django__django-14997", + "matplotlib__matplotlib-24334", + "pytest-dev__pytest-11148", + "scikit-learn__scikit-learn-25747", + "sphinx-doc__sphinx-8435", + "sympy__sympy-13895", + "django__django-12497", + "matplotlib__matplotlib-26020", + "pytest-dev__pytest-7490", + "scikit-learn__scikit-learn-25638", + "sphinx-doc__sphinx-8273", + "sympy__sympy-14308", + "django__django-13401", + "matplotlib__matplotlib-25079", + "pytest-dev__pytest-8365", + "scikit-learn__scikit-learn-13142" + ], + "tasks_sha256": "933b5103f3a746835a41a71422d84487b504291ca577b5a450f9bf280b458a18", + "model": "codex-cli:gpt-6.1-sol", + "seed": 2026, + "created_at": "2026-10-08T18:05:29.872171+00:00", + "test_executor": "disabled", + "max_tool_calls": 18, + "max_search_passes": 2, + "max_agent_steps": 5, + "model_timeout": 120, + "codex_reasoning_effort": "medium", + "codex_transport": "https", + "workers": 6, + "environment": { + "python": "3.12.13", + "platform": "macOS-15.4.1-arm64-arm-64bit", + "source_tree_sha256": "d4e5b6d72a4d66ac37e10f45ae0c900d24a439de2ee2ff1d5c6ca68fd5f1ddf1", + "packages": { + "langchain": "1.4.3", + "langgraph": "1.2.14", + "fastapi": "0.143.0", + "numpy": "2.5.3", + "rank-bm25": "0.2.2" + } + }, + "codex_cli_version": "codex-cli 0.162.0-alpha.2" + } +} diff --git a/reports/model-batches/batch-03/predictions.jsonl b/reports/model-batches/batch-03/predictions.jsonl new file mode 100644 index 0000000..dbb81db --- /dev/null +++ b/reports/model-batches/batch-03/predictions.jsonl @@ -0,0 +1,25 @@ +{"instance_id": "psf__requests-2674", "mode": "model", "status": "diagnosed", "predicted_files": ["requests/adapters.py"], "root_cause": "The adapter's exception translation is incomplete: a directly raised urllib3 ConnectTimeoutError falls into the _HTTPError handler and is re-raised unchanged because only SSL and read-timeout errors are translated there. ConnectTimeoutError is translated only when wrapped in MaxRetryError. This establishes a timeout leakage path, but does not establish the reported proxy scenario or DecodeError leakage.", "model_patch": "--- a/requests/adapters.py\n+++ b/requests/adapters.py\n@@ -427,9 +427,11 @@\n except (_SSLError, _HTTPError) as e:\n if isinstance(e, _SSLError):\n raise SSLError(e, request=request)\n elif isinstance(e, ReadTimeoutError):\n raise ReadTimeout(e, request=request)\n+ elif isinstance(e, ConnectTimeoutError):\n+ raise ConnectTimeout(e, request=request)\n else:\n raise\n \n return self.build_response(request, resp)\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pydata__xarray-5131", "mode": "model", "status": "diagnosed", "predicted_files": ["xarray/core/groupby.py", "xarray/tests/test_groupby.py"], "root_cause": "GroupBy.__repr__ explicitly inserts a space immediately before the newline. Both representation tests encode this trailing space in their expected output.", "model_patch": "--- a/xarray/core/groupby.py\n+++ b/xarray/core/groupby.py\n@@ -438,5 +438,5 @@\n def __repr__(self):\n- return \"{}, grouped over {!r} \\n{!r} groups with labels {}.\".format(\n+ return \"{}, grouped over {!r}\\n{!r} groups with labels {}.\".format(\n self.__class__.__name__,\n self._unique_coord.name,\n self._unique_coord.size,\n--- a/xarray/tests/test_groupby.py\n+++ b/xarray/tests/test_groupby.py\n@@ -388,6 +388,6 @@\n def test_groupby_repr(obj, dim):\n actual = repr(obj.groupby(dim))\n expected = \"%sGroupBy\" % obj.__class__.__name__\n- expected += \", grouped over %r \" % dim\n+ expected += \", grouped over %r\" % dim\n expected += \"\\n%r groups with labels \" % (len(np.unique(obj[dim])))\n if dim == \"x\":\n@@ -405,6 +405,6 @@\n def test_groupby_repr_datetime(obj):\n actual = repr(obj.groupby(\"t.month\"))\n expected = \"%sGroupBy\" % obj.__class__.__name__\n- expected += \", grouped over 'month' \"\n+ expected += \", grouped over 'month'\"\n expected += \"\\n%r groups with labels \" % (len(np.unique(obj.t.dt.month)))\n expected += \"1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12.\"\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pylint-dev__pylint-7228", "mode": "model", "status": "diagnosed", "predicted_files": ["pylint/config/argument.py"], "root_cause": "Naming regex options use the regexp transformer, mapped directly to Python's re.compile. Python re does not support \\p{Han}, so compilation raises re.error. This exception is not converted to argparse.ArgumentTypeError and escapes configuration parsing as a traceback.", "model_patch": "--- a/pylint/config/argument.py\n+++ b/pylint/config/argument.py\n@@ -121,6 +121,16 @@\n return patterns\n \n \n+def _regexp_transformer(value: str) -> Pattern[str]:\n+ \"\"\"Compile a regular expression and report invalid syntax.\"\"\"\n+ try:\n+ return re.compile(value)\n+ except re.error as exc:\n+ raise argparse.ArgumentTypeError(\n+ f\"Invalid regular expression {value!r}: {exc}\"\n+ ) from None\n+\n+\n _TYPE_TRANSFORMERS: dict[str, Callable[[str], _ArgumentTypes]] = {\n \"choice\": str,\n \"csv\": _csv_transformer,\n@@ -130,7 +140,7 @@\n \"non_empty_string\": _non_empty_string_transformer,\n \"path\": _path_transformer,\n \"py_version\": _py_version_transformer,\n- \"regexp\": re.compile,\n+ \"regexp\": _regexp_transformer,\n \"regexp_csv\": _regexp_csv_transfomer,\n \"regexp_paths_csv\": _regexp_paths_csv_transfomer,\n \"string\": pylint_utils._unquote,\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "pytest-dev__pytest-5413", "mode": "model", "status": "diagnosed", "predicted_files": ["src/_pytest/_code/code.py", "testing/code/test_excinfo.py"], "root_cause": "pytest.raises binds an ExceptionInfo wrapper, not the caught exception. ExceptionInfo.__str__ formats a traceback location and exception summary through ReprFileLocation instead of returning str(self.value). This explains why its output differs from a normal exception catch; existing tests explicitly expect location-prefixed output.", "model_patch": "--- a/src/_pytest/_code/code.py\n+++ b/src/_pytest/_code/code.py\n@@ -537,7 +537,5 @@\n def __str__(self):\n if self._excinfo is None:\n return repr(self)\n- entry = self.traceback[-1]\n- loc = ReprFileLocation(entry.path, entry.lineno + 1, self.exconly())\n- return str(loc)\n+ return str(self.value)\n \n--- a/testing/code/test_excinfo.py\n+++ b/testing/code/test_excinfo.py\n@@ -342,7 +342,8 @@\n def test_excinfo_str():\n excinfo = pytest.raises(ValueError, h)\n s = str(excinfo)\n- assert s.startswith(__file__[:-9]) # pyc file and $py.class\n- assert s.endswith(\"ValueError\")\n- assert len(s.split(\":\")) >= 3 # on windows it's 4\n+ assert s == str(excinfo.value)\n+ with pytest.raises(ValueError) as excinfo:\n+ raise ValueError(\"hello\\nworld\")\n+ assert str(excinfo) == \"hello\\nworld\"\n \n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "scikit-learn__scikit-learn-14087", "mode": "model", "status": "diagnosed", "predicted_files": ["sklearn/linear_model/logistic.py"], "root_cause": "The refit=False branch checks unresolved self.multi_class instead of the resolved local multi_class. For binary classification, 'auto' resolves to 'ovr', producing a three-dimensional coefficient path. The incorrect check selects multinomial's four-index expression, causing IndexError.", "model_patch": "--- a/sklearn/linear_model/logistic.py\n+++ b/sklearn/linear_model/logistic.py\n@@ -2172,4 +2172,4 @@\n best_indices = np.argmax(scores, axis=1)\n- if self.multi_class == 'ovr':\n+ if multi_class == 'ovr':\n w = np.mean([coefs_paths[i, best_indices[i], :]\n for i in range(len(folds))], axis=0)\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "sphinx-doc__sphinx-8595", "mode": "model", "status": "diagnosed", "predicted_files": ["sphinx/ext/autodoc/__init__.py"], "root_cause": "ModuleDocumenter.get_object_members() uses a truthiness check that treats an empty __all__ like a missing __all__, selecting all module members. inspect.getall() correctly preserves empty lists and tuples. Checking explicitly for None allows empty exports to mark all members skipped.", "model_patch": "--- a/sphinx/ext/autodoc/__init__.py\n+++ b/sphinx/ext/autodoc/__init__.py\n@@ -1074,7 +1074,7 @@\n def get_object_members(self, want_all: bool) -> Tuple[bool, ObjectMembers]:\n members = self.get_module_members()\n if want_all:\n- if not self.__all__:\n+ if self.__all__ is None:\n # for implicit module members, check __module__ to avoid\n # documenting imported objects\n return True, list(members.values())\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "sympy__sympy-18087", "mode": "model", "status": "insufficient_evidence", "predicted_files": [], "root_cause": null, "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "astropy__astropy-6938", "mode": "model", "status": "insufficient_evidence", "predicted_files": [], "root_cause": null, "model_patch": "", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "django__django-12184", "mode": "model", "status": "diagnosed", "predicted_files": ["django/urls/resolvers.py"], "root_cause": "RegexPattern.match() removes None-valued named groups before deciding whether to pass positional arguments. For /module/, {'format': None} becomes {}, causing match.groups() to supply (None, None), including the nested capture. The handler passes these alongside request, producing the reported three-argument TypeError.", "model_patch": "--- a/django/urls/resolvers.py\n+++ b/django/urls/resolvers.py\n@@ -158,6 +158,7 @@\n # If there are any named groups, use those as kwargs, ignoring\n # non-named groups. Otherwise, pass all non-named arguments as\n # positional arguments.\n- kwargs = {k: v for k, v in match.groupdict().items() if v is not None}\n+ kwargs = match.groupdict()\n args = () if kwargs else match.groups()\n+ kwargs = {k: v for k, v in kwargs.items() if v is not None}\n return path[match.end():], args, kwargs\n", "root_cause_correct": null, "resolved": null, "test_status": "not_run"} +{"instance_id": "matplotlib__matplotlib-24265", "mode": "model", "status": "diagnosed", "predicted_files": ["lib/matplotlib/style/core.py", "lib/matplotlib/tests/test_style.py"], "root_cause": "Deprecated Seaborn names are translated to seaborn-v0_8 names only inside style.use(). Direct style.library access bypasses that translation: the library is a plain dictionary keyed by style-file stems, so 'seaborn-colorblind' raises KeyError.", "model_patch": "--- a/lib/matplotlib/style/core.py\n+++ b/lib/matplotlib/style/core.py\n@@ -246,6 +246,22 @@\n+class _StyleLibrary(dict):\n+ def __missing__(self, key):\n+ if isinstance(key, str) and (\n+ key == \"seaborn\" or key.startswith(\"seaborn-\")):\n+ replacement = \"seaborn-v0_8\" + key[len(\"seaborn\"):]\n+ if replacement in self:\n+ _api.warn_deprecated(\n+ \"3.6\", message=\"The seaborn styles shipped by Matplotlib \"\n+ \"are deprecated since %(since)s, as they no longer \"\n+ \"correspond to the styles shipped by seaborn. However, \"\n+ \"they will remain available as 'seaborn-v0_8-