From 6dd27c080ce5629f6fc00f0e2d0d6a801d60d2c4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 4 Oct 2026 18:32:11 +0000 Subject: [PATCH 1/2] Stream pytest result batches and parallelize attachments Flush JSON results during the run, serialize the fields the API already accepts, restrict urllib3 retries to idempotent methods, and upload attachments concurrently after teardown. Co-authored-by: Ben Spurlock --- src/qastudio_pytest/api_client.py | 182 ++++++++---------------------- src/qastudio_pytest/models.py | 39 ++++++- src/qastudio_pytest/plugin.py | 163 ++++++++++++++++---------- tests/test_models.py | 83 ++++++++++++++ 4 files changed, 265 insertions(+), 202 deletions(-) create mode 100644 tests/test_models.py diff --git a/src/qastudio_pytest/api_client.py b/src/qastudio_pytest/api_client.py index 15397db..47583f4 100644 --- a/src/qastudio_pytest/api_client.py +++ b/src/qastudio_pytest/api_client.py @@ -37,7 +37,7 @@ def _create_session(self) -> requests.Session: total=self.config.max_retries, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504], - allowed_methods=["HEAD", "GET", "POST", "PUT", "DELETE", "OPTIONS", "TRACE"], + allowed_methods=["HEAD", "GET", "OPTIONS"], ) adapter = HTTPAdapter(max_retries=retry_strategy) @@ -218,157 +218,63 @@ def upload_attachment( self._log(f"Uploading attachment: {filename} ({content_type})") - with open(file_path, "rb") as f: - file_data = f.read() - - # Prepare multipart form data - fields = { - "testResultId": test_result_id, + url = f"{self.base_url}/attachments" + last_error: Optional[APIError] = None + headers = { + "Authorization": f"Bearer {self.api_key}", + "User-Agent": "qastudio-pytest/1.0.0", } - if attachment_type: - fields["type"] = attachment_type - - return self._upload_multipart("/attachments", fields, filename, content_type, file_data) - - def _upload_multipart( - self, - path: str, - fields: Dict[str, str], - filename: str, - content_type: str, - file_data: bytes, - ) -> Dict[str, Any]: - """ - Upload multipart/form-data request with retry logic. - - Args: - path: API endpoint path - fields: Form fields - filename: Name of file being uploaded - content_type: MIME type of file - file_data: File content as bytes - - Returns: - Response JSON data - - Raises: - APIError: If request fails - """ - url = f"{self.base_url}{path}" - last_error: Optional[Exception] = None - for attempt in range(self.config.max_retries): try: if attempt > 0: self._log(f"Retry attempt {attempt + 1}/{self.config.max_retries}") import time - time.sleep(min(1 * (2**attempt), 10)) # Exponential backoff - - return self._make_multipart_request(url, fields, filename, content_type, file_data) - - except APIError as e: - last_error = e - self._log(f"Upload failed (attempt {attempt + 1}/{self.config.max_retries}): {e}") - - # Don't retry on 4xx errors (client errors) - if 400 <= e.status_code < 500: + time.sleep(min(1 * (2**attempt), 10)) + + with open(file_path, "rb") as file_handle: + files = {"file": (filename, file_handle, content_type)} + data = {"testResultId": test_result_id} + if attachment_type: + data["type"] = attachment_type + + response = self.session.post( + url, + files=files, + data=data, + headers=headers, + timeout=self.config.timeout, + ) + + if not response.ok: + error_msg = response.text or response.reason + raise APIError(response.status_code, error_msg) + + if response.content: + json_response: Dict[str, Any] = response.json() + return json_response + return {} + + except APIError as exc: + last_error = exc + self._log(f"Upload failed (attempt {attempt + 1}/{self.config.max_retries}): {exc}") + if 400 <= exc.status_code < 500: raise - - except Exception as e: - last_error = e - self._log(f"Upload failed (attempt {attempt + 1}/{self.config.max_retries}): {e}") + except requests.exceptions.Timeout as exc: + last_error = APIError(408, f"Request timeout: {str(exc)}") + self._log(f"Upload failed (attempt {attempt + 1}/{self.config.max_retries}): {exc}") + except requests.exceptions.ConnectionError as exc: + last_error = APIError(503, f"Connection error: {str(exc)}") + self._log(f"Upload failed (attempt {attempt + 1}/{self.config.max_retries}): {exc}") + except requests.exceptions.RequestException as exc: + last_error = APIError(500, f"Request failed: {str(exc)}") + self._log(f"Upload failed (attempt {attempt + 1}/{self.config.max_retries}): {exc}") if last_error: raise last_error raise APIError(500, "Upload failed after all retries") - def _make_multipart_request( - self, - url: str, - fields: Dict[str, str], - filename: str, - content_type: str, - file_data: bytes, - ) -> Dict[str, Any]: - """ - Make a multipart/form-data HTTP request. - - Args: - url: Full URL to request - fields: Form fields (non-file) - filename: Name of file being uploaded - content_type: MIME type of file - file_data: File content as bytes - - Returns: - Response JSON data - - Raises: - APIError: If request fails - """ - import uuid - - # Generate boundary - boundary = f"----FormBoundary{uuid.uuid4().hex}" - - # Build multipart body - body_parts = [] - - # Add form fields - for key, value in fields.items(): - body_parts.append(f"--{boundary}\r\n".encode()) - body_parts.append(f'Content-Disposition: form-data; name="{key}"\r\n\r\n'.encode()) - body_parts.append(f"{value}\r\n".encode()) - - # Add file field - body_parts.append(f"--{boundary}\r\n".encode()) - body_parts.append( - f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n'.encode() - ) - body_parts.append(f"Content-Type: {content_type}\r\n\r\n".encode()) - body_parts.append(file_data) - body_parts.append(b"\r\n") - - # Add closing boundary - body_parts.append(f"--{boundary}--\r\n".encode()) - - body = b"".join(body_parts) - - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": f"multipart/form-data; boundary={boundary}", - "User-Agent": "qastudio-pytest/1.0.0", - } - - try: - response = self.session.request( - method="POST", - url=url, - data=body, - headers=headers, - timeout=self.config.timeout, - ) - - # Raise for 4xx/5xx status codes - if not response.ok: - error_msg = response.text or response.reason - raise APIError(response.status_code, error_msg) - - # Return JSON if present - if response.content: - json_response: Dict[str, Any] = response.json() - return json_response - return {} - - except requests.exceptions.Timeout as e: - raise APIError(408, f"Request timeout: {str(e)}") - except requests.exceptions.ConnectionError as e: - raise APIError(503, f"Connection error: {str(e)}") - except requests.exceptions.RequestException as e: - raise APIError(500, f"Request failed: {str(e)}") - def _log(self, message: str) -> None: """Log message if verbose mode is enabled.""" if self.config.verbose: diff --git a/src/qastudio_pytest/models.py b/src/qastudio_pytest/models.py index 4d638db..20a35e2 100644 --- a/src/qastudio_pytest/models.py +++ b/src/qastudio_pytest/models.py @@ -43,7 +43,7 @@ def to_dict(self) -> Dict[str, Any]: """Convert to dictionary for API submission.""" result: Dict[str, Any] = { "title": self.title, - "status": self.status.value, + "status": "failed" if self.status == TestStatus.ERROR else self.status.value, "duration": int(self.duration * 1000), # Convert to milliseconds } @@ -52,10 +52,35 @@ def to_dict(self) -> Dict[str, Any]: result["fullTitle"] = self.full_title if self.error: - result["errorMessage"] = self.error + result["errorMessage"] = self._truncate(self.error) if self.stack_trace: - result["stackTrace"] = self.stack_trace + result["stackTrace"] = self._truncate(self.stack_trace) + + if self.test_case_id: + result["testCaseId"] = self.test_case_id + + if self.error_snippet: + result["errorSnippet"] = self._truncate(self.error_snippet) + + if self.error_location: + result["errorLocation"] = self.error_location + + if self.steps: + result["steps"] = self.steps[:200] + + if self.console_output: + result["consoleOutput"] = { + key: self._truncate(value, 50000) + for key, value in self.console_output.items() + if value + } + + if self.start_time: + result["startTime"] = self.start_time + + if self.end_time: + result["endTime"] = self.end_time # Add attachments array if present if self.attachments: @@ -63,6 +88,14 @@ def to_dict(self) -> Dict[str, Any]: return result + @staticmethod + def _truncate(value: Optional[str], max_len: int = 50000) -> Optional[str]: + if not value: + return value + if len(value) <= max_len: + return value + return value[:max_len] + "...[truncated]" + @classmethod def from_pytest_report( cls, item: Any, report: Any, config: Optional["ReporterConfig"] = None diff --git a/src/qastudio_pytest/plugin.py b/src/qastudio_pytest/plugin.py index ab4c837..659775e 100644 --- a/src/qastudio_pytest/plugin.py +++ b/src/qastudio_pytest/plugin.py @@ -1,7 +1,8 @@ """pytest plugin for QAStudio.dev integration.""" import time -from typing import Any, List, Optional +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, List, Optional, Tuple import pytest from .api_client import QAStudioAPIClient, APIError @@ -23,6 +24,7 @@ def __init__(self, config: ReporterConfig): self.api_client = QAStudioAPIClient(config) self.test_run_id: Optional[str] = None self.results: List[TestResult] = [] + self.pending_results: List[TestResult] = [] self.start_time: float = 0 self.session_duration: float = 0 @@ -81,6 +83,7 @@ def pytest_runtest_makereport(self, item: Any, call: Any) -> Any: result.metadata["pytest_item"] = item self.results.append(result) + self.pending_results.append(result) # Update counters self.total_tests += 1 @@ -97,6 +100,9 @@ def pytest_runtest_makereport(self, item: Any, call: Any) -> Any: f"Test completed: {item.name} - {result.status.value} ({result.duration:.2f}s)" ) + if self.test_run_id and len(self.pending_results) >= self.config.batch_size: + self._flush_pending_results() + except Exception as e: self._log(f"Error processing test result: {e}") @@ -124,8 +130,8 @@ def pytest_sessionfinish(self, session: Any) -> None: return try: - # Submit test results in batches - self._submit_results() + self._flush_pending_results() + self._collect_and_upload_attachments() # Complete the test run summary = TestRunSummary( @@ -145,49 +151,86 @@ def pytest_sessionfinish(self, session: Any) -> None: finally: self.api_client.close() - def _submit_results(self) -> None: - """Submit test results in batches and upload attachments.""" - if not self.results: - self._log("No results to submit") + def _flush_pending_results(self) -> None: + """Submit buffered JSON results without waiting for attachments.""" + if not self.pending_results: return - # Collect attachments now that all fixtures have torn down - if self.config.upload_attachments: - self._log("Collecting attachments from test items...") - for result in self.results: - if "pytest_item" in result.metadata: - item = result.metadata["pytest_item"] - result.attachment_paths = self._collect_attachments(item) - self._log( - f"Found {len(result.attachment_paths)} attachment(s) for {result.title}" - ) + if not self.test_run_id: + self._log("No test run ID available, skipping result submission") + self.pending_results = [] + return - batches = batch_list(self.results, self.config.batch_size) - self._log(f"Submitting {len(self.results)} results in {len(batches)} batch(es)") + batches = batch_list(self.pending_results, self.config.batch_size) + self._log(f"Submitting {len(self.pending_results)} results in {len(batches)} batch(es)") for i, batch in enumerate(batches, 1): try: self._log(f"Submitting batch {i}/{len(batches)} ({len(batch)} results)") response = self.api_client.submit_test_results( - self.test_run_id, # type: ignore + self.test_run_id, batch, ) - # Store result IDs for attachment uploads if response and "results" in response: - for j, result_data in enumerate(response["results"]): - if j < len(batch): - batch[j].result_id = result_data.get("testResultId") - - # Upload attachments if enabled - if self.config.upload_attachments: + used = set() for result in batch: - if result.result_id and result.attachment_paths: - self._upload_attachments(result) - + for j, result_data in enumerate(response["results"]): + if j in used: + continue + if result_data.get("title") == result.title: + result.result_id = result_data.get("testResultId") + used.add(j) + break + else: + # Fall back to order when titles are missing + idx = len(used) + if idx < len(response["results"]): + result.result_id = response["results"][idx].get("testResultId") + used.add(idx) except APIError as e: self._handle_error(f"Failed to submit batch {i}", e) + self.pending_results = [] + + def _collect_and_upload_attachments(self) -> None: + """Collect attachment paths after teardown and upload concurrently.""" + if not self.config.upload_attachments: + for result in self.results: + result.metadata.pop("pytest_item", None) + return + + self._log("Collecting attachments from test items...") + for result in self.results: + item = result.metadata.pop("pytest_item", None) + if item is not None: + result.attachment_paths = self._collect_attachments(item) + self._log(f"Found {len(result.attachment_paths)} attachment(s) for {result.title}") + + jobs: List[Tuple[TestResult, str]] = [ + (result, file_path) + for result in self.results + if result.result_id and result.attachment_paths + for file_path in result.attachment_paths + ] + if not jobs: + return + + self._log(f"Uploading {len(jobs)} attachment(s) concurrently") + with ThreadPoolExecutor(max_workers=8) as executor: + futures = { + executor.submit(self._upload_one_attachment, result, file_path): (result, file_path) + for result, file_path in jobs + } + for future in as_completed(futures): + result, file_path = futures[future] + try: + future.result() + except APIError as e: + self._handle_error(f"Failed to upload attachment {file_path}", e) + except Exception as e: + self._log(f" Error uploading {file_path}: {e}") + def _collect_attachments(self, item: Any) -> List[str]: """ Collect attachment file paths for a test. @@ -239,44 +282,42 @@ def _collect_attachments(self, item: Any) -> List[str]: return attachments - def _upload_attachments(self, result: TestResult) -> None: - """ - Upload attachments for a test result. + def _upload_one_attachment(self, result: TestResult, file_path: str) -> None: + """Upload a single attachment file for a test result.""" + import os - Args: - result: TestResult with attachment_paths and result_id - """ + if not result.result_id: + return + + ext = os.path.splitext(file_path)[1].lower() + filename = os.path.basename(file_path) + attachment_type = None + + if ext in [".png", ".jpg", ".jpeg", ".gif"]: + attachment_type = "screenshot" + elif ext in [".mp4", ".webm", ".avi", ".mov"]: + attachment_type = "video" + elif ext in [".log", ".txt"]: + attachment_type = "log" + elif ext == ".zip" and "trace" in filename.lower(): + attachment_type = "trace" + + self.api_client.upload_attachment( + test_result_id=result.result_id, + file_path=file_path, + attachment_type=attachment_type, + ) + self._log(f" Uploaded: {filename}") + + def _upload_attachments(self, result: TestResult) -> None: + """Upload attachments for a test result.""" if not result.result_id or not result.attachment_paths: return self._log(f"Uploading {len(result.attachment_paths)} attachment(s) for {result.title}") - for file_path in result.attachment_paths: try: - # Determine attachment type from file extension - import os - - ext = os.path.splitext(file_path)[1].lower() - filename = os.path.basename(file_path) - attachment_type = None - - if ext in [".png", ".jpg", ".jpeg", ".gif"]: - attachment_type = "screenshot" - elif ext in [".mp4", ".webm", ".avi", ".mov"]: - attachment_type = "video" - elif ext in [".log", ".txt"]: - attachment_type = "log" - elif ext == ".zip" and "trace" in filename.lower(): - attachment_type = "trace" - - self.api_client.upload_attachment( - test_result_id=result.result_id, - file_path=file_path, - attachment_type=attachment_type, - ) - - self._log(f" Uploaded: {os.path.basename(file_path)}") - + self._upload_one_attachment(result, file_path) except APIError as e: self._handle_error(f"Failed to upload attachment {file_path}", e) except Exception as e: diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..7f59e62 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,83 @@ +"""Tests for reporter models and streaming batch submission.""" + +from unittest.mock import MagicMock + +from qastudio_pytest.models import ReporterConfig, TestResult, TestStatus +from qastudio_pytest.plugin import QAStudioPlugin + + +def _config(**overrides): + data = dict( + api_url="https://example.test/api", + api_key="key", + project_id="proj", + batch_size=10, + silent=True, + verbose=False, + upload_attachments=False, + ) + data.update(overrides) + return ReporterConfig(**data) + + +def _result(title, status=TestStatus.PASSED): + return TestResult( + test_case_id="QA-1", + title=title, + full_title=f"suite > {title}", + status=status, + duration=0.12, + error="boom" if status == TestStatus.FAILED else None, + stack_trace="trace" if status == TestStatus.FAILED else None, + error_snippet="expect(1).toBe(2)" if status == TestStatus.FAILED else None, + error_location={"file": "test.py", "line": 4}, + steps=[{"title": "step", "status": "passed"}], + console_output={"stdout": "hello"}, + start_time="2024-01-01T00:00:00", + end_time="2024-01-01T00:00:01", + ) + + +def test_to_dict_includes_api_fields(): + payload = _result("login", TestStatus.FAILED).to_dict() + assert payload["status"] == "failed" + assert payload["testCaseId"] == "QA-1" + assert payload["errorMessage"] == "boom" + assert payload["errorSnippet"] == "expect(1).toBe(2)" + assert payload["errorLocation"]["line"] == 4 + assert payload["steps"][0]["title"] == "step" + assert payload["consoleOutput"]["stdout"] == "hello" + assert payload["startTime"] + assert payload["duration"] == 120 + + +def test_to_dict_maps_error_status_to_failed(): + payload = _result("setup", TestStatus.ERROR).to_dict() + assert payload["status"] == "failed" + + +def test_to_dict_truncates_long_stack_traces(): + result = _result("login", TestStatus.FAILED) + result.stack_trace = "x" * 60000 + payload = result.to_dict() + assert payload["stackTrace"].endswith("...[truncated]") + assert len(payload["stackTrace"]) < 60000 + + +def test_plugin_flushes_when_batch_size_is_reached(): + plugin = QAStudioPlugin(_config(batch_size=10)) + plugin.test_run_id = "run-1" + plugin.api_client = MagicMock() + plugin.api_client.submit_test_results.return_value = {"results": []} + + for i in range(25): + result = _result(f"test {i}") + plugin.results.append(result) + plugin.pending_results.append(result) + if len(plugin.pending_results) >= plugin.config.batch_size: + plugin._flush_pending_results() + + plugin._flush_pending_results() + assert plugin.api_client.submit_test_results.call_count == 3 + sizes = [len(call.args[1]) for call in plugin.api_client.submit_test_results.call_args_list] + assert sizes == [10, 10, 5] From c1591cc463f57b0ab195b4f7314632f2bc18b8e3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 4 Oct 2026 18:53:21 +0000 Subject: [PATCH 2/2] Address PR review feedback (#1) - Retry result POSTs at the application layer without discarding failed batches - Assign attachment result IDs by title first, then unused response slots - Collect every attachment upload APIError before reporting Co-authored-by: Ben Spurlock --- src/qastudio_pytest/api_client.py | 36 +++++++- src/qastudio_pytest/plugin.py | 52 +++++++---- tests/test_plugin.py | 148 ++++++++++++++++++++++++++++++ 3 files changed, 216 insertions(+), 20 deletions(-) create mode 100644 tests/test_plugin.py diff --git a/src/qastudio_pytest/api_client.py b/src/qastudio_pytest/api_client.py index 47583f4..e4f2697 100644 --- a/src/qastudio_pytest/api_client.py +++ b/src/qastudio_pytest/api_client.py @@ -1,5 +1,6 @@ """API client for QAStudio.dev integration.""" +import time from typing import Any, Dict, List, Optional import requests from requests.adapters import HTTPAdapter @@ -17,6 +18,9 @@ def __init__(self, status_code: int, message: str): super().__init__(f"API Error {status_code}: {message}") +_TRANSIENT_STATUS_CODES = {408, 429, 500, 502, 503, 504} + + class QAStudioAPIClient: """Client for communicating with QAStudio.dev API.""" @@ -32,7 +36,8 @@ def _create_session(self) -> requests.Session: """Create requests session with retry logic.""" session = requests.Session() - # Configure retry strategy + # Transport retries stay on idempotent verbs. POST /runs is not + # idempotent; result submissions retry in submit_test_results instead. retry_strategy = Retry( total=self.config.max_retries, backoff_factor=1, @@ -102,6 +107,33 @@ def _make_request( except requests.exceptions.RequestException as e: raise APIError(500, f"Request failed: {str(e)}") + def _is_retryable(self, error: APIError) -> bool: + return error.status_code in _TRANSIENT_STATUS_CODES or error.status_code >= 500 + + def _request_with_retry( + self, + method: str, + path: str, + json_data: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + """Retry transient failures for endpoints that can be safely repeated.""" + attempts = max(self.config.max_retries, 1) + last_error: Optional[APIError] = None + for attempt in range(attempts): + try: + if attempt > 0: + self._log(f"Retry attempt {attempt + 1}/{attempts} for {method} {path}") + time.sleep(min(1 * (2**attempt), 10)) + return self._make_request(method, path, json_data) + except APIError as exc: + last_error = exc + if not self._is_retryable(exc) or attempt == attempts - 1: + raise + self._log(f"{method} {path} failed (attempt {attempt + 1}/{attempts}): {exc}") + if last_error: + raise last_error + raise APIError(500, "Request failed after all retries") + def create_test_run( self, name: str, @@ -155,7 +187,7 @@ def submit_test_results( "results": [result.to_dict() for result in results], } - response = self._make_request("POST", "/results", json_data=data) + response = self._request_with_retry("POST", "/results", json_data=data) self._log(f"Successfully submitted {len(results)} results") return response diff --git a/src/qastudio_pytest/plugin.py b/src/qastudio_pytest/plugin.py index 659775e..6dcb0a1 100644 --- a/src/qastudio_pytest/plugin.py +++ b/src/qastudio_pytest/plugin.py @@ -2,7 +2,7 @@ import time from concurrent.futures import ThreadPoolExecutor, as_completed -from typing import Any, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple import pytest from .api_client import QAStudioAPIClient, APIError @@ -15,6 +15,31 @@ ) +def assign_result_ids(batch: List[TestResult], response_results: List[Dict[str, Any]]) -> None: + """Map submitted results to API result IDs for later attachment uploads.""" + if not response_results: + return + + used = set() + for result in batch: + for index, result_data in enumerate(response_results): + if index in used: + continue + if result_data.get("title") == result.title: + result.result_id = result_data.get("testResultId") + used.add(index) + break + + for result in batch: + if result.result_id: + continue + unused = next((index for index in range(len(response_results)) if index not in used), None) + if unused is None: + return + result.result_id = response_results[unused].get("testResultId") + used.add(unused) + + class QAStudioPlugin: """pytest plugin for reporting test results to QAStudio.dev.""" @@ -164,6 +189,7 @@ def _flush_pending_results(self) -> None: batches = batch_list(self.pending_results, self.config.batch_size) self._log(f"Submitting {len(self.pending_results)} results in {len(batches)} batch(es)") + remaining: List[TestResult] = [] for i, batch in enumerate(batches, 1): try: self._log(f"Submitting batch {i}/{len(batches)} ({len(batch)} results)") @@ -173,25 +199,12 @@ def _flush_pending_results(self) -> None: ) if response and "results" in response: - used = set() - for result in batch: - for j, result_data in enumerate(response["results"]): - if j in used: - continue - if result_data.get("title") == result.title: - result.result_id = result_data.get("testResultId") - used.add(j) - break - else: - # Fall back to order when titles are missing - idx = len(used) - if idx < len(response["results"]): - result.result_id = response["results"][idx].get("testResultId") - used.add(idx) + assign_result_ids(batch, response["results"]) except APIError as e: + remaining.extend(batch) self._handle_error(f"Failed to submit batch {i}", e) - self.pending_results = [] + self.pending_results = remaining def _collect_and_upload_attachments(self) -> None: """Collect attachment paths after teardown and upload concurrently.""" @@ -217,6 +230,7 @@ def _collect_and_upload_attachments(self) -> None: return self._log(f"Uploading {len(jobs)} attachment(s) concurrently") + upload_errors: List[str] = [] with ThreadPoolExecutor(max_workers=8) as executor: futures = { executor.submit(self._upload_one_attachment, result, file_path): (result, file_path) @@ -227,9 +241,11 @@ def _collect_and_upload_attachments(self) -> None: try: future.result() except APIError as e: - self._handle_error(f"Failed to upload attachment {file_path}", e) + upload_errors.append(f"Failed to upload attachment {file_path}: {e}") except Exception as e: self._log(f" Error uploading {file_path}: {e}") + if upload_errors: + self._handle_error("Attachment uploads failed", Exception("; ".join(upload_errors))) def _collect_attachments(self, item: Any) -> List[str]: """ diff --git a/tests/test_plugin.py b/tests/test_plugin.py new file mode 100644 index 0000000..7654773 --- /dev/null +++ b/tests/test_plugin.py @@ -0,0 +1,148 @@ +"""Tests for result ID assignment, batch flush, and API retries.""" + +from unittest.mock import Mock, patch + +from qastudio_pytest.api_client import APIError, QAStudioAPIClient +from qastudio_pytest.models import ReporterConfig, TestResult, TestStatus +from qastudio_pytest.plugin import QAStudioPlugin, assign_result_ids + + +def _result(title: str) -> TestResult: + return TestResult( + test_case_id=None, + title=title, + full_title=title, + status=TestStatus.PASSED, + duration=0.1, + ) + + +def _config(**overrides: object) -> ReporterConfig: + values = { + "api_url": "https://example.test/api", + "api_key": "key", + "project_id": "proj", + "silent": True, + "verbose": False, + "max_retries": 3, + "batch_size": 10, + } + values.update(overrides) + return ReporterConfig(**values) # type: ignore[arg-type] + + +def test_assign_result_ids_title_match_does_not_steal_used_slots(): + batch = [_result("A"), _result("B")] + assign_result_ids( + batch, + [{"title": "B", "testResultId": "id-b"}, {"testResultId": "id-a"}], + ) + + assert batch[0].result_id == "id-a" + assert batch[1].result_id == "id-b" + + +def test_assign_result_ids_skips_used_slots_when_counts_differ(): + batch = [_result("A"), _result("B")] + assign_result_ids( + batch, + [{"title": "B", "testResultId": "id-b"}], + ) + + assert batch[0].result_id is None + assert batch[1].result_id == "id-b" + + +def test_assign_result_ids_fills_unused_after_title_match(): + batch = [_result("A"), _result("B"), _result("C")] + assign_result_ids( + batch, + [ + {"title": "B", "testResultId": "id-b"}, + {"title": "Other", "testResultId": "id-other"}, + ], + ) + + assert [result.result_id for result in batch] == ["id-other", "id-b", None] + + +def test_flush_keeps_failed_batches(): + plugin = QAStudioPlugin(_config()) + plugin.test_run_id = "run1" + failed = _result("keep-me") + plugin.pending_results = [failed] + plugin.api_client.submit_test_results = Mock(side_effect=APIError(503, "unavailable")) + + plugin._flush_pending_results() + + assert plugin.pending_results == [failed] + + +def test_flush_clears_successful_batches(): + plugin = QAStudioPlugin(_config()) + plugin.test_run_id = "run1" + result = _result("ok") + plugin.pending_results = [result] + plugin.api_client.submit_test_results = Mock( + return_value={"results": [{"title": "ok", "testResultId": "tr1"}]} + ) + + plugin._flush_pending_results() + + assert plugin.pending_results == [] + assert result.result_id == "tr1" + + +def test_attachment_uploads_report_every_api_error(): + plugin = QAStudioPlugin(_config()) + result = _result("with-files") + result.result_id = "tr1" + result.attachment_paths = ["one.png", "two.png"] + plugin.results = [result] + plugin._handle_error = Mock() + + def fail(_result: TestResult, file_path: str) -> None: + raise APIError(500, file_path) + + plugin._upload_one_attachment = fail # type: ignore[method-assign] + plugin._collect_and_upload_attachments() + + plugin._handle_error.assert_called_once() + message, error = plugin._handle_error.call_args[0] + assert message == "Attachment uploads failed" + combined = str(error) + assert "one.png" in combined + assert "two.png" in combined + + +def test_submit_test_results_retries_transient_post_failures(): + client = QAStudioAPIClient(_config(max_retries=3)) + result = _result("ok") + responses = [ + APIError(503, "unavailable"), + APIError(500, "boom"), + {"results": [{"testResultId": "tr1"}]}, + ] + + with patch.object(client, "_make_request", side_effect=responses) as request: + with patch("qastudio_pytest.api_client.time.sleep"): + payload = client.submit_test_results("run1", [result]) + + assert payload == {"results": [{"testResultId": "tr1"}]} + assert request.call_count == 3 + + +def test_submit_test_results_does_not_retry_client_errors(): + client = QAStudioAPIClient(_config(max_retries=3)) + result = _result("ok") + + with patch.object(client, "_make_request", side_effect=APIError(400, "bad")) as request: + with patch("qastudio_pytest.api_client.time.sleep"): + try: + client.submit_test_results("run1", [result]) + except APIError as exc: + assert exc.status_code == 400 + else: + raise AssertionError("expected APIError") + + assert request.call_count == 1