diff --git a/README.md b/README.md index fe2860b0e33..f3597d715f1 100644 --- a/README.md +++ b/README.md @@ -100,14 +100,15 @@ Submit the job and retrieve results: ds.submit_python_job( user="do@org.com", code_path="analysis.py", + job_name="analysis", ) ds.sync(); do.sync() # Data owner Approves & runs job -do.jobs[0].approve() +do.jobs["ds@org.com"]["analysis"].approve() do.process_approved_jobs(share_outputs_with_submitter=True) do.sync(); ds.sync() -result = open(ds.jobs[-1].output_paths[0]).read() +result = open(ds.jobs["do@org.com"]["analysis"].output_paths[0]).read() ``` ## Packages diff --git a/docs/API.md b/docs/API.md index 32b0ab2841d..6ef905bd580 100644 --- a/docs/API.md +++ b/docs/API.md @@ -59,7 +59,49 @@ Returns a `PeerList`. Get the list of jobs. Auto-syncs before returning. -Returns a `JobsList`. +Returns a `JobsList`. Address a job by **email, then name** — a job name is +unique per datasite and submitter, so the email is what makes the name resolve +to one job, and both parts stay the same as jobs are added: + +```python +# as the data scientist, naming the data owner +client.jobs["do@org.com"]["analysis"].output_paths + +# as the data owner, naming the submitter +client.jobs["ds@org.com"]["analysis"].approve() +``` + +**A single email keeps the jobs it is a party to**, on either side: the datasite they +sit on, or the person who submitted them. Usually that is the other party — a +data scientist names the data owner, a data owner names the submitter — but +naming yourself works and keeps your own, which is what a `PermissionError` on +someone else's job suggests. Job names cannot contain `@`, so the two kinds of +key never collide. + +Two emails select the jobs on one datasite from one submitter. Give the email +of the datasite owner first and the email of the submitter second: + +```python +client.jobs["do@org.com"]["ds@org.com"]["analysis"].approve() +``` + +Use two emails when one email and a name select more than one job: + +- One submitter sent jobs with the same name to two datasites. +- Two people each sent the other a job with the same name. + +A bare name (`client.jobs["analysis"]`) searches every datasite at once. It +still works, but it raises when more than one job answers to it, and the message +names whichever key separates them. + +A job name can no longer hold an `@`. A job submitted before that rule still +resolves by name, with a `DeprecationWarning`; the next version will not resolve +it, so rename such a job. + +Positional indexing (`client.jobs[0]`) also works and matches the Index column +in the table, but positions shift as jobs are added, so prefer an email and a +name. A position is worth using in one case: two jobs that share a datasite, a +submitter and a name, which no email separates. ### `client.datasets` @@ -174,6 +216,7 @@ Submit a Python job to a Data Owner. **DS only.** ds_client.submit_python_job( user="owner@example.com", code_path="/path/to/script.py", + job_name="analysis", ) ``` @@ -195,11 +238,23 @@ Run all approved jobs. **DO only.** - `stream_output`: Stream stdout/stderr in real-time. - `timeout`: Timeout in seconds per job (default: 300). - `force_execution`: Skip version compatibility checks. +- `ignore_peer_version`: Run jobs from peers whose version is incompatible. ```python do_client.process_approved_jobs() ``` +A job whose submitter runs an incompatible version is not run. Each one is +reported by name, with the submitter and the reason: + +``` +⏭️ 1 approved job(s) did not run: + • analysis (submitted by ds@example.com): Skipping peer ds@example.com: incompatible version. + Pass ignore_peer_version=True to run them anyway. +``` + +The job stays at `approved`, so it runs on the next call once the versions match. + --- ## Cleanup diff --git a/packages/syft-bg/pyproject.toml b/packages/syft-bg/pyproject.toml index 21081790ca9..511c4ce5f49 100644 --- a/packages/syft-bg/pyproject.toml +++ b/packages/syft-bg/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ "google-auth-oauthlib>=1.0.0", "pydantic>=2.0.0", "google-cloud-pubsub>=2.18.0", - "syft-job==0.1.40", + "syft-job>=0.1.41", "syft-permissions>=0.1.15", "syft-rds>=0.6.1", ] diff --git a/packages/syft-bg/src/syft_bg/__init__.py b/packages/syft-bg/src/syft_bg/__init__.py index 89e5239ffa2..e293ece76b9 100644 --- a/packages/syft-bg/src/syft_bg/__init__.py +++ b/packages/syft-bg/src/syft_bg/__init__.py @@ -1,5 +1,7 @@ __version__ = "0.2.2" +from syft_job.logging_config import configure_package_logger + from syft_bg.api import ( AuthResult, AutoApproveResult, @@ -61,3 +63,7 @@ def __getattr__(name: str): print("No config file found, run syft_bg.init() first") return raise AttributeError(f"module 'syft_bg' has no attribute {name!r}") + + +# Last, so the imports stay at the top. Nothing here logs at import time. +configure_package_logger(__name__) diff --git a/packages/syft-enclave/pyproject.toml b/packages/syft-enclave/pyproject.toml index 2ef6f2c6d41..f829a412bb7 100644 --- a/packages/syft-enclave/pyproject.toml +++ b/packages/syft-enclave/pyproject.toml @@ -10,6 +10,7 @@ requires-python = ">=3.10" dependencies = [ "syft>=0.10.0", # floor: the `syft` PyPI project also hosts legacy PySyft <=0.9 "syft-rds>=0.6.1", + "syft-job>=0.1.41", # imported directly for logging_config "pydantic-settings>=2.11.0", "requests>=2.32.0", "google-auth[pyjwt]>=2.22.0", @@ -32,6 +33,7 @@ build-backend = "hatchling.build" [tool.uv.sources] "syft" = { workspace = true } "syft-rds" = { workspace = true } +"syft-job" = { workspace = true } [tool.hatch.build.targets.wheel] packages = ["src/syft_enclaves"] diff --git a/packages/syft-enclave/src/syft_enclaves/__init__.py b/packages/syft-enclave/src/syft_enclaves/__init__.py index c7c5e509ef7..35855ad8b98 100644 --- a/packages/syft-enclave/src/syft_enclaves/__init__.py +++ b/packages/syft-enclave/src/syft_enclaves/__init__.py @@ -1,3 +1,4 @@ +from syft_job.logging_config import configure_package_logger from syft_enclaves.client import SyftEnclaveClient from syft_enclaves.login import login_do, login_ds from syft_enclaves.runner import EnclaveRunner @@ -10,3 +11,6 @@ "login_do", "login_ds", ] + +# Last, so the imports stay at the top. Nothing here logs at import time. +configure_package_logger(__name__) diff --git a/packages/syft-enclave/src/syft_enclaves/__main__.py b/packages/syft-enclave/src/syft_enclaves/__main__.py index a9e6587c369..c0efc79631e 100644 --- a/packages/syft-enclave/src/syft_enclaves/__main__.py +++ b/packages/syft-enclave/src/syft_enclaves/__main__.py @@ -8,6 +8,7 @@ import sys from pydantic import ValidationError +from syft_job.logging_config import defer_to_root_logging from syft_enclaves.client import SyftEnclaveClient from syft_enclaves.receipt.writer import ReceiptSettings @@ -23,6 +24,7 @@ def _configure_logging(log_level: str) -> None: format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S", ) + defer_to_root_logging() def _load_settings() -> EnclaveSettings: diff --git a/packages/syft-enclave/src/syft_enclaves/bootstrap.py b/packages/syft-enclave/src/syft_enclaves/bootstrap.py index 2fc4838bdb0..24b0a366fe2 100644 --- a/packages/syft-enclave/src/syft_enclaves/bootstrap.py +++ b/packages/syft-enclave/src/syft_enclaves/bootstrap.py @@ -29,6 +29,7 @@ from pathlib import Path import requests +from syft_job.logging_config import defer_to_root_logging from syft_enclaves._unix_socket import UnixSocketConnection from syft_enclaves.evidence.tinfoil import TINFOIL_ATTESTATION_PATH @@ -205,6 +206,7 @@ def run() -> None: level="INFO", format="%(asctime)s [%(levelname)s] bootstrap: %(message)s", ) + defer_to_root_logging() token_path = Path( os.environ.get("SYFT_ENCLAVE_TOKEN_PATH", str(DEFAULT_TOKEN_PATH)) diff --git a/packages/syft-enclave/src/syft_enclaves/client.py b/packages/syft-enclave/src/syft_enclaves/client.py index 8cbbe109a66..739593760fd 100644 --- a/packages/syft-enclave/src/syft_enclaves/client.py +++ b/packages/syft-enclave/src/syft_enclaves/client.py @@ -322,7 +322,7 @@ def _wrap_enclave_jobs(self, jobs_list: JobsList) -> JobsList: self._as_enclave_job(j) if j.job_headers.get("job_type") == "enclave" else j for j in jobs_list ] - return JobsList(wrapped, jobs_list._root_email) + return JobsList(wrapped, jobs_list._root_email, jobs_list._has_do_role) def wait_until_peered( self, diff --git a/packages/syft-enclave/src/syft_enclaves/enclave_job_info.py b/packages/syft-enclave/src/syft_enclaves/enclave_job_info.py index 4b9044364f3..ef02bfbf3d4 100644 --- a/packages/syft-enclave/src/syft_enclaves/enclave_job_info.py +++ b/packages/syft-enclave/src/syft_enclaves/enclave_job_info.py @@ -117,8 +117,11 @@ def _approval_file(self) -> Path: def _load_own_approval(self) -> PartyApprovalStatus: if not self._approval_file.exists(): raise PermissionError( - f"No approval file found for {self.current_user_email}. " - f"You may not be a designated party for this job." + f"No approval file for {self.current_user_email} on job " + f"'{self.name}'. The enclave writes one per designated party " + f"when it distributes the job, so either it has not distributed " + f"this job yet — run client.sync() and retry — or you are not a " + f"party to it." ) return PartyApprovalStatus.load_json(self._approval_file) diff --git a/packages/syft-enclave/tests/test_enclave_job_info.py b/packages/syft-enclave/tests/test_enclave_job_info.py new file mode 100644 index 00000000000..2af7fd55813 --- /dev/null +++ b/packages/syft-enclave/tests/test_enclave_job_info.py @@ -0,0 +1,82 @@ +"""Unit tests for EnclaveJobInfo, the per-party approval gate. + +The gate lives here rather than in SyftEnclaveClient.approve_job, so these +build a job on a tmp_path SyftBox folder instead of a four-party enclave flow. +""" + +from datetime import datetime, timezone +from pathlib import Path + +import pytest +from syft_enclaves.enclave_job_info import ( + EnclaveJobInfo, + PartyApprovalStatus, + enclave_approval_file_name, +) +from syft_job.client import JobClient +from syft_job.config import SyftJobConfig +from syft_job.job import JobInfo +from syft_job.job_storage import JobRef +from syft_job.models import JobState, JobStatus, JobSubmissionMetadata + +DO_EMAIL = "do@test.org" +DS_EMAIL = "ds@test.org" + + +def _make_enclave_job(tmp_path: Path, job_name: str = "test_job") -> EnclaveJobInfo: + """An enclave job on the DO's datasite, with no approval file written yet.""" + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + client = JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=DO_EMAIL) + ) + ref = JobRef( + datasite_email=DO_EMAIL, + ds_email=DS_EMAIL, + job_name=job_name, + protocol_version="1", + ) + job = JobInfo( + job_metadata=JobSubmissionMetadata( + name=job_name, + type="python", + submitted_by=DS_EMAIL, + datasite_email=DO_EMAIL, + submitted_at=datetime.now(timezone.utc), + ), + state=JobState(status=JobStatus.PENDING), + client=client, + current_user_email=DO_EMAIL, + ref=ref, + ) + return EnclaveJobInfo.from_job_info(job) + + +def test_approve_refuses_when_approval_file_missing(tmp_path: Path): + """No approval file means the enclave has not distributed the job yet. + + The message used to say the caller may not be a designated party, which is + the wrong cause for the common case and offers nothing to do about it. + """ + job = _make_enclave_job(tmp_path) + + with pytest.raises(PermissionError) as exc: + job.approve() + + message = str(exc.value) + assert DO_EMAIL in message + assert "test_job" in message + assert "client.sync()" in message + + +def test_approve_refuses_when_already_approved(tmp_path: Path): + """A second approval must not overwrite the first one's timestamp.""" + job = _make_enclave_job(tmp_path) + approval_file = job.job_review_path / enclave_approval_file_name(DO_EMAIL) + PartyApprovalStatus(party=DO_EMAIL).save_json(approval_file) + + job.approve() + assert PartyApprovalStatus.load_json(approval_file).status == JobStatus.APPROVED + + with pytest.raises(ValueError, match="Already in status: approved"): + job.approve() diff --git a/packages/syft-job/src/syft_job/__init__.py b/packages/syft-job/src/syft_job/__init__.py index 1341d77d48f..4bd49e496e3 100644 --- a/packages/syft-job/src/syft_job/__init__.py +++ b/packages/syft-job/src/syft_job/__init__.py @@ -1,5 +1,6 @@ # __version__ comes from the installed distribution metadata (see version.py). from .version import __version__ +from .logging_config import configure_package_logger from .client import BaseJobClient, JobClient, get_client from .config import SyftJobConfig @@ -32,3 +33,6 @@ # Migration registry "job_registry", ] + +# Last, so the imports stay at the top. Nothing here logs at import time. +configure_package_logger(__name__) diff --git a/packages/syft-job/src/syft_job/client.py b/packages/syft-job/src/syft_job/client.py index d3a303f4526..3ba7c2357db 100644 --- a/packages/syft-job/src/syft_job/client.py +++ b/packages/syft-job/src/syft_job/client.py @@ -741,8 +741,18 @@ def jobs(self) -> JobsList: """ Get all jobs from all peer directories as an indexable list grouped by user. + Order is the same as the jobs table: jobs on your own datasite first, + then the other datasites alphabetically, newest-first within each one. + ``jobs[N]`` is the row labelled ``[N]``. Prefer an email and a name + (``jobs["do@org.com"]["analysis"]``); both parts stay the same as jobs + are added, and the email is what makes the name resolve to one job. + Returns a JobsList object that can be: - - Indexed: jobs[0], jobs[1], etc. + - Indexed by email, then name (preferred): jobs["do@org.com"]["analysis"] + - Indexed by datasite and submitter, for a name they share: + jobs["do@org.com"]["ds@org.com"]["analysis"] + - Indexed by name across every datasite: jobs["analysis"] + - Indexed by position: jobs[0], jobs[1] — matches the table Index column - Iterated: for job in jobs - Displayed: print(jobs) shows separate tables for each user - HTML display: in Jupyter, shows separate tables for each user with jobs @@ -755,25 +765,18 @@ def jobs(self) -> JobsList: current_jobs = self._get_all_jobs() - # Sort jobs by recent submissions first (newest first), then by user/status def job_sort_key(job): - # Parse submitted_at timestamp for sorting (most recent first) + # Root owner first, then peers, newest-first within each owner. + # This list is the authority for both jobs[N] and the table's [N]. try: if job.submitted_at: - from datetime import datetime as dt - - # Parse ISO format timestamp - ts = dt.fromisoformat(job.submitted_at.replace("Z", "+00:00")) - # Use negative timestamp for reverse chronological order (newest first) + ts = datetime.fromisoformat(job.submitted_at.replace("Z", "+00:00")) time_priority = -ts.timestamp() else: - # Jobs without submitted_at go to the end time_priority = float("inf") except Exception: - # Invalid timestamps go to the end time_priority = float("inf") - # Secondary sorting: user priority (root first), then user name, then status user_priority = ( 0 if job.datasite_owner_email == self.current_user_email else 1 ) @@ -789,9 +792,9 @@ def job_sort_key(job): status_priority = status_order.get(job.status, 7) return ( - time_priority, user_priority, job.datasite_owner_email, + time_priority, status_priority, job.name.lower(), ) diff --git a/packages/syft-job/src/syft_job/job.py b/packages/syft-job/src/syft_job/job.py index 2eb5ab38cad..66d2a5e13bf 100644 --- a/packages/syft-job/src/syft_job/job.py +++ b/packages/syft-job/src/syft_job/job.py @@ -2,6 +2,7 @@ import json import shutil +import warnings from datetime import datetime, timezone from pathlib import Path from typing import TYPE_CHECKING, Iterable, List, Optional @@ -265,7 +266,7 @@ def _list_output_files() -> List[Path]: for item in outputs_dir.iterdir() if item.name != PERMISSION_FILE_NAME ] - except Exception: + except OSError: return [] if status == JobStatus.FAILED: @@ -319,7 +320,7 @@ def files(self) -> List[Path]: ): continue all_files.append(f) - except Exception: + except OSError: pass return all_files @@ -375,8 +376,10 @@ def approve( if self.datasite_owner_email != self.current_user_email: raise PermissionError( - f"Only the admin user ({self.datasite_owner_email}) can approve jobs in their folder. " - f"Current job is in {self.datasite_owner_email}'s folder." + f"You are {self.current_user_email}, and job '{self.name}' is on " + f"{self.datasite_owner_email}'s datasite. Only they can approve " + f"it. If you meant one of your own, select your datasite " + f'first: jobs["{self.current_user_email}"][""].' ) # The approved hash and the grant go first, so the runner never sees an @@ -449,7 +452,8 @@ def reject(self, reason: Optional[str] = None) -> None: if self.datasite_owner_email != self.current_user_email: raise PermissionError( - f"Only the admin user ({self.datasite_owner_email}) can reject jobs." + f"You are {self.current_user_email}, and job '{self.name}' is on " + f"{self.datasite_owner_email}'s datasite. Only they can reject it." ) self._state.status = JobStatus.REJECTED @@ -638,25 +642,207 @@ def _repr_html_(self) -> str: return job_info_repr_html(self) +def _with_party(jobs: List[JobInfo], email: str) -> List[JobInfo]: + """The jobs ``email`` is a party to: on its datasite, or submitted by it.""" + return [j for j in jobs if email in (j.datasite_owner_email, j.submitted_by)] + + +def _with_name(jobs: List[JobInfo], name: str) -> List[JobInfo]: + """The jobs called ``name``.""" + return [j for j in jobs if j.name == name] + + +def _with_submission( + jobs: List[JobInfo], datasite: str, submitter: str +) -> List[JobInfo]: + """The jobs on ``datasite`` submitted by ``submitter``.""" + return [ + j + for j in jobs + if j.datasite_owner_email == datasite and j.submitted_by == submitter + ] + + +def _keep(jobs: List[JobInfo], key: str, first_email: str | None) -> List[JobInfo]: + """The jobs one subscript key keeps, read the way ``__getitem__`` reads it. + + ``first_email`` is the email an earlier key in the chain gave, or None. + The '@' tells the two kinds of key apart, so the usage hint can measure a + chain before offering it. A deprecated job name holding an '@' reads here + as an email and keeps nothing, where ``__getitem__`` falls back to the + name; the hint then offers a position, which is the safe direction to err. + """ + if "@" not in key: + return _with_name(jobs, key) + if first_email is None: + return _with_party(jobs, key) + return _with_submission(jobs, first_email, key) + + class JobsList: """A list-like container for JobInfo objects with nice display.""" - def __init__(self, jobs: List[JobInfo], root_email: str, has_do_role: bool = False): + def __init__( + self, + jobs: List[JobInfo], + root_email: str, + has_do_role: bool = False, + first_email: str | None = None, + ): self._jobs = jobs self._root_email = root_email self._has_do_role = has_do_role - - def __getitem__(self, index: int | str) -> JobInfo: + # The email that selected this list, which a second email key reads as + # the datasite. None for a list no email has selected. + self._first_email = first_email + + def __getitem__(self, index: int | str) -> "JobInfo | JobsList": + """A job by position or name, or the jobs of one party by email. + + A first email keeps the jobs it is a party to, on either side. That is + usually the other party — a data scientist names the data owner, a data + owner names the submitter — but naming yourself keeps your own. So + ``jobs["do@x.org"]["analysis"]`` reads as one job. A second email names + the submitter, and the first one is then the datasite: + ``jobs["do@x.org"]["ds@y.org"]["analysis"]`` pins the pair a job name is + unique under, even when the two people sent each other a job of that + name. A bare name searches + every datasite at once and raises when more than one job answers to it. + Job names cannot contain ``@``, so the two kinds of key never collide. + """ if isinstance(index, int): return self._jobs[index] elif isinstance(index, str): - for job in self._jobs: - if job.name == index: - return job - raise ValueError(f"Job with name '{index}' not found") + if "@" in index: + return self._by_email(index) + return self._by_name(index) else: raise TypeError(f"Invalid index type: {type(index)}") + def _by_email(self, email: str) -> "JobInfo | JobsList": + """The jobs this email is a party to, or, after a first email, submitted. + + A first email covers both roles because which one narrows depends on + who is asking. A data scientist names the data owner's datasite; a data + owner names the submitter. A second email has to fix the roles, because + two people with both roles can each hold a job of one name that the + other submitted, and each email is a party to both. + + A job submitted before names could not hold an '@' answers to no party, + and reading it is the one thing this key must not take away, so a key + that names no party falls back to the name. + """ + matches = _keep(self._jobs, email, self._first_email) + if matches: + return JobsList( + matches, + self._root_email, + self._has_do_role, + first_email=self._first_email or email, + ) + + if any(job.name == email for job in self._jobs): + warnings.warn( + f"Job name {email!r} holds an '@', which now marks a datasite or " + "submitter email. Such names are deprecated and the next version " + "will not resolve them. Rename the job.", + DeprecationWarning, + stacklevel=3, + ) + return self._by_name(email) + + datasites = ", ".join(sorted({job.datasite_owner_email for job in self._jobs})) + submitters = ", ".join(sorted({job.submitted_by for job in self._jobs})) + raise ValueError( + f"No jobs involving {email}. These jobs are on: {datasites}. " + f"They were submitted by: {submitters}." + ) + + def _by_name(self, name: str) -> JobInfo: + matches = _with_name(self._jobs, name) + if not matches: + raise ValueError(f"Job with name '{name}' not found") + if len(matches) > 1: + raise ValueError(self._ambiguous_name_message(name, matches)) + return matches[0] + + def _ambiguous_name_message(self, name: str, matches: List[JobInfo]) -> str: + """Why the name did not resolve, and a subscript chain for each job. + + An email key keeps jobs on either side, so one email can be the + datasite of one candidate and the submitter of another. Each chain is + measured with ``_reaches_only`` before it is offered. One submitter + holding a name twice across protocol layouts defeats every chain, so + such a job gets its position in this list. + """ + lines = [ + f" {self._accessor_for(job)} — on {job.datasite_owner_email} " + f"from {job.submitted_by}" + for job in matches + ] + return ( + f"Multiple jobs are named '{name}'. Select one by subscripting this " + "list with:\n" + "\n".join(lines) + ) + + def _accessor_for(self, job: JobInfo) -> str: + """The shortest subscript chain that reaches ``job`` alone, or its position.""" + chains = ( + (job.datasite_owner_email, job.name), + (job.submitted_by, job.name), + (job.datasite_owner_email, job.submitted_by, job.name), + ) + for keys in chains: + if self._reaches_only(keys, job): + return "".join(f'["{key}"]' for key in keys) + return f"[{self._jobs.index(job)}]" + + def hint_accessor(self) -> str | None: + """The subscript chain the jobs table tells a data owner to type. + + Names a pending job on the DO's own datasite and gives the shortest + chain that reaches it and nothing else. The email in a chain is the + other party, so the submitter comes first; the datasite joins it only + when that submitter used the name on another datasite too. One + submitter holding a name twice across protocol layouts defeats every + chain, which is what the position is for. + + Returns None when no such job is there to name, because the hint's + ``approve()`` takes only a pending job on your own datasite: a client + with no data-owner role, a DO who owns none of these jobs, or a DO + whose own jobs have all been reviewed already. The hint also offers + ``accept_by_depositing_result()``, which an approved job would still + take; withholding both is the cost of never printing a command that + raises. + """ + if not self._has_do_role: + return None + owned = [ + j + for j in self._jobs + if j.datasite_owner_email == self._root_email and j.status == "pending" + ] + if not owned: + return None + pick = owned[0] + chains = ( + (pick.submitted_by, pick.name), + (self._root_email, pick.submitted_by, pick.name), + ) + for keys in chains: + if self._reaches_only(keys, pick): + return "".join(f'["{key}"]' for key in keys) + return f"[{self._jobs.index(pick)}]" + + def _reaches_only(self, keys: tuple[str, ...], job: JobInfo) -> bool: + """Whether subscripting by ``keys`` in turn reaches ``job`` and nothing else.""" + reached, first_email = self._jobs, self._first_email + for key in keys: + reached = _keep(reached, key, first_email) + if "@" in key and first_email is None: + first_email = key + return len(reached) == 1 and reached[0] is job + def __len__(self) -> int: return len(self._jobs) @@ -664,10 +850,10 @@ def __iter__(self): return iter(self._jobs) def __str__(self) -> str: - return jobs_list_str(self._jobs, self._root_email, self._has_do_role) + return jobs_list_str(self._jobs, self.hint_accessor()) def __repr__(self) -> str: return f"JobsList({len(self._jobs)} jobs)" def _repr_html_(self) -> str: - return jobs_list_repr_html(self._jobs, self._root_email, self._has_do_role) + return jobs_list_repr_html(self._jobs, self.hint_accessor()) diff --git a/packages/syft-job/src/syft_job/job_repr.py b/packages/syft-job/src/syft_job/job_repr.py index 654f32d8a99..5e2242d3579 100644 --- a/packages/syft-job/src/syft_job/job_repr.py +++ b/packages/syft-job/src/syft_job/job_repr.py @@ -741,18 +741,30 @@ def job_info_repr_html(job: "JobInfo") -> str: """ -def jobs_list_str( - jobs: List["JobInfo"], root_email: str, has_do_role: bool = False -) -> str: +def _owner_groups(jobs: List["JobInfo"]) -> List[tuple[str, List["JobInfo"]]]: + """Split jobs into consecutive owner sections without reordering. + + JobClient.jobs is the authority for order. Re-sorting here is what + made the table's [N] disagree with jobs[N]. A list that is not already + grouped by owner gives an owner more than one section; the row indexes + stay correct. + """ + groups: List[tuple[str, List["JobInfo"]]] = [] + for job in jobs: + owner = job.datasite_owner_email + if groups and groups[-1][0] == owner: + groups[-1][1].append(job) + else: + groups.append((owner, [job])) + return groups + + +def jobs_list_str(jobs: List["JobInfo"], hint_accessor: str | None = None) -> str: """Format jobs list as separate tables grouped by user.""" if not jobs: return "📭 No jobs found.\n" - jobs_by_user: dict[str, list["JobInfo"]] = {} - for job in jobs: - if job.datasite_owner_email not in jobs_by_user: - jobs_by_user[job.datasite_owner_email] = [] - jobs_by_user[job.datasite_owner_email].append(job) + owner_groups = _owner_groups(jobs) status_emojis = { "received": "📨", @@ -768,25 +780,12 @@ def jobs_list_str( lines.append("📊 Jobs Overview") lines.append("=" * 50) - total_jobs = 0 + total_jobs = len(jobs) global_status_counts: dict[str, int] = {} - def user_sort_key(item): - user_email, _user_jobs = item - if user_email == root_email: - return (0, user_email) - return (1, user_email) - - sorted_users = sorted(jobs_by_user.items(), key=user_sort_key) - job_index = 0 - for user_email, user_jobs in sorted_users: - if not user_jobs: - continue - - total_jobs += len(user_jobs) - + for user_email, user_jobs in owner_groups: lines.append("") lines.append(f"👤 {user_email}") lines.append("-" * 60) @@ -805,9 +804,7 @@ def user_sort_key(item): lines.append(header) lines.append("-" * len(header)) - sorted_jobs = user_jobs - - for job in sorted_jobs: + for job in user_jobs: emoji = status_emojis.get(job.status, "❓") status_display = f"{emoji} {job.status}" approval_display = job.approval_method or "—" @@ -834,7 +831,8 @@ def user_sort_key(item): lines.append("") lines.append("=" * 50) - lines.append(f"📈 Total: {total_jobs} jobs across {len(jobs_by_user)} users") + owner_count = len({job.datasite_owner_email for job in jobs}) + lines.append(f"📈 Total: {total_jobs} jobs across {owner_count} users") global_summary_parts = [] for status, count in global_status_counts.items(): @@ -844,18 +842,17 @@ def user_sort_key(item): if global_summary_parts: lines.append("📋 Global: " + " | ".join(global_summary_parts)) - if has_do_role: + if hint_accessor is not None: lines.append("") lines.append( - "💡 Use job_client.jobs[0].approve() to approve jobs or job_client.jobs[0].accept_by_depositing_result('file_or_folder') to complete jobs" + f"💡 Use job_client.jobs{hint_accessor}.approve() to approve jobs or " + f"job_client.jobs{hint_accessor}.accept_by_depositing_result('file_or_folder') to complete jobs" ) return "\n".join(lines) -def jobs_list_repr_html( - jobs: List["JobInfo"], root_email: str, has_do_role: bool = False -) -> str: +def jobs_list_repr_html(jobs: List["JobInfo"], hint_accessor: str | None = None) -> str: """HTML representation for Jupyter notebooks with enhanced visual appeal.""" if not jobs: return """ @@ -929,11 +926,7 @@ def jobs_list_repr_html( """ - jobs_by_user: dict[str, list["JobInfo"]] = {} - for job in jobs: - if job.datasite_owner_email not in jobs_by_user: - jobs_by_user[job.datasite_owner_email] = [] - jobs_by_user[job.datasite_owner_email].append(job) + owner_groups = _owner_groups(jobs) status_styles = { "received": { @@ -974,6 +967,7 @@ def jobs_list_repr_html( } total_jobs = len(jobs) + owner_count = len({job.datasite_owner_email for job in jobs}) global_status_counts: dict[str, int] = {} for job in jobs: global_status_counts[job.status] = global_status_counts.get(job.status, 0) + 1 @@ -1227,25 +1221,12 @@ def jobs_list_repr_html(

📊 Jobs Overview

-

Total: {total_jobs} jobs across {len(jobs_by_user)} users

+

Total: {total_jobs} jobs across {owner_count} users

""" - def user_sort_key(item): - user_email, _user_jobs = item - if user_email == root_email: - return (0, user_email) - return (1, user_email) - - sorted_users = sorted(jobs_by_user.items(), key=user_sort_key) - job_index = 0 - for user_email, user_jobs in sorted_users: - if not user_jobs: - continue - - sorted_user_jobs = user_jobs - + for user_email, user_jobs in owner_groups: user_status_counts: dict[str, int] = {} for job in user_jobs: user_status_counts[job.status] = user_status_counts.get(job.status, 0) + 1 @@ -1274,7 +1255,7 @@ def user_sort_key(item): """ - for i, job in enumerate(sorted_user_jobs): + for i, job in enumerate(user_jobs): style_info = status_styles.get(job.status, {"emoji": "❓"}) row_class = "syftjob-row-even" if i % 2 == 0 else "syftjob-row-odd" @@ -1324,10 +1305,10 @@ def user_sort_key(item): html += """
""" - if has_do_role: - html += """ + if hint_accessor is not None: + html += f"""
- 💡 Use jobs[0].approve() to approve jobs or jobs[0].accept_by_depositing_result('file_or_folder') to complete jobs + 💡 Use jobs{hint_accessor}.approve() to approve jobs or jobs{hint_accessor}.accept_by_depositing_result('file_or_folder') to complete jobs
""" html += """ diff --git a/packages/syft-job/src/syft_job/job_runner.py b/packages/syft-job/src/syft_job/job_runner.py index baa50d28f64..fe2f8f41724 100644 --- a/packages/syft-job/src/syft_job/job_runner.py +++ b/packages/syft-job/src/syft_job/job_runner.py @@ -4,6 +4,7 @@ import subprocess import tempfile import time +import warnings from datetime import datetime, timezone from pathlib import Path from typing import List, Set @@ -653,26 +654,69 @@ def _get_job_info(self, ref: JobRef) -> JobInfo: ref=ref, ) + @staticmethod + def _resolve_skip_jobs( + approved_jobs: List[JobRef], + skip_jobs: list[tuple[str, str]] | None, + skip_job_names: list[str] | None, + ) -> Set[tuple[str, str]]: + """The (job_name, ds_email) pairs to skip, accepting the old name-only form. + + A bare name expands to every submitter who used it, which is what + ``skip_job_names`` did. Bare names in ``skip_jobs`` are read the same + way, so a caller who passed the old third positional argument keeps the + behaviour it had. + """ + pairs: Set[tuple[str, str]] = set() + names: List[str] = list(skip_job_names or []) + for entry in skip_jobs or []: + if isinstance(entry, str): + names.append(entry) + else: + pairs.add((entry[0], entry[1])) + + if names: + warnings.warn( + "Skipping jobs by name alone is deprecated: a name is unique " + "per datasite and submitter, so it also drops other " + "submitters' jobs of the same name. Pass " + "skip_jobs=[(job_name, ds_email), ...] instead.", + DeprecationWarning, + stacklevel=3, + ) + wanted = set(names) + pairs.update( + (ref.job_name, ref.ds_email) + for ref in approved_jobs + if ref.job_name in wanted + ) + return pairs + def process_approved_jobs( self, stream_output: bool = True, timeout: int | None = None, - skip_job_names: list[str] | None = None, + skip_jobs: list[tuple[str, str]] | None = None, share_outputs_with_submitter: bool = False, share_logs_with_submitter: bool = False, + skip_job_names: list[str] | None = None, ) -> None: """Process all jobs in approved status. Args: stream_output: If True (default), stream output in real-time. timeout: Timeout in seconds per job. Defaults to the configured value or 600. - skip_job_names: Optional list of job names to skip. + skip_jobs: Optional (job_name, ds_email) pairs to skip. A name alone + does not identify a job — it is unique per datasite and + submitter — so skipping by name drops every other job that + shares it. share_outputs_with_submitter: If True, grant read access on outputs to submitter. share_logs_with_submitter: If True, release the logs and the exit code to the submitter, whatever the job requested. False (default) releases only the items that the submitter requested and the approval granted. The rest stays in staging, where only the datasite owner reads it. + skip_job_names: Deprecated. The name-only form of ``skip_jobs``. """ self._replace_ds_folder_reads() approved_jobs = self._get_jobs_in_approved() @@ -680,10 +724,11 @@ def process_approved_jobs( if not approved_jobs: return - # Filter out jobs to skip - if skip_job_names: - skip_set = set(skip_job_names) - approved_jobs = [j for j in approved_jobs if j.job_name not in skip_set] + skip_set = self._resolve_skip_jobs(approved_jobs, skip_jobs, skip_job_names) + if skip_set: + approved_jobs = [ + j for j in approved_jobs if (j.job_name, j.ds_email) not in skip_set + ] if not approved_jobs: return diff --git a/packages/syft-job/src/syft_job/job_storage.py b/packages/syft-job/src/syft_job/job_storage.py index eccba625421..df6d21068c6 100644 --- a/packages/syft-job/src/syft_job/job_storage.py +++ b/packages/syft-job/src/syft_job/job_storage.py @@ -164,6 +164,10 @@ def validate_job_name(job_name: str) -> None: raise ValueError( f"Job name {job_name!r} is reserved for protocol version directories" ) + if "@" in job_name: + # An '@' marks a datasite email in JobsList.__getitem__, which is + # how jobs["do@x.org"]["analysis"] tells the two keys apart. + raise ValueError(f"Job name {job_name!r} cannot contain '@'") # -- scanning (union over all protocol layouts) ------------------------------- def iter_submission_refs(self, datasite_email: str) -> Iterator[JobRef]: diff --git a/packages/syft-job/src/syft_job/logging_config.py b/packages/syft-job/src/syft_job/logging_config.py new file mode 100644 index 00000000000..c047a752cd5 --- /dev/null +++ b/packages/syft-job/src/syft_job/logging_config.py @@ -0,0 +1,57 @@ +"""Shared logger setup for syft-job and the packages built on it. + +``syft`` configures the ``syft`` logger only. ``syft_job``, ``syft_rds``, +``syft_enclaves`` and ``syft_bg`` are sibling top-level namespaces, so they +inherit the root logger, which has no handler. INFO records are then dropped, +and WARNING and above fall back to ``logging.lastResort`` — bare text on +stderr, with no way to raise or lower the level. Each package calls +``configure_package_logger`` once, at the end of its ``__init__``. + +The helper lives here, not in ``syft``, because syft-job does not depend on +syft, while the other three packages depend on syft-job. +""" + +import logging + + +PACKAGE_LOGGERS = ("syft_job", "syft_rds", "syft_enclaves", "syft_bg") + + +class _FallbackHandler(logging.StreamHandler): + """Prints a record only while the root logger has no handler. + + When a program configures the root logger, records propagate to it and + print there, so this handler stays silent and nothing prints twice. + """ + + def emit(self, record: logging.LogRecord) -> None: + if not logging.getLogger().handlers: + super().emit(record) + + +def configure_package_logger(name: str, level: int = logging.INFO) -> logging.Logger: + """Give the ``name`` logger a fallback handler and a level, if it has none. + + Records still propagate to the root logger, so pytest's caplog and any + root handler see them. A program that configures the root logger calls + ``defer_to_root_logging`` to make the root level apply here too. + """ + logger = logging.getLogger(name) + if not logger.handlers: + handler = _FallbackHandler() + handler.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) + logger.addHandler(handler) + logger.setLevel(level) + return logger + + +def defer_to_root_logging(names: tuple[str, ...] = PACKAGE_LOGGERS) -> None: + """Remove the level that ``configure_package_logger`` set on each logger. + + Call it after ``logging.basicConfig``. The root level then applies to + these loggers, as it does to every logger with no level of its own. + """ + for name in names: + logger = logging.getLogger(name) + if any(isinstance(h, _FallbackHandler) for h in logger.handlers): + logger.setLevel(logging.NOTSET) diff --git a/packages/syft-job/tests/test_job_flow.py b/packages/syft-job/tests/test_job_flow.py index 87c2b873e90..b24b6c098b7 100644 --- a/packages/syft-job/tests/test_job_flow.py +++ b/packages/syft-job/tests/test_job_flow.py @@ -4,17 +4,19 @@ import time from pathlib import Path +import pytest from syft_job.client import JobClient from syft_job.config import SyftJobConfig from syft_job.job_ref import JobRef from syft_job.job_runner import SyftJobRunner from syft_job.migrations.registry import JOB_PROTOCOL_VERSION -from syft_perms import SyftPermContext from syft_job.models import JobState - +from syft_perms import SyftPermContext DO_EMAIL = "do@test.org" DS_EMAIL = "ds@test.org" +PEER_EMAIL = "peer@test.org" +DS2_EMAIL = "ds2@test.org" LOG_FILES = ("stdout.txt", "stderr.txt", "returncode.txt") MAIN_PY = """\ @@ -268,6 +270,175 @@ def test_timeout_does_not_hang_runner(tmp_path: Path): assert do_client.jobs[0].status == "failed" +def test_skipping_one_job_spares_its_same_named_sibling(tmp_path: Path): + """A skip must name the submitter as well as the job. + + Two data scientists can submit the same job name to one data owner. Given + only the name, the runner dropped both — so a job its peer was compatible + with never ran, and nothing said so. + """ + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + do_config = SyftJobConfig(syftbox_folder=syftbox, current_user_email=DO_EMAIL) + do_client = JobClient(config=do_config) + do_runner = SyftJobRunner(config=do_config) + + for ds_email in (DS_EMAIL, DS2_EMAIL): + JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=ds_email) + ).submit_python_job( + user=DO_EMAIL, code_path=str(code_file), job_name="shared.job" + ) + + for job in do_client.jobs: + job.approve() + + do_runner.process_approved_jobs( + stream_output=False, timeout=60, skip_jobs=[("shared.job", DS_EMAIL)] + ) + + final = {job.submitted_by: job.status for job in do_client.jobs} + assert final[DS_EMAIL] == "approved", "the skipped job must not run" + assert final[DS2_EMAIL] == "done", "its same-named sibling must still run" + + +def test_runner_ignores_approved_jobs_on_another_datasite(tmp_path: Path): + """`jobs` spans every datasite in the folder; the runner runs only its own. + + A peer's approved job is visible but was never a candidate, which is why the + skip report in syft-rds filters on the datasite owner before naming a job as + one that did not run. + """ + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + do_config = SyftJobConfig(syftbox_folder=syftbox, current_user_email=DO_EMAIL) + peer_config = SyftJobConfig(syftbox_folder=syftbox, current_user_email=PEER_EMAIL) + do_client = JobClient(config=do_config) + peer_client = JobClient(config=peer_config) + do_runner = SyftJobRunner(config=do_config) + + ds_client = JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=DS_EMAIL) + ) + ds_client.submit_python_job( + user=DO_EMAIL, code_path=str(code_file), job_name="mine.job" + ) + ds_client.submit_python_job( + user=PEER_EMAIL, code_path=str(code_file), job_name="theirs.job" + ) + + do_client.jobs["mine.job"].approve() + peer_client.jobs["theirs.job"].approve() + + # The DO sees both, including the peer's approved job. + visible = {job.name: job.status for job in do_client.jobs} + assert visible == {"mine.job": "approved", "theirs.job": "approved"} + + do_runner.process_approved_jobs(stream_output=False, timeout=60) + + after = {job.name: job.status for job in do_client.jobs} + assert after["mine.job"] == "done" + assert after["theirs.job"] == "approved", "a peer's job is not this runner's to run" + + +def test_deprecated_skip_job_names_still_skips_by_name(tmp_path: Path): + """The old name-only argument keeps its old behaviour, and says it is old. + + It drops every submitter's job of that name — which is the bug — so it warns + rather than silently changing what an existing caller gets. + """ + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + do_config = SyftJobConfig(syftbox_folder=syftbox, current_user_email=DO_EMAIL) + do_client = JobClient(config=do_config) + do_runner = SyftJobRunner(config=do_config) + + for ds_email in (DS_EMAIL, DS2_EMAIL): + JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=ds_email) + ).submit_python_job( + user=DO_EMAIL, code_path=str(code_file), job_name="shared.job" + ) + for job in do_client.jobs: + job.approve() + + with pytest.warns(DeprecationWarning, match="Skipping jobs by name alone"): + do_runner.process_approved_jobs( + stream_output=False, timeout=60, skip_job_names=["shared.job"] + ) + + assert [job.status for job in do_client.jobs] == ["approved", "approved"] + + +def test_approval_error_names_both_parties(tmp_path: Path): + """The old message called the peer "the admin user" and never named you.""" + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + do_client = JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=DO_EMAIL) + ) + JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=DS_EMAIL) + ).submit_python_job( + user=PEER_EMAIL, code_path=str(code_file), job_name="peer-owned.job" + ) + # The owner scans it into pending, so approve() reaches the ownership check. + JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=PEER_EMAIL) + ).scan_inbox() + + job = do_client.jobs["peer-owned.job"] + assert job.status == "pending" + with pytest.raises(PermissionError) as exc: + job.approve() + + message = str(exc.value) + assert DO_EMAIL in message, "must say who you are" + assert PEER_EMAIL in message, "must say whose datasite it is" + assert "peer-owned.job" in message + assert "admin user" not in message + # The remedy must not send them back to the same job: a bare name searches + # every datasite, and this job is the only one that answers to this one. + assert f'jobs["{DO_EMAIL}"]' in message, "must point at your own datasite" + + +def test_job_files_do_not_hide_non_filesystem_error(tmp_path: Path, monkeypatch): + """Only the filesystem may truncate the file list; other errors propagate.""" + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + do_client = JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=DO_EMAIL) + ) + JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=DS_EMAIL) + ).submit_python_job(user=DO_EMAIL, code_path=str(code_file), job_name="files.job") + + job = do_client.jobs["files.job"] + assert job.files, "sanity: the job has files" + + def boom(*args, **kwargs): + raise RuntimeError("not a filesystem problem") + + monkeypatch.setattr(Path, "rglob", boom) + with pytest.raises(RuntimeError, match="not a filesystem problem"): + job.files + + PARTIAL_OUTPUT_PY = """\ import time diff --git a/packages/syft-job/tests/test_jobs_addressing.py b/packages/syft-job/tests/test_jobs_addressing.py new file mode 100644 index 00000000000..b9ab569f481 --- /dev/null +++ b/packages/syft-job/tests/test_jobs_addressing.py @@ -0,0 +1,431 @@ +"""How a job is addressed: the subscript chain, and the hint that teaches it. + +These drive JobClient over a tmp_path SyftBox folder, like the lifecycle tests +next door, but assert on what jobs[...] returns and what the table's hint says +rather than on a job running. +""" + +import re +from pathlib import Path + +import pytest +from syft_job.client import JobClient +from syft_job.config import SyftJobConfig + +DO_EMAIL = "do@test.org" +DS_EMAIL = "ds@test.org" +PEER_EMAIL = "peer@test.org" +DS2_EMAIL = "ds2@test.org" + +MAIN_PY = """\ +import os + +print("hello from job") +os.makedirs("outputs", exist_ok=True) +with open("outputs/result.txt", "w") as f: + f.write("done") +""" + + +def test_hint_names_submitter_and_job(tmp_path: Path): + """The DO hint must point at jobs["submitter"]["name"], not jobs[0]. + + Positional indexing is not safe to recommend: positions shift as jobs are + added. A bare name is not safe either — it searches every datasite, so a + peer's job of the same name can answer instead. The email in a chain is the + other party, which for a data owner is the submitter. + """ + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + # has_do_role gates the hint — it is only shown to a data owner. + do_config = SyftJobConfig( + syftbox_folder=syftbox, current_user_email=DO_EMAIL, has_do_role=True + ) + ds_config = SyftJobConfig(syftbox_folder=syftbox, current_user_email=DS_EMAIL) + ds_client = JobClient(config=ds_config) + do_client = JobClient(config=do_config) + + ds_client.submit_python_job( + user=DO_EMAIL, code_path=str(code_file), job_name="analysis.job" + ) + + jobs = do_client.jobs + text = str(jobs) + html = jobs._repr_html_() + + for rendering in (text, html): + assert f'jobs["{DS_EMAIL}"]["analysis.job"].approve()' in rendering + assert "jobs[0]" not in rendering + assert DO_EMAIL not in rendering.split("💡")[1], "the DO's own email is noise" + + # The hint names a job the DO can actually approve, as written. + assert do_client.jobs[DS_EMAIL]["analysis.job"].status == "pending" + + +def _index_name_pairs_from_text(text: str) -> list[tuple[int, str]]: + return [(int(i), name) for i, name in re.findall(r"\[(\d+)\s*\]\s+(\S+)", text)] + + +def _index_name_pairs_from_html(html: str) -> list[tuple[int, str]]: + return [ + (int(i), name.strip()) + for i, name in re.findall( + r'class="syftjob-index">\[(\d+)\].*?' + r'class="syftjob-td syftjob-job-name">\s*([^<]+)', + html, + flags=re.DOTALL, + ) + ] + + +def test_jobs_table_index_matches_getitem(tmp_path: Path): + """The [N] printed in the table must be the N that jobs[N] returns. + + The table groups by datasite owner (root first, then peers). __getitem__ + used to subscript a newest-first list, so with two owners the row labelled + [0] was not jobs[0]. + """ + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + do_config = SyftJobConfig( + syftbox_folder=syftbox, current_user_email=DO_EMAIL, has_do_role=True + ) + ds_config = SyftJobConfig(syftbox_folder=syftbox, current_user_email=DS_EMAIL) + ds_client = JobClient(config=ds_config) + do_client = JobClient(config=do_config) + + # Older job on the root datasite, then a newer one on a peer datasite. + ds_client.submit_python_job( + user=DO_EMAIL, code_path=str(code_file), job_name="older-root-job" + ) + ds_client.submit_python_job( + user=PEER_EMAIL, code_path=str(code_file), job_name="newest-peer-job" + ) + + jobs = do_client.jobs + assert [job.name for job in jobs] == ["older-root-job", "newest-peer-job"] + + text_pairs = _index_name_pairs_from_text(str(jobs)) + html_pairs = _index_name_pairs_from_html(jobs._repr_html_()) + assert text_pairs == [(0, "older-root-job"), (1, "newest-peer-job")] + assert html_pairs == [(0, "older-root-job"), (1, "newest-peer-job")] + + for index, name in text_pairs + html_pairs: + assert jobs[index].name == name + + +def test_hint_names_submitter_for_shared_name(tmp_path: Path): + """A name two submitters share needs the submitter to reach one job. + + Job names are unique per datasite and submitter, so two data scientists can + submit "analysis" to the same data owner. The hint adds the submitter rather + than giving up on the name. + """ + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + do_config = SyftJobConfig( + syftbox_folder=syftbox, current_user_email=DO_EMAIL, has_do_role=True + ) + do_client = JobClient(config=do_config) + ds1_client = JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=DS_EMAIL) + ) + ds2_client = JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=DS2_EMAIL) + ) + + # "solo" first, so the ambiguous name is the newest and would otherwise win. + ds1_client.submit_python_job( + user=DO_EMAIL, code_path=str(code_file), job_name="solo" + ) + ds1_client.submit_python_job( + user=DO_EMAIL, code_path=str(code_file), job_name="analysis" + ) + ds2_client.submit_python_job( + user=DO_EMAIL, code_path=str(code_file), job_name="analysis" + ) + + jobs = do_client.jobs + for rendering in (str(jobs), jobs._repr_html_()): + assert f'jobs["{DS2_EMAIL}"]["analysis"].approve()' in rendering + + # The chain the hint gives reaches exactly one job, and the datasite alone + # does not, which is why it names the submitter. + job = jobs[DS2_EMAIL]["analysis"] + assert (job.submitted_by, job.status) == (DS2_EMAIL, "pending") + with pytest.raises(ValueError, match="Multiple jobs are named 'analysis'"): + jobs[DO_EMAIL]["analysis"] + + +def test_hint_names_every_submitter_of_shared_name(tmp_path: Path): + """Every job in the table shares its name, so every chain needs a submitter. + + The datasite alone reaches neither, and the hint still has to name something + the data owner can run. + """ + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + do_config = SyftJobConfig( + syftbox_folder=syftbox, current_user_email=DO_EMAIL, has_do_role=True + ) + do_client = JobClient(config=do_config) + for ds_email in (DS_EMAIL, DS2_EMAIL): + JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=ds_email) + ).submit_python_job( + user=DO_EMAIL, code_path=str(code_file), job_name="analysis" + ) + + jobs = do_client.jobs + for rendering in (str(jobs), jobs._repr_html_()): + assert f'jobs["{DS2_EMAIL}"]["analysis"].approve()' in rendering + assert "jobs[0]" not in rendering + + for ds_email in (DS_EMAIL, DS2_EMAIL): + assert jobs[ds_email]["analysis"].submitted_by == ds_email + + +def test_no_hint_when_do_owns_no_jobs(tmp_path: Path): + """Every subscript would name a job on another datasite, which approve() refuses. + + A hint is worse than no hint when the only command it can give raises + PermissionError. + """ + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + do_config = SyftJobConfig( + syftbox_folder=syftbox, current_user_email=DO_EMAIL, has_do_role=True + ) + do_client = JobClient(config=do_config) + JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=DS_EMAIL) + ).submit_python_job( + user=PEER_EMAIL, code_path=str(code_file), job_name="peer-owned.job" + ) + + jobs = do_client.jobs + assert [job.datasite_owner_email for job in jobs] == [PEER_EMAIL] + for rendering in (str(jobs), jobs._repr_html_()): + assert "peer-owned.job" in rendering + assert "💡" not in rendering + + +def test_email_key_resolves_name_shared_by_two_datasites(tmp_path: Path): + """One submitter can send the same job name to two data owners. + + The bare name has no way to choose between them and raises. Selecting the + datasite first leaves one job, which is the whole point of the two-step + subscript. + """ + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + ds_client = JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=DS_EMAIL) + ) + for owner in (DO_EMAIL, PEER_EMAIL): + ds_client.submit_python_job( + user=owner, code_path=str(code_file), job_name="analysis.job" + ) + JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=owner) + ).scan_inbox() + + jobs = ds_client.jobs + with pytest.raises(ValueError, match="Multiple jobs are named 'analysis.job'"): + jobs["analysis.job"] + + for owner in (DO_EMAIL, PEER_EMAIL): + job = jobs[owner]["analysis.job"] + assert job.datasite_owner_email == owner + + +def _apply_suggestions(jobs, error: ValueError) -> list: + """Apply each subscript the error suggests, chain or position, to the list that raised.""" + reached = [] + for accessor in re.findall(r"^ (\S+) — on ", str(error), flags=re.MULTILINE): + target = jobs + for key, position in re.findall(r'\["([^"]+)"\]|\[(\d+)\]', accessor): + target = target[key] if key else target[int(position)] + reached.append(target) + return reached + + +def test_ambiguous_name_suggests_chains_that_resolve(tmp_path: Path): + """Each chain the error suggests must reach one job, on the list that raised. + + An email key keeps jobs on either side, so the datasite email of one job + can be the submitter email of the other. Here SHARED owns one 'an' and + submitted the other: jobs[SHARED]["an"] still holds both, and suggesting + it sends the caller back to the same error. In jobs[SHARED] a second email + names the submitter of a job on SHARED's datasite, so the job SHARED + submitted elsewhere gets its position there. + """ + shared, other_ds, other_do = "shared@test.org", "c@test.org", "d@test.org" + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + def client(email): + return JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=email) + ) + + client(other_ds).submit_python_job( + user=shared, code_path=str(code_file), job_name="an" + ) + client(shared).scan_inbox() + client(shared).submit_python_job( + user=other_do, code_path=str(code_file), job_name="an" + ) + client(other_do).scan_inbox() + + jobs = client(shared).jobs + for candidates in (jobs, jobs[shared]): + with pytest.raises(ValueError, match="Multiple jobs are named 'an'") as exc: + candidates["an"] + reached = _apply_suggestions(candidates, exc.value) + assert len(reached) == 2, str(exc.value) + assert all(not hasattr(job, "__len__") for job in reached), str(exc.value) + assert {job.datasite_owner_email for job in reached} == {shared, other_do} + + +def test_email_key_names_emails_it_has(tmp_path: Path): + """An email with no jobs on it is a typo the message has to help with.""" + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + ds_client = JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=DS_EMAIL) + ) + ds_client.submit_python_job( + user=DO_EMAIL, code_path=str(code_file), job_name="analysis.job" + ) + + with pytest.raises(ValueError) as exc: + ds_client.jobs["nobody@test.org"] + + message = str(exc.value) + assert "nobody@test.org" in message + assert DO_EMAIL in message, "must name the datasites that do have jobs" + assert DS_EMAIL in message, "must name the submitters too" + + +def test_hint_adds_datasite_when_submitter_used_name_twice( + tmp_path: Path, +): + """One submitter can send the same name to the DO and to a peer. + + The submitter alone then reaches both, so this is the one case where the + DO's own datasite has to join the chain, and the only thing that earns the + third key. + """ + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + do_client = JobClient( + config=SyftJobConfig( + syftbox_folder=syftbox, current_user_email=DO_EMAIL, has_do_role=True + ) + ) + ds_client = JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=DS_EMAIL) + ) + for owner in (DO_EMAIL, PEER_EMAIL): + ds_client.submit_python_job( + user=owner, code_path=str(code_file), job_name="analysis.job" + ) + JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=owner) + ).scan_inbox() + + jobs = do_client.jobs + assert {job.datasite_owner_email for job in jobs} == {DO_EMAIL, PEER_EMAIL} + chain = f'jobs["{DO_EMAIL}"]["{DS_EMAIL}"]["analysis.job"]' + for rendering in (str(jobs), jobs._repr_html_()): + assert f"{chain}.approve()" in rendering + + # The submitter alone reaches both datasites, so the chain needs both keys. + with pytest.raises(ValueError, match="Multiple jobs are named 'analysis.job'"): + jobs[DS_EMAIL]["analysis.job"] + assert jobs[DO_EMAIL][DS_EMAIL]["analysis.job"].datasite_owner_email == DO_EMAIL + + +def test_job_name_cannot_contain_at_sign(tmp_path: Path): + """An '@' is how a subscript tells a datasite email from a job name.""" + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + ds_client = JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=DS_EMAIL) + ) + with pytest.raises(ValueError, match="cannot contain '@'"): + ds_client.submit_python_job( + user=DO_EMAIL, code_path=str(code_file), job_name="ds@test.org" + ) + + +def test_chained_emails_read_as_datasite_then_submitter(tmp_path: Path): + """Two people with both roles can send each other jobs of one name. + + Each email is a party to both jobs, so only the order of the two emails + separates them: the first names the datasite, the second the submitter. + """ + syftbox = tmp_path / "SyftBox" + syftbox.mkdir() + code_file = tmp_path / "main.py" + code_file.write_text(MAIN_PY) + + do_client = JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=DO_EMAIL) + ) + peer_client = JobClient( + config=SyftJobConfig(syftbox_folder=syftbox, current_user_email=PEER_EMAIL) + ) + peer_client.submit_python_job( + user=DO_EMAIL, code_path=str(code_file), job_name="analysis" + ) + do_client.scan_inbox() + do_client.submit_python_job( + user=PEER_EMAIL, code_path=str(code_file), job_name="analysis" + ) + peer_client.scan_inbox() + + jobs = do_client.jobs + mine = jobs[DO_EMAIL][PEER_EMAIL]["analysis"] + assert (mine.datasite_owner_email, mine.submitted_by) == (DO_EMAIL, PEER_EMAIL) + theirs = jobs[PEER_EMAIL][DO_EMAIL]["analysis"] + assert (theirs.datasite_owner_email, theirs.submitted_by) == (PEER_EMAIL, DO_EMAIL) + + with pytest.raises(ValueError, match="Multiple jobs are named 'analysis'") as exc: + jobs["analysis"] + assert not re.search(r"\[\d+\]", str(exc.value)), "an email chain separates them" + reached = _apply_suggestions(jobs, exc.value) + assert {(job.datasite_owner_email, job.submitted_by) for job in reached} == { + (DO_EMAIL, PEER_EMAIL), + (PEER_EMAIL, DO_EMAIL), + } diff --git a/packages/syft-rds/pyproject.toml b/packages/syft-rds/pyproject.toml index 785fb34e339..51476669bbf 100644 --- a/packages/syft-rds/pyproject.toml +++ b/packages/syft-rds/pyproject.toml @@ -9,7 +9,7 @@ requires-python = ">=3.10" dependencies = [ "syft>=0.10.0", # floor: the `syft` PyPI project also hosts legacy PySyft <=0.9 - "syft-job==0.1.40", + "syft-job>=0.1.41", "syft-dataset==0.1.21", ] diff --git a/packages/syft-rds/src/syft_rds/__init__.py b/packages/syft-rds/src/syft_rds/__init__.py index 3954622f053..7a26e46dfa6 100644 --- a/packages/syft-rds/src/syft_rds/__init__.py +++ b/packages/syft-rds/src/syft_rds/__init__.py @@ -1,5 +1,7 @@ """syft-rds: Remote Data Science product composed on top of syft.""" +from syft_job.logging_config import configure_package_logger + from syft_rds.client import DatasetUpgrade, SyftRDSClient, UpgradeReport from syft_rds.config import SyftRDSClientConfig from syft_rds.job_auto_approval import auto_approve_and_run_jobs, job_matches_criteria @@ -27,3 +29,6 @@ "Environment", "check_env", ] + +# Last, so the imports stay at the top. Nothing here logs at import time. +configure_package_logger(__name__) diff --git a/packages/syft-rds/src/syft_rds/client.py b/packages/syft-rds/src/syft_rds/client.py index c30f4960f97..1965ad259a6 100644 --- a/packages/syft-rds/src/syft_rds/client.py +++ b/packages/syft-rds/src/syft_rds/client.py @@ -99,6 +99,14 @@ def _repr_html_(self) -> str: return f"{self}{rows}
" if rows else f"{self}" +def _print_skipped_jobs(skipped: list[tuple[str, str, str]]) -> None: + """Report the approved jobs that did not run, and why.""" + print(f"\n⏭️ {len(skipped)} approved job(s) did not run:") + for job_name, peer_email, reason in skipped: + print(f" • {job_name} (submitted by {peer_email}): {reason}") + print(" Pass ignore_peer_version=True to run them anyway.") + + class SyftRDSClient(BaseModel): # Holds live service objects (sync engine + RDS-owned managers), not # serializable data, so arbitrary types are allowed. @@ -594,11 +602,18 @@ def process_approved_jobs( if self.job_runner is None: raise ValueError("Job runner is not configured for this client") - skip_job_names = [] + skipped: list[tuple[str, str, str]] = [] if not force_execution: + # Only jobs on this client's own datasite: job_client.jobs spans + # every datasite in the folder, but the runner never runs a job on + # a peer's. Reporting those as skipped would offer a remedy that + # cannot help. approved_jobs = [ - job for job in self.job_client.jobs if job.status == "approved" + job + for job in self.job_client.jobs + if job.status == "approved" + and job.datasite_owner_email == self.job_client.current_user_email ] for job in approved_jobs: result = self.sync_engine.peer_manager.get_peer_compatibility_status( @@ -608,16 +623,26 @@ def process_approved_jobs( ) result.maybe_warn() if result.should_skip: - skip_job_names.append(job.name) + skipped.append( + ( + job.name, + job.submitted_by, + result.explanation_skip or "peer version is not compatible", + ) + ) self.job_runner.process_approved_jobs( stream_output=stream_output, timeout=timeout, - skip_job_names=skip_job_names if skip_job_names else None, + skip_jobs=[(name, peer) for name, peer, _ in skipped] or None, share_outputs_with_submitter=share_outputs_with_submitter, share_logs_with_submitter=share_logs_with_submitter, ) + # maybe_warn() above logs the peer, never the job. + if skipped: + _print_skipped_jobs(skipped) + if self._pre_sync_enabled: self.sync_engine.sync() diff --git a/packages/syft-rds/tests/test_datasets_jobs_repr.py b/packages/syft-rds/tests/test_datasets_jobs_repr.py index 8d63b88cfca..65994119c51 100644 --- a/packages/syft-rds/tests/test_datasets_jobs_repr.py +++ b/packages/syft-rds/tests/test_datasets_jobs_repr.py @@ -1,5 +1,7 @@ """Tests for SyftDatasetManager and JobsList repr and indexing.""" +import warnings + import pytest from syft_rds import SyftRDSClient @@ -134,7 +136,12 @@ def test_dataset_repr_html_mentions_mock_files(): # --- JobsList tests --- -def _make_job_info(name: str, status: str = "pending") -> JobInfo: +def _make_job_info( + name: str, + status: str = "pending", + ds_email: str = "ds@test.com", + owner_email: str = "test@test.com", +) -> JobInfo: """Create a minimal JobInfo for testing.""" from datetime import datetime, timezone from pathlib import Path @@ -151,15 +158,15 @@ def _make_job_info(name: str, status: str = "pending") -> JobInfo: submission_config = JobSubmissionMetadata( name=name, type="python", - submitted_by="ds@test.com", - datasite_email="ds@test.com", + submitted_by=ds_email, + datasite_email=ds_email, submitted_at=datetime.now(timezone.utc), ) state = JobState(status=JobStatus(status)) # Identity (owner, submitter, name) comes from the path-derived ref. ref = JobRef( - datasite_email="test@test.com", - ds_email="ds@test.com", + datasite_email=owner_email, + ds_email=ds_email, job_name=name, protocol_version="1", ) @@ -198,6 +205,262 @@ def test_jobs_list_getitem_str_not_found(): jobs["nonexistent"] +def test_jobs_list_getitem_str_ambiguous(): + """Two submitters can use one job name; the lookup must not guess between them. + + Names are unique per datasite and submitter, so the same name can appear + more than once in one list. Returning the first match approves the wrong + job and says nothing. + """ + jobs = JobsList( + [ + _make_job_info("analysis", ds_email="ds1@test.com"), + _make_job_info("analysis", ds_email="ds2@test.com"), + ], + root_email="test@test.com", + ) + with pytest.raises(ValueError, match="Multiple jobs are named 'analysis'") as exc: + jobs["analysis"] + + message = str(exc.value) + # One datasite holds both, so the submitter is what separates them. + assert ( + '["ds1@test.com"]["analysis"] — on test@test.com from ds1@test.com' in message + ) + assert ( + '["ds2@test.com"]["analysis"] — on test@test.com from ds2@test.com' in message + ) + + +def test_jobs_list_getitem_str_ambiguous_names_the_datasite(): + """One submitter can send the same job name to two data owners. + + The submitter is then identical on every row, so the message has to name + the datasite as well. This is the DS-side call the README recommends. + """ + jobs = JobsList( + [ + _make_job_info("analysis", owner_email="do1@test.com"), + _make_job_info("analysis", owner_email="do2@test.com"), + ], + root_email="ds@test.com", + ) + with pytest.raises(ValueError) as exc: + jobs["analysis"] + + message = str(exc.value) + # One submitter sent both, so the datasite is what separates them. + assert '["do1@test.com"]["analysis"] — on do1@test.com' in message + assert '["do2@test.com"]["analysis"] — on do2@test.com' in message + + +def test_jobs_list_getitem_email_selects_datasite(): + """An email key narrows to one datasite, where the name is unique.""" + jobs = JobsList( + [ + _make_job_info("analysis", owner_email="do1@test.com"), + _make_job_info("analysis", owner_email="do2@test.com"), + ], + root_email="ds@test.com", + ) + on_do1 = jobs["do1@test.com"] + assert isinstance(on_do1, JobsList) + assert len(on_do1) == 1 + assert on_do1["analysis"].datasite_owner_email == "do1@test.com" + + +def test_jobs_list_getitem_email_not_found(): + jobs = JobsList( + [_make_job_info("analysis", owner_email="do1@test.com")], + root_email="ds@test.com", + ) + with pytest.raises(ValueError, match="No jobs involving nobody@test.com"): + jobs["nobody@test.com"] + + +def test_jobs_list_getitem_submitter_narrows_one_datasite(): + """Two submitters to one datasite share a name the datasite cannot separate. + + Chaining the submitter is what reaches one job, and without it the lookup + must keep raising rather than guess. + """ + jobs = JobsList( + [ + _make_job_info("analysis", ds_email="ds1@test.com"), + _make_job_info("analysis", ds_email="ds2@test.com"), + ], + root_email="test@test.com", + ) + with pytest.raises(ValueError, match="Multiple jobs are named 'analysis'"): + jobs["test@test.com"]["analysis"] + + job = jobs["test@test.com"]["ds2@test.com"]["analysis"] + assert job.submitted_by == "ds2@test.com" + + +def test_jobs_list_getitem_legacy_name_with_at_warns(): + """A job named before the '@' ban must stay readable, and say it is on notice. + + Nothing warned the submitter at the time, so the name is already on disk. + The email key answers first; only a key no party answers to falls back. + """ + jobs = JobsList( + [ + _make_job_info( + "report@2026-09", owner_email="do@test.com", ds_email="ds@test.com" + ) + ], + root_email="do@test.com", + ) + with pytest.warns(DeprecationWarning, match="deprecated"): + job = jobs["report@2026-09"] + assert job.name == "report@2026-09" + + # A real party still wins the key, and warns about nothing. + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert len(jobs["do@test.com"]) == 1 + + +def test_hint_prefers_pending_job_over_unique_name(): + """The hint's approve() only accepts a pending job, so status wins. + + A finished job with a name no one shares once won the pick, and handed the + data owner a command that raises on status. + """ + jobs = JobsList( + [ + _make_job_info("solo", ds_email="ds1@test.com", status="done"), + _make_job_info("analysis", ds_email="ds1@test.com", status="pending"), + _make_job_info("analysis", ds_email="ds2@test.com", status="pending"), + ], + root_email="test@test.com", + has_do_role=True, + ) + assert jobs.hint_accessor() == '["ds1@test.com"]["analysis"]' + + +def test_hint_chain_reaches_job_it_names(): + """Whatever chain the hint gives, subscripting by it must return that job. + + The hint and the lookup read a chain through the same evaluator, so this + holds for the two-key and three-key forms alike. + """ + jobs = JobsList( + [ + _make_job_info( + "analysis", owner_email="do@test.com", ds_email="ds@test.com" + ), + _make_job_info( + "analysis", owner_email="peer@test.com", ds_email="ds@test.com" + ), + ], + root_email="do@test.com", + has_do_role=True, + ) + # One submitter, two datasites: the chain needs all three keys. + assert jobs.hint_accessor() == '["do@test.com"]["ds@test.com"]["analysis"]' + + reached = jobs["do@test.com"]["ds@test.com"]["analysis"] + assert reached.datasite_owner_email == "do@test.com" + + +def test_hint_orders_datasite_before_submitter(): + """The hint's three-key chain must reach the DO's own job, not the one sent out.""" + jobs = JobsList( + [ + _make_job_info( + "analysis", owner_email="me@test.com", ds_email="peer@test.com" + ), + _make_job_info( + "analysis", owner_email="peer@test.com", ds_email="me@test.com" + ), + ], + root_email="me@test.com", + has_do_role=True, + ) + assert jobs.hint_accessor() == '["me@test.com"]["peer@test.com"]["analysis"]' + assert jobs["me@test.com"]["peer@test.com"]["analysis"].datasite_owner_email == ( + "me@test.com" + ) + + +def test_hint_gives_position_when_no_chain_resolves(): + """One submitter can hold a name twice across protocol layouts. + + Datasite and submitter are then identical on both rows, so no chain reaches + one job and the position is all that is left. + """ + jobs = JobsList( + [ + _make_job_info("analysis", ds_email="ds1@test.com"), + _make_job_info("analysis", ds_email="ds1@test.com"), + ], + root_email="test@test.com", + has_do_role=True, + ) + assert jobs.hint_accessor() == "[0]" + + +def test_ambiguous_name_offers_position_when_no_email_narrows(): + """One submitter can hold a name twice across protocol layouts. + + Datasite and submitter are identical on both rows, so naming either would + return the caller to this same error. Only the position separates them. + """ + jobs = JobsList( + [ + _make_job_info("analysis", ds_email="ds1@test.com"), + _make_job_info("analysis", ds_email="ds1@test.com"), + ], + root_email="test@test.com", + ) + with pytest.raises(ValueError) as exc: + jobs["analysis"] + + message = str(exc.value) + assert "[0] — on test@test.com from ds1@test.com" in message + assert "[1] — on test@test.com from ds1@test.com" in message + assert '["' not in message, "no email narrows this" + + +def test_no_hint_when_no_owned_job_is_pending(): + """The hint's approve() takes a pending job, so a reviewed list gets no hint. + + Every job here is finished, and naming one would print a command that + raises on status rather than on ownership. + """ + jobs = JobsList( + [ + _make_job_info("analysis", status="done"), + _make_job_info("failed-one", status="failed"), + _make_job_info("approved-one", status="approved"), + ], + root_email="test@test.com", + has_do_role=True, + ) + assert jobs.hint_accessor() is None + + +def test_jobs_list_repr_counts_distinct_owners(): + """The owner total counts owners, not table sections. + + The renderers group consecutive rows to keep the table order equal to the + list order. A list that is not grouped by owner gives an owner more than + one section, which must not inflate the total. + """ + jobs = JobsList( + [ + _make_job_info("job-a", owner_email="do1@test.com"), + _make_job_info("job-b", owner_email="do2@test.com"), + _make_job_info("job-c", owner_email="do1@test.com"), + ], + root_email="do1@test.com", + ) + assert "3 jobs across 2 users" in str(jobs) + assert "3 jobs across 2 users" in jobs._repr_html_() + + def test_jobs_list_getitem_invalid_type(): jobs = JobsList( [_make_job_info("job-a")], diff --git a/packages/syft-rds/tests/test_version_negotiation.py b/packages/syft-rds/tests/test_version_negotiation.py index aaf422a92b5..56da05bd1b5 100644 --- a/packages/syft-rds/tests/test_version_negotiation.py +++ b/packages/syft-rds/tests/test_version_negotiation.py @@ -1,9 +1,12 @@ """Version gating on the RDS job submission and execution paths.""" +from unittest.mock import patch + import pytest from syft.sync.version.exceptions import VersionUnknownError -from syft.sync.version.version_info import VersionInfo +from syft.sync.version.peer_manager import PeerCompatibilityResult, PeerManager +from syft.sync.version.version_info import CompatibilityStatus, VersionInfo from syft_rds import SyftRDSClient @@ -113,9 +116,9 @@ def test_job_execution_forced_with_incompatible_version(self): executed_jobs = [] def mock_process_approved_jobs( - stream_output=True, timeout=None, skip_job_names=None, **kwargs + stream_output=True, timeout=None, skip_jobs=None, **kwargs ): - executed_jobs.append(skip_job_names) + executed_jobs.append(skip_jobs) do.job_runner.process_approved_jobs = mock_process_approved_jobs @@ -123,3 +126,44 @@ def mock_process_approved_jobs( assert len(executed_jobs) == 1 assert executed_jobs[0] is None # No jobs skipped when force=True + + +class TestSkippedJobsAreReported: + """process_approved_jobs must say which approved jobs it did not run. + + maybe_warn() logs the peer, not the job, so on its own it leaves the data + owner with a job stuck at 'approved' and nothing naming it. + """ + + def test_skipped_job_name_and_reason_reach_stdout(self, tmp_path, capfd): + ds, do = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + sync_automatically=False, + ) + + code_path = tmp_path / "skipped.py" + code_path.write_text('print("hello")') + ds.submit_python_job( + user=do.email, code_path=str(code_path), job_name="skipped.job" + ) + do.sync() + do.jobs["skipped.job"].approve() + + skip = PeerCompatibilityResult( + peer_email=ds.email, + status=CompatibilityStatus.INCOMPATIBLE, + should_skip=True, + explanation_skip=f"Skipping peer {ds.email}: incompatible version.", + ) + capfd.readouterr() + with patch.object( + PeerManager, "get_peer_compatibility_status", return_value=skip + ): + do.process_approved_jobs() + out, _ = capfd.readouterr() + + assert "skipped.job" in out + assert ds.email in out + assert "incompatible version" in out + assert "ignore_peer_version=True" in out + assert do.jobs["skipped.job"].status == "approved" diff --git a/syft/__init__.py b/syft/__init__.py index 04033450b27..5a58db3189d 100644 --- a/syft/__init__.py +++ b/syft/__init__.py @@ -20,6 +20,7 @@ from syft.version import SYFT_VERSION as __version__ # noqa: F401, E402 from syft.sync.login import login_do, login_ds, login # noqa: F401, E402 +from syft.sync.peers.exceptions import SyftPeerNotReadyError # noqa: F401, E402 from syft.utils import ( # noqa: F401, E402 resolve_path, resolve_dataset_file_path, diff --git a/syft/sync/peers/exceptions.py b/syft/sync/peers/exceptions.py new file mode 100644 index 00000000000..88d136ac3f7 --- /dev/null +++ b/syft/sync/peers/exceptions.py @@ -0,0 +1,22 @@ +"""Peer-related exceptions for syft.""" + + +class SyftPeerNotReadyError(ValueError): + """A peer is known but not yet usable for the operation. + + Carries the cause and the remedy separately so callers — a readiness + helper, or a diagnostic — can show either one on its own. + + Subclasses ValueError because these conditions were reported as a bare + ValueError before, and callers catch that. + """ + + def __init__(self, peer_email: str, cause: str, remedy: str): + self.peer_email = peer_email + self.cause = cause + self.remedy = remedy + super().__init__(f"{cause} {remedy}") + + def __reduce__(self): + # BaseException rebuilds from self.args, which holds only the message. + return (type(self), (self.peer_email, self.cause, self.remedy)) diff --git a/syft/sync/peers/peer_store.py b/syft/sync/peers/peer_store.py index eef769016e7..c94a21f6a15 100644 --- a/syft/sync/peers/peer_store.py +++ b/syft/sync/peers/peer_store.py @@ -17,6 +17,7 @@ import syft_crypto_python as syc from pydantic import BaseModel, PrivateAttr +from syft.sync.peers.exceptions import SyftPeerNotReadyError from syft.sync.peers.key_bundle import ( InvalidPeerBundleError, PeerKeyChangedError, @@ -25,7 +26,7 @@ format_fingerprint, parse_and_validate_bundle, ) -from syft.sync.peers.peer import Peer +from syft.sync.peers.peer import Peer, PeerState logger = logging.getLogger(__name__) @@ -234,21 +235,65 @@ def _is_trusted_replacement( def _ensure_private_keys(self) -> syc.SyftPrivateKeys: if self._private_keys is None: - raise ValueError("No private keys — call generate_keys() first") + raise ValueError( + f"No private keys for {self.email}. Encryption is on for this " + "client, but it holds no key pair. Keys are created at login, " + "so log in again to create them." + ) return self._private_keys def _ensure_peer(self, email: str) -> Peer: peer = self.get_cached_peer(email) if peer is None: - raise ValueError(f"No cached peer for {email}") + raise SyftPeerNotReadyError( + email, + cause=f"No cached peer for {email}.", + remedy=( + "Run client.sync() to refresh your peers, or " + f"client.add_peer('{email}') if you have not added them." + ), + ) return peer def _ensure_peer_bundle(self, email: str) -> dict: peer = self._ensure_peer(email) if peer.public_encryption_bundle is None: - raise ValueError(f"No public encryption bundle for {email}") + cause, remedy = self._missing_bundle_reason(peer) + raise SyftPeerNotReadyError(email, cause=cause, remedy=remedy) return peer.public_encryption_bundle + @staticmethod + def _missing_bundle_reason(peer: Peer) -> tuple[str, str]: + """The cause and the remedy for a peer that has no encryption bundle. + + A bundle is only stored for a peer who is accepted (or requested by + you) and who has already published one, so the four states below are + the four reasons the bundle can be missing. + """ + email = peer.email + if peer.state == PeerState.REQUESTED_BY_PEER: + return ( + f"{email} asked to peer with you, but you have not accepted yet.", + f"Run client.approve_peer_request('{email}').", + ) + if peer.state == PeerState.REJECTED: + return ( + f"You rejected {email}, so no encryption bundle was kept.", + f"Run client.add_peer('{email}') to peer with them again.", + ) + if peer.state == PeerState.REQUESTED_BY_ME: + return ( + f"You asked to peer with {email}, but they have not accepted yet.", + "Wait for them to accept, then run client.sync().", + ) + return ( + f"{email} is an accepted peer, but you do not have their public " + "encryption bundle. Either they have not published one yet, or you " + "have not synced since they did.", + "Run client.sync(). If it continues, the peer may not have " + "encryption enabled.", + ) + # ========== Crypto methods ========== def generate_keys(self) -> None: diff --git a/tests/unit/test_encryption.py b/tests/unit/test_encryption.py index cfc839d871f..81fc9dbaeb1 100644 --- a/tests/unit/test_encryption.py +++ b/tests/unit/test_encryption.py @@ -1,9 +1,12 @@ +import copy +import pickle import tempfile from pathlib import Path import pytest -from syft.sync.peers.peer import Peer +from syft.sync.peers.exceptions import SyftPeerNotReadyError +from syft.sync.peers.peer import Peer, PeerState from syft.sync.peers.peer_store import PeerStore from syft.sync.syftbox_manager import SyftboxManager from tests.unit.test_sync_manager import path_for_job @@ -49,22 +52,73 @@ def test_encrypt_decrypt_roundtrip(): def test_encrypt_without_keys_raises(): ps = PeerStore(email="alice@example.com", use_encryption=True) ps.add_peer(Peer(email="bob@example.com")) - with pytest.raises(ValueError, match="No private keys"): + with pytest.raises(ValueError, match="Keys are created at login"): ps.encrypt("bob@example.com", b"data") -def test_encrypt_without_peer_bundle_raises(): +@pytest.mark.parametrize( + "state, cause, remedy", + [ + ( + PeerState.ACCEPTED, + "accepted peer, but you do not have their public", + "Run client.sync()", + ), + ( + PeerState.REQUESTED_BY_ME, + "they have not accepted yet", + "Wait for them to accept", + ), + ( + PeerState.REQUESTED_BY_PEER, + "you have not accepted yet", + "client.approve_peer_request('bob@example.com')", + ), + ( + PeerState.REJECTED, + "You rejected bob@example.com", + "client.add_peer('bob@example.com')", + ), + ], +) +def test_encrypt_without_peer_bundle_names_the_cause(state, cause, remedy): + """A missing bundle means one of four things; the message must say which. + + Naming neither the cause nor the remedy is what left data owners stuck on + this error with nothing to act on. + """ ps = PeerStore(email="alice@example.com", use_encryption=True) ps.generate_keys() - ps.add_peer(Peer(email="bob@example.com")) - with pytest.raises(ValueError, match="No public encryption bundle"): + ps.add_peer(Peer(email="bob@example.com", state=state)) + + with pytest.raises(SyftPeerNotReadyError) as exc: ps.encrypt("bob@example.com", b"data") + assert cause in exc.value.cause + assert remedy in exc.value.remedy + assert str(exc.value) == f"{exc.value.cause} {exc.value.remedy}" + + +def test_peer_not_ready_error_is_a_value_error(): + """Callers that catch ValueError keep working.""" + assert issubclass(SyftPeerNotReadyError, ValueError) + + +def test_peer_not_ready_error_survives_pickle_and_copy(): + error = SyftPeerNotReadyError("p@test.org", cause="Cause.", remedy="Remedy.") + for clone in (pickle.loads(pickle.dumps(error)), copy.copy(error)): + assert (clone.peer_email, clone.cause, clone.remedy) == ( + "p@test.org", + "Cause.", + "Remedy.", + ) + assert str(clone) == "Cause. Remedy." + def test_try_decrypt_no_keys(): ps = PeerStore(email="alice@example.com", use_encryption=True) data = b"some unencrypted data" - with pytest.raises(ValueError, match="No private keys"): + with pytest.raises(ValueError, match="Keys are created at login"): ps.decrypt("bob@example.com", data) == data @@ -72,8 +126,10 @@ def test_try_decrypt_no_peer_bundle(): ps = PeerStore(email="alice@example.com", use_encryption=True) ps.generate_keys() data = b"some unencrypted data" - with pytest.raises(ValueError, match="No cached peer for"): - ps.decrypt("bob@example.com", data) == data + with pytest.raises(SyftPeerNotReadyError) as exc: + ps.decrypt("bob@example.com", data) + assert "No cached peer for bob@example.com" in exc.value.cause + assert "client.add_peer('bob@example.com')" in exc.value.remedy def test_try_decrypt_invalid_envelope(): diff --git a/tests/unit/test_package_logging.py b/tests/unit/test_package_logging.py new file mode 100644 index 00000000000..5e22624f071 --- /dev/null +++ b/tests/unit/test_package_logging.py @@ -0,0 +1,112 @@ +"""Each package must configure its own logger when imported on its own. + +`syft/__init__.py` configures only the `syft` logger. The other packages are +sibling top-level namespaces, so without this they inherit the root logger — +no handler, level WARNING — and every logger.info/warning/error call in them +is dropped, swallowed tracebacks included. + +Each case runs in its own interpreter: the point is that importing the package +alone is enough, with no dependence on `syft` being imported first. +""" + +import subprocess +import sys +from importlib.metadata import requires + +import pytest +from packaging.requirements import Requirement +from packaging.version import Version + +PACKAGES = ["syft", "syft_job", "syft_rds", "syft_enclaves", "syft_bg"] + +# The last syft-job release without logging_config. +SYFT_JOB_WITHOUT_LOGGING_CONFIG = Version("0.1.40") + +PROBE = """ +import importlib, logging, sys +importlib.import_module({name!r}) +logger = logging.getLogger({name!r}) +print(logger.getEffectiveLevel(), bool(logger.handlers)) +""" + + +@pytest.mark.parametrize("package", PACKAGES) +def test_package_logger_is_audible(package): + result = subprocess.run( + [sys.executable, "-c", PROBE.format(name=package)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + level, has_handler = result.stdout.strip().splitlines()[-1].split() + assert int(level) == 20, f"{package} effective level is {level}, want INFO (20)" + assert has_handler == "True", f"{package} logger has no handler" + + +DUPLICATE_PROBE = """ +import importlib, logging +importlib.import_module({name!r}) +logging.basicConfig(level=logging.INFO, format="ROOT %(message)s") +logging.getLogger({name!r} + ".child").warning("probe-record") +""" + + +@pytest.mark.parametrize("package", PACKAGES[1:]) +def test_package_logger_does_not_duplicate_root_output(package): + result = subprocess.run( + [sys.executable, "-c", DUPLICATE_PROBE.format(name=package)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert result.stderr.count("probe-record") == 1, result.stderr + assert "ROOT probe-record" in result.stderr + + +ENCLAVE_LEVEL_PROBE = """ +import logging +from syft_enclaves.__main__ import _configure_logging +_configure_logging({level!r}) +for name in ("syft_enclaves.runner", "syft_rds.client", "syft_job.job_runner"): + log = logging.getLogger(name) + log.debug("debug-" + name) + log.info("info-" + name) + log.warning("warning-" + name) +""" + + +@pytest.mark.parametrize( + "level, shown, hidden", + [ + ("WARNING", ["warning"], ["info", "debug"]), + ("DEBUG", ["debug", "info", "warning"], []), + ], +) +def test_enclave_log_level_applies_to_package_loggers(level, shown, hidden): + result = subprocess.run( + [sys.executable, "-c", ENCLAVE_LEVEL_PROBE.format(level=level)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + for name in ("syft_enclaves.runner", "syft_rds.client", "syft_job.job_runner"): + for kind in shown: + assert result.stderr.count(f"{kind}-{name}") == 1, result.stderr + for kind in hidden: + assert f"{kind}-{name}" not in result.stderr, result.stderr + + +@pytest.mark.parametrize("distribution", ["syft-rds", "syft-bg", "syft-enclave"]) +def test_dependents_require_syft_job_with_logging_config(distribution): + """A package that imports syft_job.logging_config must not accept an older syft-job. + + The workspace resolves syft-job from the local source, so the import tests + above pass whatever the pin says. A released wheel installs from PyPI, where + an older syft-job has no logging_config and the import fails. + """ + syft_job = next( + Requirement(line) + for line in requires(distribution) or [] + if Requirement(line).name == "syft-job" + ) + assert SYFT_JOB_WITHOUT_LOGGING_CONFIG not in syft_job.specifier, str(syft_job) diff --git a/uv.lock b/uv.lock index 32257233c0f..3abc04f4434 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", @@ -983,7 +983,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1421,8 +1421,8 @@ name = "httpcore2" version = "2.13.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "h11", marker = "python_full_version < '3.11' or sys_platform != 'emscripten'" }, - { name = "truststore", marker = "python_full_version < '3.11' or sys_platform != 'emscripten'" }, + { name = "h11" }, + { name = "truststore" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/f3/1db7aa2bc2524062192bb0e0323969492d1883152a232fe36eea65f4e35c/httpcore2-2.13.1.tar.gz", hash = "sha256:e0aa977abe17e69a3b820a24542a6fa88702676d83880b8d194dcd18408e5103", size = 68071, upload-time = "2026-09-23T07:47:22.372Z" } wheels = [ @@ -1563,17 +1563,17 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version < '3.11'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "jedi", marker = "python_full_version < '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, - { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "stack-data", marker = "python_full_version < '3.11'" }, - { name = "traitlets", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "exceptiongroup" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } wheels = [ @@ -1602,17 +1602,17 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, - { name = "jedi", marker = "python_full_version >= '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, - { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, - { name = "psutil", marker = "python_full_version >= '3.11' and sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "stack-data", marker = "python_full_version >= '3.11'" }, - { name = "traitlets", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/52/49/04360f83b4d110195751b4171b75dc1cd7b97ba122b18da34b5828172d59/ipython-9.16.0.tar.gz", hash = "sha256:d2f92587b1ef51d84f934dffe05fabb9255f0038ed0a21426f2ea761e39ad09a", size = 4515375, upload-time = "2026-07-31T08:02:51.977Z" } wheels = [ @@ -1624,7 +1624,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -3109,10 +3109,10 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "python-dateutil", marker = "python_full_version < '3.11'" }, - { name = "pytz", marker = "python_full_version < '3.11'" }, - { name = "tzdata", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -3187,10 +3187,10 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, - { name = "tzdata", marker = "(python_full_version >= '3.11' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } wheels = [ @@ -3269,7 +3269,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "ptyprocess" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -4959,6 +4959,7 @@ dependencies = [ { name = "pydantic-settings" }, { name = "requests" }, { name = "syft" }, + { name = "syft-job" }, { name = "syft-rds" }, ] @@ -4975,6 +4976,7 @@ requires-dist = [ { name = "pyyaml", marker = "extra == 'tinfoil'", specifier = ">=6.0" }, { name = "requests", specifier = ">=2.32.0" }, { name = "syft", editable = "." }, + { name = "syft-job", editable = "packages/syft-job" }, { name = "syft-rds", editable = "packages/syft-rds" }, { name = "tinfoil", marker = "extra == 'tinfoil'", specifier = ">=0.14.0" }, ]