Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions application/single_app/admin_settings_fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -5015,9 +5015,9 @@
"label": "Screen workspace content before publication",
"help": (
"Hold extracted workspace knowledge until required checks complete. "
"Findings require an authorized workspace review. Configure an active "
"screening policy before enabling; disabling future scans never releases "
"existing holds."
"Enabling creates an enabled empty baseline if none exists. With no "
"applicable checks, new uploads use normal processing. Add checks later; "
"emptying a policy or disabling future scans never releases existing holds."
),
"default": False,
"requires": {
Expand All @@ -5033,7 +5033,7 @@
"type": "component",
"component": "content-screening-policy",
"label": "Screening policies and scans",
"help": "Edit required PII, regex, value, and model checks; policies are saved separately from Admin Settings.",
"help": "Save an empty policy or configure PII, regex, value, and model checks; policies are saved separately from Admin Settings.",
},
],
"content-safety-section": [
Expand Down
2 changes: 1 addition & 1 deletion application/single_app/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@
EXECUTOR_TYPE = 'thread'
EXECUTOR_MAX_WORKERS = 30
SESSION_TYPE = 'filesystem'
VERSION = "0.261.113"
VERSION = "0.261.114"
IS_DEVELOPMENT = is_development_env_enabled()

SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax')
Expand Down
13 changes: 11 additions & 2 deletions application/single_app/content_screening/contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,17 @@ class ScreeningPolicyRequiredError(ScreeningConfigurationError):
code = "screening_policy_required"
status_code = 400
public_message = (
"Save an enabled policy with at least one rule or model check under "
"Security > Content Screening before enabling new scans."
"The saved content screening policy is unavailable. Save Content Screening "
"settings again to initialize a missing policy. Existing holds are unchanged."
)


class ScreeningChecksRequiredError(ScreeningConfigurationError):
code = "screening_policy_empty"
status_code = 400
public_message = (
"No active checks are configured for this workspace. Add and save a rule or "
"AI check before starting a scan. An empty policy can stay enabled; existing holds are unchanged."
)


Expand Down
9 changes: 9 additions & 0 deletions application/single_app/content_screening/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@
HELD_STATES,
SCREENING_FIELD,
SCOPE_TYPES,
ScreeningChecksRequiredError,
ScreeningConfigurationError,
ScreeningConflictError,
ScreeningError,
ScreeningPolicyRequiredError,
ScreeningValidationError,
Subject,
content_fingerprint,
Expand All @@ -35,6 +37,7 @@
)
from content_screening.repository import MAX_DOCUMENT_SELECTION, get_repository
from content_screening.permissions import ScreeningPermissionError, assert_scope_access
from content_screening.policies import compose_policy, policy_is_active


LEASE_SECONDS = 300
Expand Down Expand Up @@ -162,9 +165,13 @@ def _configuration_snapshot(selection, repository):
raise ScreeningConfigurationError("Enable content screening before creating a scan job.")
service.validate_screening_configuration(settings, repository=repository, check_storage=True)
baseline = repository.get_policy("global", "global")
if baseline is None:
raise ScreeningPolicyRequiredError()
workspace = None
if not selection.get("all_workspaces"):
workspace = repository.get_policy(selection["scope_type"], selection["scope_id"])
if not policy_is_active(compose_policy(baseline["policy"], workspace["policy"] if workspace else None)):
raise ScreeningChecksRequiredError()
return {
"baseline": baseline.get("policy_fingerprint") or hash_payload(baseline["policy"]) if baseline else None,
"workspace": workspace.get("policy_fingerprint") or hash_payload(workspace["policy"]) if workspace else None,
Expand Down Expand Up @@ -1129,6 +1136,8 @@ def _process_item(repository, job, item, owner, processor, *, completion_only=Fa
status, code = "incomplete" if item.get("started") else "skipped", "screening_permission_revoked"
elif isinstance(exc, ScreeningConflictError):
status = "incomplete" if item.get("started") else "skipped"
elif code == "screening_policy_empty":
status = "incomplete" if item.get("started") else "skipped"
elif code == "screening_table_source_missing":
status = "incomplete"
elif int(item.get("attempts", 0)) >= MAX_ATTEMPTS:
Expand Down
8 changes: 1 addition & 7 deletions application/single_app/content_screening/policies.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
Starter rules are indicators, not a complete PII or prompt-injection classifier.
Email support is Unicode-aware; phone checks cover common North American and
international-plus formats, SSNs are US-specific, and cards use Luhn validation.
Administrators must explicitly select checks before enabling screening.
An enabled empty policy is valid configuration, but does not enroll new content.
"""

import math
Expand Down Expand Up @@ -407,8 +407,6 @@ def normalize_policy(value, *, scope_type="global"):
}
if scope_type != "global" and result["allowed_models"]:
_invalid("Only administrators can approve scanner models.")
if result["enabled"] and not (any(rule["enabled"] for rule in result["rules"]) or result["ai"]["enabled"]):
_invalid("Select at least one active check before enabling screening.", code="screening_policy_empty")
result["fingerprint"] = hash_payload(result)
return result

Expand Down Expand Up @@ -479,12 +477,8 @@ def normalize_effective_policy(value):
"baseline_fingerprint": _fingerprint(value["baseline_fingerprint"]),
"workspace_fingerprint": _fingerprint(value["workspace_fingerprint"], allow_none=True),
}
if enabled and not (rules or ai_checks):
_invalid("The effective policy has no active checks.", code="screening_policy_empty")
if not enabled and (rules or ai_checks):
_invalid("A disabled effective policy cannot contain required checks.")
if enabled and not any(item["origin"] == "global" for item in rules + ai_checks):
_invalid("An effective policy must retain required baseline checks.")
for check in ai_checks:
if check["model_selection"] not in result["allowed_models"]:
_invalid("An effective AI check selected an unapproved model.")
Expand Down
55 changes: 43 additions & 12 deletions application/single_app/content_screening/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
DocumentHeldError,
Finding,
InspectionResult,
ScreeningChecksRequiredError,
ScreeningCitationsRequiredError,
ScreeningConfigurationError,
ScreeningConflictError,
Expand All @@ -42,6 +43,7 @@
capture_table_source,
publication_context,
)
from content_screening.policies import compose_policy, default_policy, policy_is_active


LEASE_SECONDS = 1800
Expand Down Expand Up @@ -104,35 +106,64 @@ def _save_scan(repository, scan, **updates):
return repository.replace({**scan, **updates, "updated_at": _timestamp()}, scan["_etag"])


def get_effective_policy(subject, *, repository=None):
from content_screening.policies import compose_policy, default_policy, policy_is_active
def initialize_screening_policy(*, repository=None):
"""Create the first empty baseline without overwriting a concurrent policy."""
repository = _repository(repository)
baseline = repository.get_policy("global", "global")
if baseline is not None:
return baseline
policy = {**default_policy(), "enabled": True}
try:
return repository.save_policy("global", "global", policy, "system-content-screening")
except ScreeningConflictError:
baseline = repository.get_policy("global", "global")
if baseline is None:
raise
return baseline


def get_effective_policy(subject, *, repository=None, require_active=True):
repository = _repository(repository)
baseline = repository.get_policy("global", "global")
if baseline is None:
raise ScreeningPolicyRequiredError()
workspace = repository.get_policy(subject.scope_type, subject.scope_id)
policy = compose_policy(
baseline["policy"] if baseline else default_policy(),
baseline["policy"],
workspace["policy"] if workspace else None,
)
if not policy_is_active(policy):
raise ScreeningConfigurationError("An active content screening policy is required.")
if require_active and not policy_is_active(policy):
raise ScreeningChecksRequiredError()
return policy


def validate_screening_configuration(settings=None, *, repository=None, check_storage=False, proposed_settings=False):
def document_requires_screening(document, settings=None, *, repository=None):
"""Persisted enrollment always wins over the absence of checks for new uploads."""
if not isinstance(document, dict):
raise ScreeningValidationError("The document metadata is unavailable.")
if SCREENING_FIELD in document:
return True
if _settings(settings).get("enable_content_screening") is not True:
return False
return policy_is_active(get_effective_policy(
subject_from_document(document), repository=repository, require_active=False,
))


def validate_screening_configuration(settings=None, *, repository=None, check_storage=False,
proposed_settings=False, allow_missing_policy=False):
settings = _settings(settings)
if settings.get("enable_content_screening") is not True:
return
if settings.get("enable_enhanced_citations") is not True:
raise ScreeningCitationsRequiredError()

from content_screening.policies import compose_policy, default_policy, policy_is_active

repository = _repository(repository)
baseline = repository.get_policy("global", "global")
effective = compose_policy(baseline["policy"] if baseline else default_policy())
if not policy_is_active(effective):
if baseline is None and not allow_missing_policy:
raise ScreeningPolicyRequiredError()
# Settings preflight may precede first activation; the write initializes the policy.
effective = compose_policy(baseline["policy"] if baseline else default_policy())
if effective.get("ai_checks"):
from content_screening.model import validate_model_bindings

Expand All @@ -146,7 +177,7 @@ def validate_screening_configuration(settings=None, *, repository=None, check_st


def initial_document_marker(document, settings=None):
if _settings(settings).get("enable_content_screening") is not True:
if not document_requires_screening(document, settings):
return None
subject = subject_from_document(document)
return {
Expand Down Expand Up @@ -378,7 +409,7 @@ def prepare_document_upload(document_id, user_id, temp_file_path, original_filen
extraction_mode_override=None):
settings = _settings()
document = _document_for_upload(document_id, user_id, group_id, public_workspace_id)
if settings.get("enable_content_screening") is not True and SCREENING_FIELD not in document:
if not document_requires_screening(document, settings):
return None
if settings.get("enable_content_screening") is not True:
raise DocumentHeldError("Enable content screening before replacing inspected content.")
Expand Down
5 changes: 3 additions & 2 deletions application/single_app/functions_documents.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
is_publication,
)
from content_screening.service import (
document_requires_screening,
initial_document_marker,
prepare_document_deletion,
prepare_document_upload,
Expand Down Expand Up @@ -8182,7 +8183,7 @@ def _download_document_source_to_temp_file(document_item, user_id=None, group_id
def process_document_reprocess_extraction_background(document_id, user_id, target_extraction_mode, group_id=None, public_workspace_id=None):
"""Extract a stored PDF or image again with an explicit Standard/Enhanced mode."""
document = get_document_metadata(document_id, user_id, group_id, public_workspace_id)
if get_settings().get("enable_content_screening") is True or (document and SCREENING_FIELD in document):
if document_requires_screening(document, get_settings()):
return reprocess_document(
subject_from_document(document), user_id,
normalize_document_intelligence_manual_extraction_mode(target_extraction_mode),
Expand Down Expand Up @@ -9378,7 +9379,7 @@ def _resolve_processing_complete_status(total_chunks_saved, file_ext, image_exte
def process_document_upload_background(document_id, user_id, temp_file_path, original_filename, group_id=None, public_workspace_id=None, extraction_mode_override=None):
"""Keep screened intake private until its complete, revision-bound decision."""
document = get_document_metadata(document_id, user_id, group_id, public_workspace_id)
if get_settings().get("enable_content_screening") is True or (document and SCREENING_FIELD in document):
if document_requires_screening(document, get_settings()):
return process_screened_upload(
document_id, user_id, temp_file_path, original_filename,
_process_document_upload_background_impl,
Expand Down
7 changes: 7 additions & 0 deletions application/single_app/functions_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -2135,6 +2135,7 @@ def validate_content_screening_settings(new_settings, current_settings, *, repos
validate_screening_configuration(
merged, repository=repository, check_storage=activating or storage_changed,
proposed_settings=True,
allow_missing_policy=new_settings.get('enable_content_screening') is True,
)
except ScreeningError:
raise
Expand Down Expand Up @@ -2173,6 +2174,12 @@ def update_settings(new_settings):
with embedding_settings_write_guard(
original, settings_item, force_check=EMBEDDING_SELECTION_KEY in new_settings,
):
if new_settings.get('enable_content_screening') is True:
# First activation is create-only; a concurrent policy must be revalidated.
from content_screening.service import initialize_screening_policy, validate_screening_configuration

initialize_screening_policy()
validate_screening_configuration(settings_item, proposed_settings=True)
persisted = cosmos_settings_container.replace_item(
item="app_settings", body=settings_item, etag=original["_etag"],
match_condition=MatchConditions.IfNotModified,
Expand Down
7 changes: 2 additions & 5 deletions application/single_app/route_backend_content_screening.py
Original file line number Diff line number Diff line change
Expand Up @@ -383,7 +383,7 @@ def content_screening_get_policy(scope_type, scope_id):
@login_required
@user_required
def content_screening_save_policy(scope_type, scope_id):
from content_screening.policies import compose_policy, default_policy, normalize_policy, policy_is_active
from content_screening.policies import compose_policy, default_policy, normalize_policy

scope_id = normalize_identifier(scope_id, "scope_id")
_authorize_policy(scope_type, scope_id)
Expand All @@ -392,10 +392,7 @@ def content_screening_save_policy(scope_type, scope_id):
raise ScreeningValidationError()
repository = _repository()
policy = normalize_policy(value["policy"], scope_type=scope_type)
if scope_type == "global":
if (get_settings() or {}).get("enable_content_screening") is True and not policy_is_active(compose_policy(policy)):
raise ScreeningConfigurationError()
else:
if scope_type != "global":
baseline = repository.get_policy("global", "global")
compose_policy(baseline["policy"] if baseline else default_policy(), policy)
repository.save_policy(scope_type, scope_id, policy, _actor_id(), etag=value["etag"])
Expand Down
Loading