chore: end-to-end guardrails logging of traces (llm_call--> guardrails execution) in Sentry and other chores

.env.example

@@ -16,6 +16,7 @@ POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 
 SENTRY_DSN=
+OTEL_ENABLED=false
 
 # Configure these with your own Docker registry images
 

.github/workflows/continuous-integration.yml

@@ -42,7 +42,6 @@ jobs:
             AUTH_TOKEN_HASH="$(echo -n "ci-test-token" | sha256sum | awk '{print $1}')"
           fi
           sed -i "s|AUTH_TOKEN=\"<ADD-HASH-TOKEN>\"|AUTH_TOKEN=\"${AUTH_TOKEN_HASH}\"|" .env.test
-          sed -i "s|GUARDRAILS_HUB_API_KEY=\"<ADD-KEY>\"|GUARDRAILS_HUB_API_KEY=\"${{ secrets.GUARDRAILS_HUB_API_KEY }}\"|" .env.test
           cp .env.test .env
 
       - name: Install uv
@@ -55,15 +54,6 @@ jobs:
         run: uv sync
         working-directory: backend
 
-      - name: Install Guardrails hub validators
-        env:
-          GUARDRAILS_HUB_API_KEY: ${{ secrets.GUARDRAILS_HUB_API_KEY }}
-        run: |
-          source .venv/bin/activate
-          chmod +x scripts/install_guardrails_from_hub.sh
-          scripts/install_guardrails_from_hub.sh
-        working-directory: backend
-
       - name: Activate virtual environment and run Alembic migrations
         run: |
           source .venv/bin/activate

.gitignore

@@ -16,4 +16,6 @@ backend/app/evaluation/outputs/**/*.json
 
 # marimo related artefacts
 backend/notebooks/*.csv
-**/__marimo__/
+**/__marimo__/
+
+backend/logs/app.log

backend/.guardrails/hub_registry.json

@@ -6,39 +6,39 @@
       "exports": [
         "BanList"
       ],
-      "installed_at": "2026-08-18T06:55:12.094065+00:00",
+      "installed_at": "2026-08-26T05:41:11.480494+00:00",
       "package_name": "guardrails-grhub-ban-list"
     },
     "guardrails/llm_critic": {
       "import_path": "guardrails_grhub_llm_critic",
       "exports": [
         "LLMCritic"
       ],
-      "installed_at": "2026-08-18T06:57:19.392503+00:00",
+      "installed_at": "2026-08-26T05:41:14.741349+00:00",
       "package_name": "guardrails-grhub-llm-critic"
     },
     "guardrails/llamaguard_7b": {
       "import_path": "guardrails_grhub_llamaguard_7b",
       "exports": [
         "LlamaGuard7B"
       ],
-      "installed_at": "2026-08-18T06:56:38.901794+00:00",
+      "installed_at": "2026-08-26T05:41:18.742264+00:00",
       "package_name": "guardrails-grhub-llamaguard-7b"
     },
     "guardrails/profanity_free": {
       "import_path": "guardrails_grhub_profanity_free",
       "exports": [
         "ProfanityFree"
       ],
-      "installed_at": "2026-08-18T06:58:01.196192+00:00",
+      "installed_at": "2026-08-26T05:41:26.279899+00:00",
       "package_name": "guardrails-grhub-profanity-free"
     },
     "guardrails/nsfw_text": {
       "import_path": "guardrails_grhub_nsfw_text",
       "exports": [
         "NSFWText"
       ],
-      "installed_at": "2026-08-18T07:03:49.839668+00:00",
+      "installed_at": "2026-08-26T05:41:36.879459+00:00",
       "package_name": "guardrails-grhub-nsfw-text"
     }
   }

backend/Dockerfile

@@ -58,22 +58,15 @@ ENV HF_HOME=/app/hf_cache
 RUN --mount=type=secret,id=HF_TOKEN \
     HF_TOKEN="$(cat /run/secrets/HF_TOKEN 2>/dev/null || true)" \
     /app/.venv/bin/python -c "from transformers import AutoTokenizer, AutoModelForSequenceClassification; \
-AutoTokenizer.from_pretrained('textdetox/xlmr-large-toxicity-classifier', cache_dir='/app/hf_cache'); \
-AutoModelForSequenceClassification.from_pretrained('textdetox/xlmr-large-toxicity-classifier', cache_dir='/app/hf_cache')"
+AutoTokenizer.from_pretrained('michellejieli/NSFW_text_classifier', cache_dir='/app/hf_cache'); \
+AutoModelForSequenceClassification.from_pretrained('michellejieli/NSFW_text_classifier', cache_dir='/app/hf_cache')"
 
 RUN --mount=type=secret,id=HF_TOKEN \
     HF_TOKEN="$(cat /run/secrets/HF_TOKEN 2>/dev/null || true)" \
     /app/.venv/bin/python -c "from transformers import AutoTokenizer, AutoModelForTokenClassification; \
 AutoTokenizer.from_pretrained('dslim/bert-base-NER-uncased', cache_dir='/app/hf_cache'); \
 AutoModelForTokenClassification.from_pretrained('dslim/bert-base-NER-uncased', cache_dir='/app/hf_cache')"
 
-# Pre-install Guardrails hub validators so container startup is not blocked by downloads
-RUN --mount=type=secret,id=GUARDRAILS_HUB_API_KEY \
-    GUARDRAILS_HUB_API_KEY="$(cat /run/secrets/GUARDRAILS_HUB_API_KEY 2>/dev/null || true)" \
-    ENABLE_METRICS="false" \
-    ENABLE_REMOTE_INFERENCING="true" \
-    /app/scripts/install_guardrails_from_hub.sh
-
 # -------------------------------
 # Entrypoint (runtime setup)
 # -------------------------------

backend/app/alembic/versions/011_rename_validator_log_request_id.py

@@ -0,0 +1,35 @@
+"""Rename validator_log.request_id to request_log_id
+
+The column is a foreign key to request_log.id (the surrogate PK), not to
+request_log.request_id (the caller-supplied business id) - the old name
+made it easy to join on the wrong column.
+
+Revision ID: 011
+Revises: 010
+Create Date: 2026-09-04 00:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "011"
+down_revision = "010"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.alter_column("validator_log", "request_id", new_column_name="request_log_id")
+    op.execute(
+        "ALTER INDEX idx_validator_log_request_id RENAME TO idx_validator_log_request_log_id"
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        "ALTER INDEX idx_validator_log_request_log_id RENAME TO idx_validator_log_request_id"
+    )
+    op.alter_column("validator_log", "request_log_id", new_column_name="request_id")

backend/app/api/deps.py

@@ -50,7 +50,7 @@ def check_source_ip(request: Request) -> None:
         if client_ip not in allowed:
             raise HTTPException(
                 status_code=status.HTTP_403_FORBIDDEN,
-                detail="Forbidden",
+                detail=f"Source IP '{client_ip}' is not permitted to access this service.",
             )
 
 

backend/app/api/routes/guardrails.py

@@ -6,6 +6,7 @@
 from fastapi import APIRouter, HTTPException
 from guardrails.guard import Guard
 from guardrails.validators import FailResult, PassResult
+from opentelemetry import trace
 from sqlmodel import Session
 
 from app.api.deps import AuthDep, SessionDep, TenantContext
@@ -44,6 +45,8 @@
 from app.utils import APIResponse, load_description
 
 logger = logging.getLogger(__name__)
+exec_logger = logging.getLogger("guardrails.execution")
+tracer = trace.get_tracer(__name__)
 
 router = APIRouter(prefix="/guardrails", tags=["guardrails"])
 
@@ -339,7 +342,31 @@ def _finalize(
 
     try:
         guard = build_guard(validators)
-        result = guard.validate(data)
+        exec_logger.info(
+            "request=%s starting guardrails execution (validators=%s) input=%r",
+            request_log_id,
+            [v.type for v in validators],
+            _strip_for_log(data),
+        )
+        # The only manual span in business code; per-validator detail lives
+        # in validator_log, joinable via kaapi.request_log_id.
+        with tracer.start_as_current_span(
+            "guardrails.validate",
+            attributes={
+                "kaapi.request_log_id": str(request_log_id),
+                "kaapi.validator_types": [v.type for v in validators],
+            },
+        ) as span:
+            result = guard.validate(data)
+            span.set_attribute("guardrails.passed", result.validated_output is not None)
+        exec_logger.info(
+            "request=%s guardrails execution finished (passed=%s) output=%r",
+            request_log_id,
+            result.validated_output is not None,
+            _strip_for_log(result.validated_output)
+            if result.validated_output is not None
+            else None,
+        )
 
         # Case 1: validation passed OR failed-with-fix (on_fail=FIX)
         if result.validated_output is not None:
@@ -407,6 +434,13 @@ def _redact_input(error_message: str, input_text: str) -> str:
     return error_message.replace(input_text, "")
 
 
+def _strip_for_log(value, limit: int = 120) -> str:
+    text = " ".join(str(value).split())
+    if len(text) > limit:
+        return f"{text[:limit]}…"
+    return text
+
+
 def _map_validator_configs(
     guard: Guard, validator_configs: list[ValidatorConfigItem] | None
 ) -> dict[str, ValidatorConfigItem]:
@@ -455,15 +489,14 @@ def add_validator_logs(
 
     config_by_alias = _map_validator_configs(guard, validator_configs)
 
+    total_steps = len(iteration.outputs.validator_logs)
+
     for order, log in enumerate(iteration.outputs.validator_logs, start=1):
         result = log.validation_result
 
         if result is None:
             continue
 
-        if suppress_pass_logs and isinstance(result, PassResult):
-            continue
-
         error_message = None
         if isinstance(result, FailResult):
             error_message = result.error_message
@@ -481,6 +514,33 @@ def add_validator_logs(
             stage = config.stage.value if config.stage else Stage.Input.value
             meta = config.model_dump(mode="json")
 
+        duration_ms = None
+        start_time = getattr(log, "start_time", None)
+        end_time = getattr(log, "end_time", None)
+        if start_time and end_time:
+            duration_ms = int((end_time - start_time).total_seconds() * 1000)
+
+        exec_logger.info(
+            "request=%s step=%d/%d validator=%s stage=%s outcome=%s "
+            "start=%s end=%s duration_ms=%s input=%r output=%r",
+            request_log_id,
+            order,
+            total_steps,
+            log.validator_name,
+            stage,
+            result.outcome,
+            start_time.isoformat() if start_time else None,
+            end_time.isoformat() if end_time else None,
+            duration_ms,
+            _strip_for_log(log.value_before_validation),
+            _strip_for_log(log.value_after_validation)
+            if log.value_after_validation is not None
+            else None,
+        )
+
+        if suppress_pass_logs and isinstance(result, PassResult):
+            continue
+
         # Verdict detail the validator attached to its result (e.g. topic
         # relevance scope_score/reasoning); stored beside the config dump.
         result_metadata = getattr(result, "metadata", None)
@@ -492,14 +552,8 @@ def add_validator_logs(
                 json.dumps(result_metadata, default=str)
             )
 
-        duration_ms = None
-        start_time = getattr(log, "start_time", None)
-        end_time = getattr(log, "end_time", None)
-        if start_time and end_time:
-            duration_ms = int((end_time - start_time).total_seconds() * 1000)
-
         validator_log = ValidatorLog(
-            request_id=request_log_id,
+            request_log_id=request_log_id,
             organization_id=auth.organization_id,
             project_id=auth.project_id,
             name=log.validator_name,

backend/app/core/config.py

@@ -49,6 +49,14 @@ class Settings(BaseSettings):
     AUTH_TOKEN: str
     PROJECT_NAME: str
     SENTRY_DSN: HttpUrl | None = None
+    SENTRY_TRACES_SAMPLE_RATE: float = 0.1
+    OTEL_ENABLED: bool = False
+    OTEL_SERVICE_NAME: str = "kaapi-guardrails"
+    # Streams app logs to CloudWatch on top of terminal/file logging.
+    # No-op unless set (see app/core/logging_config.py).
+    CLOUDWATCH_LOG_GROUP: str | None = None
+    CLOUDWATCH_LOG_STREAM: str | None = None
+    AWS_REGION: str | None = None
     POSTGRES_SERVER: str
     POSTGRES_PORT: int = 5432
     POSTGRES_USER: str

backend/app/core/logging_config.py

@@ -0,0 +1,66 @@
+import logging
+import sys
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+
+from app.core.config import settings
+
+logger = logging.getLogger(__name__)
+
+LOG_DIR = Path(__file__).resolve().parents[2] / "logs"
+LOG_FILE = LOG_DIR / "app.log"
+
+LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s %(message)s"
+
+
+def setup_logging() -> None:
+    root = logging.getLogger()
+    root.setLevel(logging.INFO)
+    root.handlers.clear()
+
+    formatter = logging.Formatter(LOG_FORMAT)
+
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setFormatter(formatter)
+    root.addHandler(stream_handler)
+
+    LOG_DIR.mkdir(parents=True, exist_ok=True)
+    file_handler = RotatingFileHandler(
+        LOG_FILE, maxBytes=10 * 1024 * 1024, backupCount=5
+    )
+    file_handler.setFormatter(formatter)
+    root.addHandler(file_handler)
+
+    if settings.CLOUDWATCH_LOG_GROUP:
+        cloudwatch_handler = _build_cloudwatch_handler(settings.CLOUDWATCH_LOG_GROUP)
+        if cloudwatch_handler is not None:
+            cloudwatch_handler.setFormatter(formatter)
+            root.addHandler(cloudwatch_handler)
+
+
+def _build_cloudwatch_handler(log_group: str) -> logging.Handler | None:
+    try:
+        import boto3
+        import watchtower
+    except ImportError:
+        logger.warning(
+            "CLOUDWATCH_LOG_GROUP is set but watchtower/boto3 aren't installed; "
+            "skipping CloudWatch log streaming."
+        )
+        return None
+
+    try:
+        boto3_client = boto3.client("logs", region_name=settings.AWS_REGION)
+        return watchtower.CloudWatchLogHandler(
+            log_group_name=log_group,
+            log_stream_name=settings.CLOUDWATCH_LOG_STREAM
+            or f"{settings.PROJECT_NAME}-{settings.ENVIRONMENT}",
+            boto3_client=boto3_client,
+            create_log_group=True,
+        )
+    except Exception:
+        logger.warning(
+            "Failed to initialize CloudWatch log handler; continuing without it.",
+            exc_info=True,
+        )
+        return None