Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 74 additions & 22 deletions api/main.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
from pathlib import Path
import logging
from functools import lru_cache
from numbers import Real
from pathlib import Path
from threading import RLock
from typing import Literal

import joblib
import numpy as np
import pandas as pd
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel, ConfigDict, Field
from src.model_contracts import model_feature_order, positive_class_index
from src.artifacts import read_json
from pydantic import BaseModel, ConfigDict, Field, ValidationError

from src.artifacts import read_json
from src.model_contracts import model_feature_order, positive_class_index

PROJECT_ROOT = Path(__file__).resolve().parents[1]
MODEL_PATH = PROJECT_ROOT / "models" / "baseline_model.joblib"
Expand All @@ -27,20 +29,57 @@
logger = logging.getLogger(__name__)


_model_lock = RLock()


@lru_cache(maxsize=1)
def load_model():
if not MODEL_PATH.exists():
raise HTTPException(status_code=503, detail="Baseline model is not available")
def _load_model_version(path, signature):
try:
model = joblib.load(MODEL_PATH)
model = joblib.load(path)
except Exception as exc:
logger.exception("Unable to load the baseline model")
raise HTTPException(status_code=503, detail="Baseline model is unreadable") from exc
raise HTTPException(
status_code=503, detail="Baseline model is unreadable"
) from exc
if not callable(getattr(model, "predict_proba", None)):
raise HTTPException(status_code=503, detail="Baseline model does not support prediction")
raise HTTPException(
status_code=503, detail="Baseline model does not support prediction"
)
return model


def load_model():
"""Reload on artifact changes while reusing unchanged models within a worker.

Deploy new artifacts with an atomic replacement. Inode and ctime are included
so replacements preserving mtime and file size also invalidate the cache.
"""
with _model_lock:
path = Path(MODEL_PATH).absolute()
try:
stat = path.stat()
except FileNotFoundError as exc:
raise HTTPException(
status_code=503, detail="Baseline model is not available"
) from exc
except OSError as exc:
raise HTTPException(
status_code=503, detail="Baseline model is unreadable"
) from exc
signature = (
stat.st_dev,
stat.st_ino,
stat.st_size,
stat.st_mtime_ns,
stat.st_ctime_ns,
)
return _load_model_version(path, signature)


# Preserve the existing explicit cache-reset hook used by tests and operators.
load_model.cache_clear = _load_model_version.cache_clear


class PredictionRequest(BaseModel):
model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
tenure: float
Expand All @@ -58,6 +97,18 @@ class PredictionBatchRequest(BaseModel):
predictions: list[PredictionRequest] = Field(min_length=1, max_length=100)


class HealthReport(BaseModel):
"""Required top-level contract, shared by health reads and readiness."""

model_config = ConfigDict(strict=True, allow_inf_nan=False)
health_score: float = Field(ge=0, le=100)
health_status: Literal["HEALTHY", "WATCH", "DEGRADED", "CRITICAL"]
metrics: dict
drift: dict
top_high_risk_drivers: dict
retraining: dict


@app.get("/")
def root():
return {
Expand All @@ -69,23 +120,31 @@ def root():
@app.get("/health")
def model_health():
if not HEALTH_PATH.exists():
raise HTTPException(status_code=503, detail="Model health report is not available")
raise HTTPException(
status_code=503, detail="Model health report is not available"
)
try:
health = read_json(HEALTH_PATH)
except (OSError, ValueError) as exc:
raise HTTPException(
status_code=503, detail="Model health report is unreadable"
) from exc
if not isinstance(health, dict):
raise HTTPException(status_code=503, detail="Invalid model health report")
try:
HealthReport.model_validate(health)
except ValidationError as exc:
raise HTTPException(
status_code=503, detail="Invalid model health report"
) from exc

return health


@app.get("/metrics")
def baseline_metrics():
if not METRICS_PATH.exists():
raise HTTPException(status_code=503, detail="Baseline metrics are not available")
raise HTTPException(
status_code=503, detail="Baseline metrics are not available"
)
try:
metrics = read_json(METRICS_PATH)
except (OSError, ValueError) as exc:
Expand Down Expand Up @@ -113,14 +172,7 @@ def readiness():
missing = [str(path) for path in (MODEL_PATH, HEALTH_PATH) if not path.exists()]
if missing:
raise HTTPException(status_code=503, detail={"missing_artifacts": missing})
try:
report = read_json(HEALTH_PATH)
except (OSError, ValueError) as exc:
raise HTTPException(
status_code=503, detail="Model health report is unreadable"
) from exc
if not isinstance(report, dict):
raise HTTPException(status_code=503, detail="Invalid model health report")
model_health()
load_model()
return {"status": "ready"}

Expand Down
27 changes: 21 additions & 6 deletions tests/test_api_model_loading.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
import json
from pathlib import Path

import joblib
import pytest
from fastapi.testclient import TestClient
Expand All @@ -12,11 +15,14 @@ def clear_model_cache():
main.load_model.cache_clear()


@pytest.mark.parametrize("path_kind, detail", [
("corrupt", "Baseline model is unreadable"),
("directory", "Baseline model is unreadable"),
("wrong_object", "Baseline model does not support prediction"),
])
@pytest.mark.parametrize(
"path_kind, detail",
[
("corrupt", "Baseline model is unreadable"),
("directory", "Baseline model is unreadable"),
("wrong_object", "Baseline model does not support prediction"),
],
)
def test_invalid_artifacts_return_503_for_prediction_and_readiness(
tmp_path, monkeypatch, path_kind, detail
):
Expand All @@ -28,7 +34,16 @@ def test_invalid_artifacts_return_503_for_prediction_and_readiness(
else:
model_path.write_bytes(b"corrupt artifact")
report = tmp_path / "health.json"
report.write_text("{}", encoding="utf-8")
report.write_text(
json.dumps(
json.loads(
(
Path(main.PROJECT_ROOT) / "models/model_health_report.json"
).read_text()
)
),
encoding="utf-8",
)
monkeypatch.setattr(main, "MODEL_PATH", model_path)
monkeypatch.setattr(main, "HEALTH_PATH", report)
client = TestClient(main.app)
Expand Down
77 changes: 77 additions & 0 deletions tests/test_health_schema.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import json

import pytest
from fastapi.testclient import TestClient

from api import main


@pytest.fixture
def valid_report():
return {
"health_score": 90.0,
"health_status": "HEALTHY",
"metrics": {},
"drift": {},
"top_high_risk_drivers": {},
"retraining": {},
}


@pytest.fixture
def client(tmp_path, monkeypatch):
model = tmp_path / "model.joblib"
model.touch()
monkeypatch.setattr(main, "MODEL_PATH", model)
monkeypatch.setattr(main, "HEALTH_PATH", tmp_path / "health.json")
monkeypatch.setattr(main, "load_model", lambda: object())
return TestClient(main.app)


@pytest.mark.parametrize(
"field",
[
"health_score",
"health_status",
"metrics",
"drift",
"top_high_risk_drivers",
"retraining",
],
)
@pytest.mark.parametrize("route", ["/health", "/ready"])
def test_missing_health_fields_are_not_ready(client, valid_report, field, route):
del valid_report[field]
main.HEALTH_PATH.write_text(json.dumps(valid_report), encoding="utf-8")
assert client.get(route).status_code == 503


@pytest.mark.parametrize(
"field,value",
[
("health_score", True),
("health_score", "90"),
("health_score", -1),
("health_score", 101),
("health_status", "UNKNOWN"),
("health_status", []),
("metrics", []),
("drift", None),
("top_high_risk_drivers", 3),
("retraining", False),
],
)
@pytest.mark.parametrize("route", ["/health", "/ready"])
def test_invalid_health_field_types_are_rejected(
client, valid_report, field, value, route
):
valid_report[field] = value
main.HEALTH_PATH.write_text(json.dumps(valid_report), encoding="utf-8")
assert client.get(route).status_code == 503


def test_valid_health_report_keeps_optional_fields(client, valid_report):
valid_report["scoring_note"] = "Review recommended"
main.HEALTH_PATH.write_text(json.dumps(valid_report), encoding="utf-8")
assert client.get("/health").json() == valid_report
assert client.get("/ready").json() == {"status": "ready"}
75 changes: 75 additions & 0 deletions tests/test_model_reload.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import os

import joblib
import pytest
from fastapi.testclient import TestClient
from sklearn.dummy import DummyClassifier

from api import main


@pytest.fixture
def runtime(tmp_path, monkeypatch):
path = tmp_path / "model.joblib"
monkeypatch.setattr(main, "MODEL_PATH", path)
main.load_model.cache_clear()
yield path, TestClient(main.app)
main.load_model.cache_clear()


def save_model(path, prediction):
model = DummyClassifier(strategy="constant", constant=prediction)
model.fit([[0] * 8, [1] * 8], [0, 1])
joblib.dump(model, path)


def test_predictions_reload_after_atomic_replacement(runtime, monkeypatch):
path, client = runtime
save_model(path, 0)
original_load = main.joblib.load
loads = []

def counted_load(*args, **kwargs):
loads.append(1)
return original_load(*args, **kwargs)

monkeypatch.setattr(main.joblib, "load", counted_load)
payload = dict.fromkeys(main.PredictionRequest.model_fields, 1.0)
for _ in range(2):
assert client.post("/predict", json=payload).json()["prediction"] == 0
assert len(loads) == 1

replacement = path.with_name("replacement.joblib")
save_model(replacement, 1)
# Replacement must be detected even when deployment preserves timestamps.
timestamp = path.stat().st_mtime_ns
os.utime(replacement, ns=(timestamp, timestamp))
os.replace(replacement, path)
for _ in range(2):
assert client.post("/predict", json=payload).json()["prediction"] == 1
assert len(loads) == 2


@pytest.mark.parametrize("replacement", ["corrupt", "deleted"])
def test_cached_model_does_not_hide_invalid_replacement(runtime, replacement):
path, client = runtime
save_model(path, 0)
payload = dict.fromkeys(main.PredictionRequest.model_fields, 1.0)
assert client.post("/predict", json=payload).status_code == 200
if replacement == "corrupt":
path.write_bytes(b"broken model")
else:
path.unlink()
assert client.post("/predict", json=payload).status_code == 503
save_model(path, 1)
assert client.post("/predict", json=payload).json()["prediction"] == 1


def test_cache_identity_includes_model_path(runtime, monkeypatch):
path, _ = runtime
save_model(path, 0)
first = main.load_model()
other = path.with_name("other.joblib")
save_model(other, 1)
monkeypatch.setattr(main, "MODEL_PATH", other)
assert main.load_model() is not first
16 changes: 14 additions & 2 deletions tests/test_readiness.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
from fastapi.testclient import TestClient
import json
from pathlib import Path

import joblib
from fastapi.testclient import TestClient
from sklearn.dummy import DummyClassifier

from api import main
Expand All @@ -24,7 +27,16 @@ def test_readiness_succeeds_when_required_artifacts_exist(tmp_path, monkeypatch)
model_path = tmp_path / "model.joblib"
health_path = tmp_path / "health.json"
joblib.dump(DummyClassifier().fit([[0], [1]], [0, 1]), model_path)
health_path.write_text("{}", encoding="utf-8")
health_path.write_text(
json.dumps(
json.loads(
(
Path(main.PROJECT_ROOT) / "models/model_health_report.json"
).read_text()
)
),
encoding="utf-8",
)
monkeypatch.setattr(main, "MODEL_PATH", model_path)
monkeypatch.setattr(main, "HEALTH_PATH", health_path)

Expand Down
Loading