diff --git a/api/main.py b/api/main.py index 21fddeb..f3e896e 100644 --- a/api/main.py +++ b/api/main.py @@ -1,16 +1,18 @@ -from pathlib import Path import logging from functools import lru_cache from numbers import Real +from pathlib import Path +from threading import RLock +from typing import Literal import joblib import numpy as np import pandas as pd from fastapi import FastAPI, HTTPException -from pydantic import BaseModel, ConfigDict, Field -from src.model_contracts import model_feature_order, positive_class_index -from src.artifacts import read_json +from pydantic import BaseModel, ConfigDict, Field, ValidationError +from src.artifacts import read_json +from src.model_contracts import model_feature_order, positive_class_index PROJECT_ROOT = Path(__file__).resolve().parents[1] MODEL_PATH = PROJECT_ROOT / "models" / "baseline_model.joblib" @@ -27,20 +29,57 @@ logger = logging.getLogger(__name__) +_model_lock = RLock() + + @lru_cache(maxsize=1) -def load_model(): - if not MODEL_PATH.exists(): - raise HTTPException(status_code=503, detail="Baseline model is not available") +def _load_model_version(path, signature): try: - model = joblib.load(MODEL_PATH) + model = joblib.load(path) except Exception as exc: logger.exception("Unable to load the baseline model") - raise HTTPException(status_code=503, detail="Baseline model is unreadable") from exc + raise HTTPException( + status_code=503, detail="Baseline model is unreadable" + ) from exc if not callable(getattr(model, "predict_proba", None)): - raise HTTPException(status_code=503, detail="Baseline model does not support prediction") + raise HTTPException( + status_code=503, detail="Baseline model does not support prediction" + ) return model +def load_model(): + """Reload on artifact changes while reusing unchanged models within a worker. + + Deploy new artifacts with an atomic replacement. Inode and ctime are included + so replacements preserving mtime and file size also invalidate the cache. + """ + with _model_lock: + path = Path(MODEL_PATH).absolute() + try: + stat = path.stat() + except FileNotFoundError as exc: + raise HTTPException( + status_code=503, detail="Baseline model is not available" + ) from exc + except OSError as exc: + raise HTTPException( + status_code=503, detail="Baseline model is unreadable" + ) from exc + signature = ( + stat.st_dev, + stat.st_ino, + stat.st_size, + stat.st_mtime_ns, + stat.st_ctime_ns, + ) + return _load_model_version(path, signature) + + +# Preserve the existing explicit cache-reset hook used by tests and operators. +load_model.cache_clear = _load_model_version.cache_clear + + class PredictionRequest(BaseModel): model_config = ConfigDict(extra="forbid", allow_inf_nan=False) tenure: float @@ -58,6 +97,18 @@ class PredictionBatchRequest(BaseModel): predictions: list[PredictionRequest] = Field(min_length=1, max_length=100) +class HealthReport(BaseModel): + """Required top-level contract, shared by health reads and readiness.""" + + model_config = ConfigDict(strict=True, allow_inf_nan=False) + health_score: float = Field(ge=0, le=100) + health_status: Literal["HEALTHY", "WATCH", "DEGRADED", "CRITICAL"] + metrics: dict + drift: dict + top_high_risk_drivers: dict + retraining: dict + + @app.get("/") def root(): return { @@ -69,15 +120,21 @@ def root(): @app.get("/health") def model_health(): if not HEALTH_PATH.exists(): - raise HTTPException(status_code=503, detail="Model health report is not available") + raise HTTPException( + status_code=503, detail="Model health report is not available" + ) try: health = read_json(HEALTH_PATH) except (OSError, ValueError) as exc: raise HTTPException( status_code=503, detail="Model health report is unreadable" ) from exc - if not isinstance(health, dict): - raise HTTPException(status_code=503, detail="Invalid model health report") + try: + HealthReport.model_validate(health) + except ValidationError as exc: + raise HTTPException( + status_code=503, detail="Invalid model health report" + ) from exc return health @@ -85,7 +142,9 @@ def model_health(): @app.get("/metrics") def baseline_metrics(): if not METRICS_PATH.exists(): - raise HTTPException(status_code=503, detail="Baseline metrics are not available") + raise HTTPException( + status_code=503, detail="Baseline metrics are not available" + ) try: metrics = read_json(METRICS_PATH) except (OSError, ValueError) as exc: @@ -113,14 +172,7 @@ def readiness(): missing = [str(path) for path in (MODEL_PATH, HEALTH_PATH) if not path.exists()] if missing: raise HTTPException(status_code=503, detail={"missing_artifacts": missing}) - try: - report = read_json(HEALTH_PATH) - except (OSError, ValueError) as exc: - raise HTTPException( - status_code=503, detail="Model health report is unreadable" - ) from exc - if not isinstance(report, dict): - raise HTTPException(status_code=503, detail="Invalid model health report") + model_health() load_model() return {"status": "ready"} diff --git a/tests/test_api_model_loading.py b/tests/test_api_model_loading.py index b611e64..82b3a65 100644 --- a/tests/test_api_model_loading.py +++ b/tests/test_api_model_loading.py @@ -1,3 +1,6 @@ +import json +from pathlib import Path + import joblib import pytest from fastapi.testclient import TestClient @@ -12,11 +15,14 @@ def clear_model_cache(): main.load_model.cache_clear() -@pytest.mark.parametrize("path_kind, detail", [ - ("corrupt", "Baseline model is unreadable"), - ("directory", "Baseline model is unreadable"), - ("wrong_object", "Baseline model does not support prediction"), -]) +@pytest.mark.parametrize( + "path_kind, detail", + [ + ("corrupt", "Baseline model is unreadable"), + ("directory", "Baseline model is unreadable"), + ("wrong_object", "Baseline model does not support prediction"), + ], +) def test_invalid_artifacts_return_503_for_prediction_and_readiness( tmp_path, monkeypatch, path_kind, detail ): @@ -28,7 +34,16 @@ def test_invalid_artifacts_return_503_for_prediction_and_readiness( else: model_path.write_bytes(b"corrupt artifact") report = tmp_path / "health.json" - report.write_text("{}", encoding="utf-8") + report.write_text( + json.dumps( + json.loads( + ( + Path(main.PROJECT_ROOT) / "models/model_health_report.json" + ).read_text() + ) + ), + encoding="utf-8", + ) monkeypatch.setattr(main, "MODEL_PATH", model_path) monkeypatch.setattr(main, "HEALTH_PATH", report) client = TestClient(main.app) diff --git a/tests/test_health_schema.py b/tests/test_health_schema.py new file mode 100644 index 0000000..dd1df9e --- /dev/null +++ b/tests/test_health_schema.py @@ -0,0 +1,77 @@ +import json + +import pytest +from fastapi.testclient import TestClient + +from api import main + + +@pytest.fixture +def valid_report(): + return { + "health_score": 90.0, + "health_status": "HEALTHY", + "metrics": {}, + "drift": {}, + "top_high_risk_drivers": {}, + "retraining": {}, + } + + +@pytest.fixture +def client(tmp_path, monkeypatch): + model = tmp_path / "model.joblib" + model.touch() + monkeypatch.setattr(main, "MODEL_PATH", model) + monkeypatch.setattr(main, "HEALTH_PATH", tmp_path / "health.json") + monkeypatch.setattr(main, "load_model", lambda: object()) + return TestClient(main.app) + + +@pytest.mark.parametrize( + "field", + [ + "health_score", + "health_status", + "metrics", + "drift", + "top_high_risk_drivers", + "retraining", + ], +) +@pytest.mark.parametrize("route", ["/health", "/ready"]) +def test_missing_health_fields_are_not_ready(client, valid_report, field, route): + del valid_report[field] + main.HEALTH_PATH.write_text(json.dumps(valid_report), encoding="utf-8") + assert client.get(route).status_code == 503 + + +@pytest.mark.parametrize( + "field,value", + [ + ("health_score", True), + ("health_score", "90"), + ("health_score", -1), + ("health_score", 101), + ("health_status", "UNKNOWN"), + ("health_status", []), + ("metrics", []), + ("drift", None), + ("top_high_risk_drivers", 3), + ("retraining", False), + ], +) +@pytest.mark.parametrize("route", ["/health", "/ready"]) +def test_invalid_health_field_types_are_rejected( + client, valid_report, field, value, route +): + valid_report[field] = value + main.HEALTH_PATH.write_text(json.dumps(valid_report), encoding="utf-8") + assert client.get(route).status_code == 503 + + +def test_valid_health_report_keeps_optional_fields(client, valid_report): + valid_report["scoring_note"] = "Review recommended" + main.HEALTH_PATH.write_text(json.dumps(valid_report), encoding="utf-8") + assert client.get("/health").json() == valid_report + assert client.get("/ready").json() == {"status": "ready"} diff --git a/tests/test_model_reload.py b/tests/test_model_reload.py new file mode 100644 index 0000000..7eea7db --- /dev/null +++ b/tests/test_model_reload.py @@ -0,0 +1,75 @@ +import os + +import joblib +import pytest +from fastapi.testclient import TestClient +from sklearn.dummy import DummyClassifier + +from api import main + + +@pytest.fixture +def runtime(tmp_path, monkeypatch): + path = tmp_path / "model.joblib" + monkeypatch.setattr(main, "MODEL_PATH", path) + main.load_model.cache_clear() + yield path, TestClient(main.app) + main.load_model.cache_clear() + + +def save_model(path, prediction): + model = DummyClassifier(strategy="constant", constant=prediction) + model.fit([[0] * 8, [1] * 8], [0, 1]) + joblib.dump(model, path) + + +def test_predictions_reload_after_atomic_replacement(runtime, monkeypatch): + path, client = runtime + save_model(path, 0) + original_load = main.joblib.load + loads = [] + + def counted_load(*args, **kwargs): + loads.append(1) + return original_load(*args, **kwargs) + + monkeypatch.setattr(main.joblib, "load", counted_load) + payload = dict.fromkeys(main.PredictionRequest.model_fields, 1.0) + for _ in range(2): + assert client.post("/predict", json=payload).json()["prediction"] == 0 + assert len(loads) == 1 + + replacement = path.with_name("replacement.joblib") + save_model(replacement, 1) + # Replacement must be detected even when deployment preserves timestamps. + timestamp = path.stat().st_mtime_ns + os.utime(replacement, ns=(timestamp, timestamp)) + os.replace(replacement, path) + for _ in range(2): + assert client.post("/predict", json=payload).json()["prediction"] == 1 + assert len(loads) == 2 + + +@pytest.mark.parametrize("replacement", ["corrupt", "deleted"]) +def test_cached_model_does_not_hide_invalid_replacement(runtime, replacement): + path, client = runtime + save_model(path, 0) + payload = dict.fromkeys(main.PredictionRequest.model_fields, 1.0) + assert client.post("/predict", json=payload).status_code == 200 + if replacement == "corrupt": + path.write_bytes(b"broken model") + else: + path.unlink() + assert client.post("/predict", json=payload).status_code == 503 + save_model(path, 1) + assert client.post("/predict", json=payload).json()["prediction"] == 1 + + +def test_cache_identity_includes_model_path(runtime, monkeypatch): + path, _ = runtime + save_model(path, 0) + first = main.load_model() + other = path.with_name("other.joblib") + save_model(other, 1) + monkeypatch.setattr(main, "MODEL_PATH", other) + assert main.load_model() is not first diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 668e665..55c7484 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -1,5 +1,8 @@ -from fastapi.testclient import TestClient +import json +from pathlib import Path + import joblib +from fastapi.testclient import TestClient from sklearn.dummy import DummyClassifier from api import main @@ -24,7 +27,16 @@ def test_readiness_succeeds_when_required_artifacts_exist(tmp_path, monkeypatch) model_path = tmp_path / "model.joblib" health_path = tmp_path / "health.json" joblib.dump(DummyClassifier().fit([[0], [1]], [0, 1]), model_path) - health_path.write_text("{}", encoding="utf-8") + health_path.write_text( + json.dumps( + json.loads( + ( + Path(main.PROJECT_ROOT) / "models/model_health_report.json" + ).read_text() + ) + ), + encoding="utf-8", + ) monkeypatch.setattr(main, "MODEL_PATH", model_path) monkeypatch.setattr(main, "HEALTH_PATH", health_path)