diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..cbf75af --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + cache: pip + - name: Install test dependencies + run: pip install fastapi pydantic-settings sqlalchemy loguru pytest + - name: Run regression tests + run: python -m pytest tests -q + - name: Parse PowerShell scripts + shell: pwsh + run: | + $errors = @() + Get-ChildItem -Recurse -Include *.ps1,*.psm1 | ForEach-Object { + $tokens = $null + $parseErrors = $null + [System.Management.Automation.Language.Parser]::ParseFile($_.FullName, [ref]$tokens, [ref]$parseErrors) > $null + $errors += $parseErrors + } + if ($errors.Count -gt 0) { + $errors | Format-List + exit 1 + } diff --git a/.gitignore b/.gitignore index 752d5af..f04fadd 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ data/storage/LegalAssociatesLLP/2024-09-08/4f380a04-02af-4a46-8b3d-f39bae42e9da_ data/storage/PropertyManagementInc/2025-01-27/53b571aa-a79f-4be7-b867-7f0c258f39c8_2025-01-27_Property_Management_Inc_Medical Record.pdf data/storage/TechStartGmbH/2025-06-09/75d86917-31e1-4a2c-b5c4-3433f9841d1c_2025-06-09_TechStart_GmbH_Contract.pdf .DS_Store +# Python bytecode and test caches +__pycache__/ +*.py[cod] +.pytest_cache/ diff --git a/Dockerfile b/Dockerfile index 7cdb018..4a99ae2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -90,7 +90,8 @@ ENV PYTHONUNBUFFERED=1 \ # Application settings (should be overridden in production) DATABASE_URL=sqlite:///./data/documents.db \ SECRET_KEY=MUST-BE-SET-IN-PRODUCTION \ - AI_PROVIDER=openai + AI_PROVIDER=openai \ + TRUSTED_PROXY_IPS=127.0.0.1 # Add metadata labels LABEL maintainer="Document Manager Team" \ @@ -106,4 +107,4 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ CMD curl -f http://localhost:8000/api/health || exit 1 # Use entrypoint script for initialization -ENTRYPOINT ["/app/docker-entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["/app/docker-entrypoint.sh"] diff --git a/app/__init__.py b/app/__init__.py index 2ddc165..37b1d2d 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -4,7 +4,7 @@ # Configure loguru to write logs to the logs directory log_dir = Path(__file__).parent.parent / "data" / "logs" -log_dir.mkdir(exist_ok=True) +log_dir.mkdir(parents=True, exist_ok=True) # Remove default logger logger.remove() diff --git a/app/config.py b/app/config.py index 772dbf0..78edfed 100644 --- a/app/config.py +++ b/app/config.py @@ -24,13 +24,6 @@ class Settings(BaseSettings): azure_openai_chat_deployment: str = "" azure_openai_embeddings_deployment: str = "" - def __init__(self, **kwargs): - # Don't use any values from kwargs that might come from env vars - # Only use explicitly passed values (which should be none for base Settings) - # Filter out any kwargs that might come from env vars - filtered_kwargs = {k: v for k, v in kwargs.items() if not k.startswith('NEVER_MATCH_THIS_PREFIX_')} - super().__init__(**filtered_kwargs) - # ChromaDB chroma_host: str = "localhost" chroma_port: int = 8001 @@ -61,7 +54,10 @@ def __init__(self, **kwargs): jwt_secret_key: Optional[str] = None # JWT secret key for authentication algorithm: str = "HS256" access_token_expire_minutes: int = 30 + environment: str = "development" production_mode: bool = False # Set to True in production for secure cookies + cors_origins: str = "http://localhost:3000,http://localhost:8000,http://127.0.0.1:8000" + trusted_proxy_ips: str = "127.0.0.1,::1" # Logging log_level: str = "INFO" @@ -73,21 +69,31 @@ def __init__(self, **kwargs): ai_max_retries: int = 2 # Maximum number of retries for failed AI requests model_config = SettingsConfigDict( - # No env_file - all settings come from database or defaults + # Runtime values may be supplied by Docker/Kubernetes environment + # variables. Database-backed settings still override these defaults. case_sensitive=False, - # Explicitly disable reading from environment variables env_file=None, - # Don't read from environment variables at all - # This ensures settings only come from database or defaults env_ignore_empty=True, - # This is the key setting to disable env vars completely - # By setting a prefix that will never match, we prevent env var loading - env_prefix="NEVER_MATCH_THIS_PREFIX_" + env_prefix="" ) + def model_post_init(self, __context: Any) -> None: + # Keep the existing ENVIRONMENT=production setup contract while still + # allowing an explicit PRODUCTION_MODE value to override it. + if "production_mode" not in self.model_fields_set: + self.production_mode = self.environment.lower() == "production" + @property def allowed_extensions_list(self) -> list: return [ext.strip().lower() for ext in self.allowed_extensions.split(",")] + + @property + def cors_origins_list(self) -> list[str]: + return [origin.strip() for origin in self.cors_origins.split(",") if origin.strip()] + + @property + def trusted_proxy_ips_list(self) -> list[str]: + return [ip.strip() for ip in self.trusted_proxy_ips.split(",") if ip.strip()] @property def max_file_size_bytes(self) -> int: @@ -107,8 +113,8 @@ class DatabaseSettings(Settings): """Settings that loads configuration from database""" def __init__(self, db: Session = None, **kwargs): - # First, load defaults WITHOUT environment variables - # We pass _env_file=None to ensure no env vars are loaded + # Load defaults and runtime environment variables first. Persisted + # application settings take precedence when a database is available. super().__init__(_env_file=None, **kwargs) # Then override with database values if available @@ -188,4 +194,3 @@ def reset_settings(): """Reset the global settings instance""" global _settings _settings = None - diff --git a/app/database.py b/app/database.py index a573d3f..0652cae 100644 --- a/app/database.py +++ b/app/database.py @@ -3,9 +3,10 @@ from sqlalchemy.orm import sessionmaker, Session from loguru import logger import os +from .config import get_settings -# Use simple SQLite database with default path -DATABASE_URL = "sqlite:///./data/documents.db" +# Honour the documented DATABASE_URL runtime setting. +DATABASE_URL = get_settings().database_url # Ensure data directory exists os.makedirs("data", exist_ok=True) @@ -13,7 +14,7 @@ # Create engine with minimal configuration engine = create_engine( DATABASE_URL, - connect_args={"check_same_thread": False} + connect_args={"check_same_thread": False} if DATABASE_URL.startswith("sqlite") else {} ) # Create session factory diff --git a/app/main.py b/app/main.py index 6d72a18..84f5768 100644 --- a/app/main.py +++ b/app/main.py @@ -29,10 +29,12 @@ version="1.0.0" ) +app_settings = get_settings() + # CORS middleware app.add_middleware( CORSMiddleware, - allow_origins=["http://localhost:3000", "http://localhost:8000", "http://127.0.0.1:8000"], # Restrict origins + allow_origins=app_settings.cors_origins_list, allow_credentials=True, allow_methods=["GET", "POST", "PUT", "DELETE"], allow_headers=["Content-Type", "Authorization", "X-CSRF-Token"], # Restrict headers @@ -44,9 +46,6 @@ app.add_exception_handler(StarletteHTTPException, ErrorHandler.starlette_exception_handler) app.add_exception_handler(Exception, ErrorHandler.general_exception_handler) -# Get settings to determine production mode -app_settings = get_settings() - # Enable CSRF protection csrf_protect = CSRFProtect( secure=app_settings.production_mode, # Use secure cookies in production @@ -71,7 +70,8 @@ default_limit=100, # 100 requests per minute for general endpoints window_seconds=60, login_limit=5, # 5 login attempts per 5 minutes - login_window_seconds=300 + login_window_seconds=300, + trusted_proxy_ips=app_settings.trusted_proxy_ips_list, ) rate_limit.init_app(app) diff --git a/app/middleware/rate_limit_middleware.py b/app/middleware/rate_limit_middleware.py index 866cf7a..a9419d6 100644 --- a/app/middleware/rate_limit_middleware.py +++ b/app/middleware/rate_limit_middleware.py @@ -2,7 +2,7 @@ Rate limiting middleware for FastAPI to prevent brute force attacks and API abuse. """ import time -from typing import Dict, Optional, Tuple +from typing import Dict, Iterable, Optional, Tuple from collections import defaultdict from fastapi import Request from fastapi.responses import JSONResponse @@ -29,7 +29,8 @@ def __init__( window_seconds: int = 60, # 1 minute window login_limit: int = 5, # stricter limit for login attempts login_window_seconds: int = 300, # 5 minute window for login - cleanup_interval: int = 300 # cleanup every 5 minutes + cleanup_interval: int = 300, # cleanup every 5 minutes + trusted_proxy_ips: Optional[Iterable[str]] = None, ): super().__init__(app) self.default_limit = default_limit @@ -37,6 +38,7 @@ def __init__( self.login_limit = login_limit self.login_window_seconds = login_window_seconds self.cleanup_interval = cleanup_interval + self.trusted_proxy_ips = set(trusted_proxy_ips or ()) # Store request counts: {ip: {endpoint: [(timestamp, count)]}} self.request_counts: Dict[str, Dict[str, list]] = defaultdict(lambda: defaultdict(list)) @@ -81,19 +83,21 @@ async def _cleanup_old_entries(self): def get_client_ip(self, request: Request) -> str: """Extract client IP address from request.""" - # Check for proxy headers - forwarded_for = request.headers.get("X-Forwarded-For") - if forwarded_for: - # Take the first IP in the chain - return forwarded_for.split(",")[0].strip() - - # Check for other proxy headers - real_ip = request.headers.get("X-Real-IP") - if real_ip: - return real_ip - - # Fallback to direct connection - return request.client.host if request.client else "unknown" + direct_ip = request.client.host if request.client else "unknown" + + # Forwarding headers are attacker-controlled unless the direct peer is + # a configured reverse proxy. Never let an arbitrary client choose the + # rate-limit bucket used for its request. + if direct_ip in self.trusted_proxy_ips: + forwarded_for = request.headers.get("X-Forwarded-For") + if forwarded_for: + return forwarded_for.split(",")[0].strip() + + real_ip = request.headers.get("X-Real-IP") + if real_ip: + return real_ip.strip() + + return direct_ip def get_rate_limit(self, path: str) -> Tuple[int, int]: """Get rate limit for a specific path.""" @@ -212,4 +216,4 @@ def __init__(self, app=None, **kwargs): def init_app(self, app): """Initialize rate limiting for the FastAPI app.""" # Add rate limit middleware - app.add_middleware(RateLimitMiddleware, **self.config) \ No newline at end of file + app.add_middleware(RateLimitMiddleware, **self.config) diff --git a/frontend/login.html b/frontend/login.html index df61331..02fb414 100644 --- a/frontend/login.html +++ b/frontend/login.html @@ -378,7 +378,10 @@

Admin User Setup

- \ No newline at end of file + diff --git a/setup.ps1 b/setup.ps1 index 897cbfa..c5ea698 100644 --- a/setup.ps1 +++ b/setup.ps1 @@ -32,11 +32,18 @@ function Require-Docker() { function New-EnvFile() { if (-not (Test-Path .env)) { Info 'Creating .env file...' - $secret = try { python - <<'PY' -import secrets -print(secrets.token_urlsafe(32)) -PY - } catch { 'change-me-in-production' } + $secret = 'change-me-in-production' + $pythonCommand = Get-Command python3, python -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($pythonCommand) { + try { + $generatedSecret = & $pythonCommand.Source -c 'import secrets; print(secrets.token_urlsafe(32))' + if ($LASTEXITCODE -eq 0 -and $generatedSecret) { + $secret = $generatedSecret.Trim() + } + } catch { + Warn 'Could not generate a random secret with Python; update SECRET_KEY manually.' + } + } @" # Security - CHANGE THIS IN PRODUCTION! @@ -94,22 +101,23 @@ function Invoke-Prod() { exit 1 } - $env = Get-Content .env | Where-Object { $_ -notmatch '^#' -and $_.Trim() } - foreach ($line in $env) { + $envValues = @{} + $envLines = Get-Content .env | Where-Object { $_ -notmatch '^#' -and $_.Trim() } + foreach ($line in $envLines) { $kv = $line.Split('=',2) - if ($kv.Length -eq 2) { $env:$($kv[0]) = $kv[1] } + if ($kv.Length -eq 2) { $envValues[$kv[0].Trim()] = $kv[1] } } $pwdPath = (Get-Location).Path docker run -d ` --name "$ContainerName" ` -p 8000:8000 ` - -e SECRET_KEY="$env:SECRET_KEY" ` - -e DATABASE_URL="$env:DATABASE_URL" ` - -e AI_PROVIDER="$env:AI_PROVIDER" ` - -e OPENAI_API_KEY="$env:OPENAI_API_KEY" ` - -e ENVIRONMENT="$env:ENVIRONMENT" ` - -e LOG_LEVEL="$env:LOG_LEVEL" ` + -e "SECRET_KEY=$($envValues['SECRET_KEY'])" ` + -e "DATABASE_URL=$($envValues['DATABASE_URL'])" ` + -e "AI_PROVIDER=$($envValues['AI_PROVIDER'])" ` + -e "OPENAI_API_KEY=$($envValues['OPENAI_API_KEY'])" ` + -e "ENVIRONMENT=$($envValues['ENVIRONMENT'])" ` + -e "LOG_LEVEL=$($envValues['LOG_LEVEL'])" ` -v "$pwdPath/data:/app/data" ` -v "$pwdPath/staging:/app/staging" ` -v "$pwdPath/storage:/app/storage" ` @@ -176,4 +184,3 @@ Examples: "@ } } - diff --git a/supervisord.conf b/supervisord.conf index 40d821d..df41bd1 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -14,7 +14,7 @@ stderr_logfile=/app/data/logs/chromadb.stderr.log environment=IS_PERSISTENT="TRUE",ANONYMIZED_TELEMETRY="FALSE",ALLOW_RESET="FALSE" [program:documentmanager] -command=python -m uvicorn app.main:app --host 0.0.0.0 --port 8000 --proxy-headers --forwarded-allow-ips='*' +command=python -m uvicorn app.main:app --host 0.0.0.0 --port 8000 --proxy-headers --forwarded-allow-ips=%(ENV_TRUSTED_PROXY_IPS)s directory=/app autostart=true autorestart=true @@ -35,4 +35,4 @@ chmod=0700 supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface [supervisorctl] -serverurl=unix:///tmp/supervisor.sock \ No newline at end of file +serverurl=unix:///tmp/supervisor.sock diff --git a/tests/test_runtime_configuration.py b/tests/test_runtime_configuration.py new file mode 100644 index 0000000..66519c6 --- /dev/null +++ b/tests/test_runtime_configuration.py @@ -0,0 +1,40 @@ +from types import SimpleNamespace + +from app.config import Settings +from app.middleware.rate_limit_middleware import RateLimitMiddleware + + +def test_settings_read_documented_environment_variables(monkeypatch): + monkeypatch.setenv("DATABASE_URL", "sqlite:///./data/test.db") + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + monkeypatch.setenv("CORS_ORIGINS", "https://documents.example.com") + monkeypatch.setenv("ENVIRONMENT", "production") + + settings = Settings() + + assert settings.database_url == "sqlite:///./data/test.db" + assert settings.openai_api_key == "test-key" + assert settings.cors_origins_list == ["https://documents.example.com"] + assert settings.production_mode is True + + +def test_forwarded_header_is_ignored_for_untrusted_client(): + middleware = RateLimitMiddleware.__new__(RateLimitMiddleware) + middleware.trusted_proxy_ips = {"127.0.0.1"} + request = SimpleNamespace( + client=SimpleNamespace(host="203.0.113.10"), + headers={"X-Forwarded-For": "198.51.100.7"}, + ) + + assert middleware.get_client_ip(request) == "203.0.113.10" + + +def test_forwarded_header_is_used_for_trusted_proxy(): + middleware = RateLimitMiddleware.__new__(RateLimitMiddleware) + middleware.trusted_proxy_ips = {"127.0.0.1"} + request = SimpleNamespace( + client=SimpleNamespace(host="127.0.0.1"), + headers={"X-Forwarded-For": "198.51.100.7, 127.0.0.1"}, + ) + + assert middleware.get_client_ip(request) == "198.51.100.7"