diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..1fa256ca --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +.venv +.pytest_cache +.ruff_cache +**/__pycache__ +ui/node_modules +ui/dist +deployment/dev +docs_jupyter +local_cache +*.h5 diff --git a/.github/workflows/deploy-api-dev.yml b/.github/workflows/deploy-api-dev.yml index 8a494572..b0552b36 100644 --- a/.github/workflows/deploy-api-dev.yml +++ b/.github/workflows/deploy-api-dev.yml @@ -60,7 +60,9 @@ jobs: REPOSITORY: dev1-bedhost IMAGE_TAG: ${{ github.sha }} run: | - docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG -f deployment/Dockerfiles/${{ env.server }}.Dockerfile . + docker build -t bedhost-base:$IMAGE_TAG -f dev.Dockerfile . + docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG --build-arg BASE_IMAGE=bedhost-base:$IMAGE_TAG -f deployment/Dockerfiles/${{ env.server }}.Dockerfile . + ./tests/scripts/container-smoke.sh $REGISTRY/$REPOSITORY:$IMAGE_TAG docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG echo "image=$REGISTRY/$REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT diff --git a/.github/workflows/deploy-api.yml b/.github/workflows/deploy-api.yml index 97226176..46c6b617 100644 --- a/.github/workflows/deploy-api.yml +++ b/.github/workflows/deploy-api.yml @@ -59,7 +59,9 @@ jobs: REPOSITORY: bedhost IMAGE_TAG: ${{ github.sha }} run: | - docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG -f deployment/Dockerfiles/${{ env.server }}.Dockerfile . + docker build -t bedhost-base:$IMAGE_TAG -f Dockerfile . + docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG --build-arg BASE_IMAGE=bedhost-base:$IMAGE_TAG -f deployment/Dockerfiles/${{ env.server }}.Dockerfile . + ./tests/scripts/container-smoke.sh $REGISTRY/$REPOSITORY:$IMAGE_TAG docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG echo "image=$REGISTRY/$REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT diff --git a/Dockerfile b/Dockerfile index 204d3975..dc12f5ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,42 +1,41 @@ FROM python:3.13-slim LABEL authors="Oleksandr Khoroshevskyi, Nathan Sheffield" -RUN apt-get update -RUN apt-get install -y gcc -RUN apt-get install -y libpq-dev -RUN apt-get install -y --no-install-recommends git +# gcc/build-essential/python3-dev: hnswlib build (geniml); libpq-dev: postgres; git: git deps +RUN apt-get update \ + && apt-get install -y --no-install-recommends gcc build-essential python3-dev libpq-dev git \ + && rm -rf /var/lib/apt/lists/* -# RUN apt-get update -y && \ -# apt-get install -y libpq-dev && \ # for postgres backend -# apt-get install -y gcc && \ -# apt-get install -y wget && \ -# apt-get install -y libcurl4 && \ -# cpp compiler required for hsnwlib +# UCSC bigBedToBed for the /v1/bed/{id}/regions route (needs only libc) +ADD --chmod=755 https://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/bigBedToBed /usr/local/bin/bigBedToBed -EXPOSE 80 -EXPOSE 6333 +ENV HNSWLIB_NO_NATIVE=1 \ + PYTHONDONTWRITEBYTECODE=1 -WORKDIR /app -COPY . /app - -RUN python -m pip install --upgrade pip - -# Need this command due to geniml dependency on hnswlib -ENV HNSWLIB_NO_NATIVE=1 -RUN apt-get install -y python3-dev -RUN apt-get install -y build-essential +RUN pip install --no-cache-dir --upgrade pip uv -RUN pip install uv - -# Install CPU-only pytorch, eliminating huge nvidia dependencies -#pip install torch==2.3.1+cpu -f https://download.pytorch.org/whl/torch_stable.html -RUN uv pip install torch --index-url https://download.pytorch.org/whl/cpu --system -# RUN uv pip install https://github.com/pepkit/pipestat/archive/refs/heads/dev.zip --system +WORKDIR /app +# Install deps before copying the full source so the dep layer caches +COPY pyproject.toml /app/pyproject.toml # Install dependencies only (from pyproject.toml), NOT bedhost itself: the app -# runs from the /app source tree copied above (uvicorn bedhost.main:app), and +# runs from the /app source tree copied below (uvicorn bedhost.main:app), and # the version is read from bedhost/_version.py, so bedhost is intentionally not # installed as a distribution here. -RUN uv pip install -r pyproject.toml --no-cache-dir --system +# --torch-backend cpu: without it, torch resolves to the CUDA build and pulls +# ~3 GB of nvidia-* libraries the server never uses. +RUN uv pip install -r pyproject.toml --torch-backend cpu --system --no-cache-dir --compile-bytecode -CMD ["uvicorn", "bedhost.main:app", "--host", "0.0.0.0", "--port", "80"] \ No newline at end of file +COPY . /app +RUN python -m compileall -q /app/bedhost + +# Unprivileged runtime user. /app and site-packages stay root-owned (read-only to the app). +RUN useradd --system --uid 10001 --user-group --create-home --home-dir /home/bedhost --shell /usr/sbin/nologin bedhost +ENV HOME=/home/bedhost \ + HF_HOME=/home/bedhost/.cache/huggingface \ + FASTEMBED_CACHE_PATH=/home/bedhost/.cache/fastembed \ + NUMBA_CACHE_DIR=/home/bedhost/.cache/numba +USER bedhost + +EXPOSE 8000 +CMD ["uvicorn", "bedhost.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md index bedac817..4a454acf 100644 --- a/README.md +++ b/README.md @@ -56,3 +56,4 @@ RUN_INTEGRATION_TESTS=true pytest tests/integration/ tests/api/ Integration tests require network access on first boot (bbconf downloads a licenses CSV from GitHub and may pull HuggingFace models unless `BEDHOST_INIT_ML=false` is set; `test-integration.sh` sets it). +`BEDHOST_LOG_LEVEL` (default `INFO`) sets the log level for bedhost, bbconf, and geniml. diff --git a/bedhost/_version.py b/bedhost/_version.py index f23a6b39..9e78220f 100644 --- a/bedhost/_version.py +++ b/bedhost/_version.py @@ -1 +1 @@ -__version__ = "0.13.0" +__version__ = "0.14.0" diff --git a/bedhost/const.py b/bedhost/const.py index c5565ae5..38433a73 100644 --- a/bedhost/const.py +++ b/bedhost/const.py @@ -21,8 +21,30 @@ MAX_FILE_SIZE: int = 1024 * 1024 * 20 +# Decompressed size cap for gzip uploads, so a small gzip cannot expand without bound. +MAX_UNCOMPRESSED_SIZE: int = 1024 * 1024 * 200 MAX_REGION_NUMBER: int = 5000000 MIN_REGION_WIDTH: int = 10 +# Chunk size used when copying an uploaded file to disk. +UPLOAD_CHUNK_SIZE: int = 1024 * 1024 +# Slack for multipart boundaries and headers in the Content-Length pre-check. +MULTIPART_OVERHEAD: int = 64 * 1024 +# Routes that accept BED file uploads (guarded by upload_size_guard). +UPLOAD_ROUTES: frozenset[str] = frozenset( + {"/v1/bed/embed", "/v1/bed/umap", "/v1/bed/search/bed"} +) + +# Upper bounds for request inputs (security review finding O1). +MAX_LIST_LIMIT: int = 10000 # paged list endpoints +MAX_SEARCH_LIMIT: int = 100 # search + neighbour endpoints (UI max is 100) +# hg38 with alts/decoys/unplaced is under 3,500 contigs (hs38DH ~3,366); +# mm10/mm39 are under 100. The UI only sends chromosomes that actually +# appear in the uploaded BED file, so 10,000 is a generous ceiling that +# still bounds the work. +MAX_CHROM_ENTRIES: int = 10000 # /analyze-genome dict size +MAX_CHROM_NAME_LENGTH: int = 256 # /analyze-genome dict key length +BIGBED_TIMEOUT_SECONDS: int = 60 # /regions bigBedToBed wall-clock cap +MAX_REGIONS_OUTPUT_BYTES: int = 1024 * 1024 * 20 # /regions response size cap # Public CDN base for bulk metadata export artifacts. These live directly on the # storage CDN (Backblaze B2 fronted by Cloudflare), NOT behind the API's diff --git a/bedhost/data_models.py b/bedhost/data_models.py index 93b32ef2..2ad2b9ce 100644 --- a/bedhost/data_models.py +++ b/bedhost/data_models.py @@ -1,7 +1,10 @@ from enum import Enum +from typing import Annotated from fastapi import Path -from pydantic import BaseModel +from pydantic import BaseModel, Field, StringConstraints + +from .const import MAX_CHROM_ENTRIES, MAX_CHROM_NAME_LENGTH RemoteClassEnum = Enum( "RemoteClassEnum", @@ -83,5 +86,16 @@ class CreateBEDsetRequest(BaseModel): registry_path: str +ChromName = Annotated[ + str, StringConstraints(min_length=1, max_length=MAX_CHROM_NAME_LENGTH) +] +ChromLength = Annotated[int, Field(ge=0)] + + class ChromLengthUploadModel(BaseModel): - bed_file: dict[str, int] + bed_file: dict[ChromName, ChromLength] = Field( + ..., + min_length=1, + max_length=MAX_CHROM_ENTRIES, + description=f"Map of chromosome name to length (1-{MAX_CHROM_ENTRIES} entries, lengths >= 0)", + ) diff --git a/bedhost/helpers.py b/bedhost/helpers.py index 7be0b56d..dbb57577 100644 --- a/bedhost/helpers.py +++ b/bedhost/helpers.py @@ -1,6 +1,9 @@ import datetime import inspect import os +import subprocess +import tempfile +import threading from collections.abc import Callable from functools import wraps from typing import Literal @@ -46,6 +49,72 @@ def serve_file( raise FileNotFoundError(msg) +class RegionOutputTooLarge(Exception): + """bigBedToBed produced more output than the allowed ceiling.""" + + +def run_bigbed_to_bed(cmd: list[str], timeout: int, max_bytes: int) -> str: + """Run bigBedToBed, wait for it, return only the first 3 BED columns. + + Output is read line by line and trimmed as it arrives, so memory is bounded + by ``max_bytes`` rather than by the size of the bigBed file. + + Args: + cmd: Full argv for the ``bigBedToBed`` invocation (no shell). + timeout: Wall-clock timeout in seconds. + max_bytes: Ceiling on the trimmed output size. + + Returns: + The first three tab-separated columns of each output line, + newline-terminated. Empty string if there is no output. + + Raises: + FileNotFoundError: If the binary is missing. + RegionOutputTooLarge: If the output exceeds ``max_bytes``. The child is + killed before this is raised. + subprocess.TimeoutExpired: If the process exceeds ``timeout``. The + child is killed before this is raised. + subprocess.CalledProcessError: If the process exits non-zero. + """ + timed_out = threading.Event() + rows: list[str] = [] + total = 0 + # stderr goes to a file so a chatty child can never block on a full pipe. + with ( + tempfile.TemporaryFile() as err, + subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=err, text=True) as proc, + ): + + def _kill_on_timeout(): + timed_out.set() + proc.kill() + + timer = threading.Timer(timeout, _kill_on_timeout) + timer.start() + try: + for line in proc.stdout: + row = "\t".join(line.rstrip("\n").split("\t", 3)[:3]) + "\n" + total += len(row) + if total > max_bytes: + proc.kill() + raise RegionOutputTooLarge( + f"bigBedToBed output exceeded {max_bytes} bytes" + ) + rows.append(row) + proc.wait() + finally: + timer.cancel() + + if timed_out.is_set(): + raise subprocess.TimeoutExpired(cmd, timeout) + if proc.returncode != 0: + err.seek(0) + raise subprocess.CalledProcessError( + proc.returncode, cmd, stderr=err.read().decode(errors="replace") + ) + return "".join(rows) + + def build_exports_url(file_path: str) -> str: """ Build a public HTTPS CDN URL for an export / analysis-file S3 key. @@ -152,8 +221,8 @@ async def wrapper(*args, **kwargs): else: function_result = await run_in_threadpool(func, *args, **kwargs) if "test_request" in kwargs and kwargs["test_request"]: - _LOGGER.info( - f"Test request was executed. For '{event}' event with: {args}, {kwargs}. No results saved." + _LOGGER.debug( + f"Test request executed for '{event}' event. No results saved." ) return function_result request = kwargs.get("request") diff --git a/bedhost/main.py b/bedhost/main.py index ae1fa816..99282429 100644 --- a/bedhost/main.py +++ b/bedhost/main.py @@ -20,6 +20,7 @@ from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import HTMLResponse, RedirectResponse, Response +from fastapi.routing import APIRoute from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates @@ -34,13 +35,14 @@ init_model_usage, upload_usage, ) +from .uploads import upload_size_guard # This module is the application entry point (uvicorn loads `bedhost.main:app`), # so it is where logging gets configured. Don't move this into `__init__.py`. -logmuse.init_logger(PKG_NAME) - -logging.getLogger("bbconf").setLevel(logging.DEBUG) -logging.getLogger("geniml").setLevel(logging.DEBUG) +LOG_LEVEL = os.environ.get("BEDHOST_LOG_LEVEL", "INFO").upper() +logmuse.init_logger(PKG_NAME, level=LOG_LEVEL) +for name in ("bbconf", "geniml"): + logging.getLogger(name).setLevel(LOG_LEVEL) tags_metadata = [ { @@ -109,21 +111,36 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: _LOGGER.info("Initializing reference genome validator...") app.state.ref_validator = ReferenceValidator() - scheduler = BackgroundScheduler() - scheduler.add_job( - upload_usage, - "interval", - hours=USAGE_SAVE_HOURS, - args=(app.state.bbagent, app.state.usage_data), - ) - scheduler.start() - app.state.scheduler = scheduler + # Read-only guard: when BEDHOST_DISABLE_USAGE_SCHEDULER is truthy, do NOT + # start the hourly usage-flush job (which writes to the DB). Default (unset) + # keeps the scheduler ON so normal/prod deployments are unaffected. + if os.environ.get("BEDHOST_DISABLE_USAGE_SCHEDULER", "").lower() not in ( + "1", + "true", + "yes", + ): + scheduler = BackgroundScheduler() + scheduler.add_job( + upload_usage, + "interval", + hours=USAGE_SAVE_HOURS, + args=(app.state.bbagent, app.state.usage_data), + ) + scheduler.start() + app.state.scheduler = scheduler + else: + app.state.scheduler = None + _LOGGER.info( + "BEDHOST_DISABLE_USAGE_SCHEDULER set; usage flush scheduler NOT " + "started (read-only mode)." + ) try: _LOGGER.info("Starting app ...") yield finally: - app.state.scheduler.shutdown(wait=False) + if app.state.scheduler is not None: + app.state.scheduler.shutdown(wait=False) app = FastAPI( @@ -144,6 +161,10 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: "*", # allow cross-origin resource sharing, since this is a public API ] +# Registered before CORSMiddleware so CORS runs outside it and its 411/413 +# responses still carry CORS headers for browser clients. +app.middleware("http")(upload_size_guard) + app.add_middleware( CORSMiddleware, allow_origins=origins, @@ -224,6 +245,29 @@ async def exc_handler_MissingObjectError(req: Request, exc: MissingObjectError): attach_routers(app) +def _answer_head_wherever_we_answer_get(routes) -> None: + """ + Starlette's plain `Route` adds HEAD alongside GET automatically; FastAPI's + `APIRoute` does not, so uptime monitors that send HEAD (UptimeRobot's basic + monitor, for one) get a 405. Adding the method makes Starlette run the GET + handler and drop the body. + + FastAPI 0.141 stopped flattening included routers into `app.routes` and + keeps a wrapper object holding the original router instead, so recurse + through those as well. Must run before the first request: the wrapper + caches an effective view of its routes on first match. + """ + for route in routes: + if isinstance(route, APIRoute) and "GET" in route.methods: + route.methods.add("HEAD") + included = getattr(route, "original_router", None) + if included is not None: + _answer_head_wherever_we_answer_get(included.routes) + + +_answer_head_wherever_we_answer_get(app.routes) + + def main() -> None: parser = build_parser() args = parser.parse_args() diff --git a/bedhost/routers/base_api.py b/bedhost/routers/base_api.py index 9b852788..2cd30843 100644 --- a/bedhost/routers/base_api.py +++ b/bedhost/routers/base_api.py @@ -1,4 +1,6 @@ +import datetime from platform import python_version +from typing import Literal from bbconf import __version__ as bbconf_version from bbconf.bbagent import BedBaseAgent @@ -7,14 +9,17 @@ BedSnapshotListResult, FileStats, StatsReturn, + UsageResponse, UsageStats, ) -from fastapi import APIRouter, Depends, Query, Request +from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi.responses import RedirectResponse from geniml import __version__ as geniml_version +from sqlalchemy import text +from .. import _LOGGER from .._version import __version__ as bedhost_version -from ..const import EXPORTS_URL_BASE +from ..const import EXPORTS_URL_BASE, MAX_LIST_LIMIT from ..data_models import ( BaseListResponse, ComponentVersions, @@ -36,6 +41,24 @@ packages_versions = {} +@router.get("/ready", include_in_schema=False) +def readiness(bbagent: BedBaseAgent = Depends(get_bbagent)): + """ + Readiness probe. Takes a connection out of the SQLAlchemy pool and runs + ``SELECT 1``. It has to go through the pool, because the pool is what + wedges when the database host reboots without closing its sockets; a + check that opens its own connection would report healthy while every + real request timed out. + """ + try: + with bbagent.config.db_engine.engine.connect() as conn: + conn.execute(text("SELECT 1")) + except Exception as e: + _LOGGER.warning(f"Readiness check failed: {e}") + raise HTTPException(status_code=503, detail="database unavailable") + return {"status": "ok"} + + @router.get( "/stats", summary="Get summary statistics for BEDbase platform", @@ -128,9 +151,12 @@ async def get_assays_list( def get_bed_exports( bbagent: BedBaseAgent = Depends(get_bbagent), limit: int = Query( - 1000, ge=1, le=10000, description="Limit (1-10000), default 1000" + 1000, + ge=1, + le=MAX_LIST_LIMIT, + description=f"Limit (1-{MAX_LIST_LIMIT}), default 1000", ), - offset: int = 0, + offset: int = Query(0, ge=0, description="Offset (>= 0)"), ) -> BedSnapshotListResult: """ Return the index of bulk metadata export artifacts published to S3, newest @@ -161,9 +187,12 @@ def get_analysis_files( genome: str | None = Query(None, description="Filter by genome/assembly"), tag: str | None = Query(None, description="Filter by a single tag"), limit: int = Query( - 1000, ge=1, le=10000, description="Limit (1-10000), default 1000" + 1000, + ge=1, + le=MAX_LIST_LIMIT, + description=f"Limit (1-{MAX_LIST_LIMIT}), default 1000", ), - offset: int = 0, + offset: int = Query(0, ge=0, description="Offset (>= 0)"), ) -> AnalysisFileListResult: """ Return the index of standalone analysis files (openSignalMatrix, models, @@ -238,3 +267,37 @@ async def redirect_to_download( ): download_url = f"{EXPORTS_URL_BASE}{file_path}" return RedirectResponse(url=download_url) + + +@router.get( + "/usage", + summary="Get per-key usage counts for one event type, optionally by date range", + response_model=UsageResponse, +) +def get_usage( + type: Literal["files", "bed_meta", "bedset_meta", "bed_search", "bedset_search"], + date_from: datetime.datetime | None = None, + date_to: datetime.datetime | None = None, + limit: int = Query( + 1000, + ge=1, + le=MAX_LIST_LIMIT, + description=f"Limit (1-{MAX_LIST_LIMIT}), default 1000", + ), + offset: int = Query(0, ge=0, description="Offset (>= 0)"), + bbagent: BedBaseAgent = Depends(get_bbagent), +): + """ + Returns per-key usage counts for a single event type + (files | bed_meta | bedset_meta | bed_search | bedset_search), grouped by key + and summed over the time-bucketed rows whose window overlaps the requested + [date_from, date_to] range. Both date bounds are optional; limit/offset + paginate the keys by count (default limit 1000, max 10000). + """ + return bbagent.get_usage( + event_type=type, + date_from=date_from, + date_to=date_to, + limit=limit, + offset=offset, + ) diff --git a/bedhost/routers/bed_api.py b/bedhost/routers/bed_api.py index 2aca9a20..2040f0c9 100644 --- a/bedhost/routers/bed_api.py +++ b/bedhost/routers/bed_api.py @@ -1,7 +1,4 @@ -import os -import shutil import subprocess -import tempfile from typing import Annotated from bbconf.bbagent import BedBaseAgent @@ -28,14 +25,14 @@ from bedboss.refgenome_validator.main import ReferenceValidator from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile from fastapi.responses import PlainTextResponse, Response -from gtars.models import RegionSet from .. import _LOGGER from ..const import ( + BIGBED_TIMEOUT_SECONDS, EXAMPLE_BED, - MAX_FILE_SIZE, - MAX_REGION_NUMBER, - MIN_REGION_WIDTH, + MAX_LIST_LIMIT, + MAX_REGIONS_OUTPUT_BYTES, + MAX_SEARCH_LIMIT, ) from ..data_models import ( CROM_NUMBERS, @@ -44,11 +41,32 @@ ChromLengthUploadModel, ) from ..dependencies import get_bbagent, get_ref_validator -from ..helpers import count_requests, test_query_parameter +from ..helpers import ( + RegionOutputTooLarge, + build_exports_url, + count_requests, + run_bigbed_to_bed, + test_query_parameter, +) +from ..uploads import uploaded_region_set router = APIRouter(prefix="/v1/bed", tags=["bed"]) +def _require_ml(bbagent: BedBaseAgent, *attrs: str) -> None: + """Return 503 up-front when bbconf was started without the ML pieces a route needs. + + With ``init_ml=False`` these config attributes are ``None`` and bbconf would + otherwise fail deep inside with a 500 AttributeError. + """ + config = getattr(bbagent, "config", None) + if any(getattr(config, attr, None) is None for attr in attrs): + raise HTTPException( + status_code=503, + detail="This endpoint is unavailable (ML models disabled)", + ) + + @router.get( "/example", summary="Get example BED record metadata", @@ -74,9 +92,28 @@ async def get_example_bed_record( ) def list_beds( limit: int = Query( - 1000, ge=1, le=10000, description="Limit (1-10000), default 1000" + 1000, + ge=1, + le=MAX_LIST_LIMIT, + description=f"Limit (1-{MAX_LIST_LIMIT}), default 1000", + ), + offset: int = Query( + 0, + ge=0, + le=10000, + description=( + "Offset (0-10000), default 0. Offset paging gets slower the deeper " + "you go, so it is capped. To go further, use after= or " + "download the bulk snapshot from /v1/exports." + ), + ), + after: str | None = Query( + None, + description=( + "Return records with id greater than this. Keyset paging; ignores " + "offset. Pass the last id of the previous page to get the next one." + ), ), - offset: int = 0, genome: str = Query( default=None, description="filter by genome of the bed file. e.g. 'hg38'" ), @@ -86,11 +123,21 @@ def list_beds( bbagent: BedBaseAgent = Depends(get_bbagent), ) -> BedListResult: """ - Returns list of BED files in the database with optional filters. + To enumerate everything, page with `after=` or download the + monthly Parquet snapshot from `/v1/exports`. + + Returns list of BED files in the database with optional filters. Results + are ordered by id. `count` (the total number of matching records) is only + filled in on the first page (`offset=0` and no `after`); it is `null` on + every other page. """ return bbagent.bed.get_ids_list( - limit=limit, offset=offset, genome=genome, bed_compliance=bed_compliance + limit=limit, + offset=offset, + genome=genome, + bed_compliance=bed_compliance, + after=after, ) @@ -266,8 +313,13 @@ async def get_bed_classification( ) def get_bed_neighbours( bed_id: str = BedDigest, - limit: int = 10, - offset: int = 0, + limit: int = Query( + 10, + ge=1, + le=MAX_SEARCH_LIMIT, + description=f"Limit (1-{MAX_SEARCH_LIMIT}), default 10", + ), + offset: int = Query(0, ge=0, description="Offset (>= 0)"), bbagent: BedBaseAgent = Depends(get_bbagent), ) -> BedListSearchResult: try: @@ -313,45 +365,29 @@ def embed_bed_file( Create embedding for bed file """ _LOGGER.info("Embedding file..") - - if file is not None: - with tempfile.TemporaryDirectory() as dirpath: - file_path = os.path.join(dirpath, file.filename) - - with open(file_path, "wb") as bed_file: - shutil.copyfileobj(file.file, bed_file) - - region_set = RegionSet(file_path) - - embedding = bbagent.bed._embed_file(region_set) + _require_ml(bbagent, "qdrant_file_backend", "r2v_encoder") + with uploaded_region_set(file) as region_set: + embedding = bbagent.bed._embed_file(region_set) return embedding.tolist()[0] @router.post( "/umap", - summary="Get embeddings for a bed file.", + summary="Get UMAP coordinates for a bed file.", response_model=list[float], ) def umap_bed_file( - file: UploadFile = File(None), + file: UploadFile = File(...), bbagent: BedBaseAgent = Depends(get_bbagent), ) -> list[float]: """ - Create embedding for bed file + Compute UMAP coordinates for bed file """ - _LOGGER.info("Embedding file..") - - if file is not None: - with tempfile.TemporaryDirectory() as dirpath: - file_path = os.path.join(dirpath, file.filename) - - with open(file_path, "wb") as bed_file: - shutil.copyfileobj(file.file, bed_file) - - region_set = RegionSet(file_path) - - embedding = bbagent.bed._get_umap_file(region_set) - return embedding.tolist()[0] + _LOGGER.info("Computing UMAP coordinates for file..") + _require_ml(bbagent, "qdrant_file_backend", "r2v_encoder", "umap_encoder") + with uploaded_region_set(file) as region_set: + coordinates = bbagent.bed._get_umap_file(region_set) + return coordinates.tolist()[0] @router.post( @@ -449,55 +485,61 @@ def get_regions_for_bedfile( bed_id: str = BedDigest, chr_num: str = CROM_NUMBERS, start: Annotated[ - str | None, Query(description="query range: start coordinate") + int | None, Query(ge=0, description="Query range: start coordinate (0-based)") ] = None, end: Annotated[ - str | None, Query(description="query range: start coordinate") + int | None, Query(ge=0, description="Query range: end coordinate") ] = None, bbagent: BedBaseAgent = Depends(get_bbagent), ): """ Returns the queried regions with provided ID and optional query parameters """ + if start is not None and end is not None and end <= start: + raise HTTPException(status_code=400, detail="end must be greater than start") + bigbedfile = bbagent.bed.get_files(bed_id).bigbed_file if not bigbedfile: raise HTTPException( status_code=404, detail="ERROR: bigBed file doesn't exists. Can't query." ) - path = bbagent.objects.get_prefixed_uri(bigbedfile.path, access_id="http") + # Use the direct storage URL: the "http" access method points at the API's + # /v1/files/ redirect, and bigBedToBed cannot follow redirects. + path = build_exports_url(bigbedfile.path) _LOGGER.debug(path) cmd = ["bigBedToBed"] if chr_num: cmd.append(f"-chrom={chr_num}") - if start: + if start is not None: cmd.append(f"-start={start}") - if end: + if end is not None: cmd.append(f"-end={end}") cmd.extend([path, "stdout"]) - _LOGGER.info(f"Command: {' '.join(map(str, cmd))} | cut -f1-3") + _LOGGER.info(f"Command: {' '.join(map(str, cmd))}") try: - cut_process = subprocess.Popen( - ["cut", "-f1-3"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - universal_newlines=True, - ) - - subprocess.Popen( - cmd, - stdout=cut_process.stdin, - text=True, + return run_bigbed_to_bed(cmd, BIGBED_TIMEOUT_SECONDS, MAX_REGIONS_OUTPUT_BYTES) + except RegionOutputTooLarge: + raise HTTPException( + status_code=413, + detail=f"Too many regions in the requested range (over " + f"{MAX_REGIONS_OUTPUT_BYTES // (1024 * 1024)} MB). Narrow it with start and end.", ) - - return cut_process.communicate()[0] - except FileNotFoundError: _LOGGER.warning("bigBedToBed is not installed.") raise HTTPException( status_code=500, detail="ERROR: bigBedToBed is not installed." ) + except subprocess.TimeoutExpired: + _LOGGER.warning(f"Region query timed out for bed_id={bed_id}, chrom={chr_num}") + raise HTTPException(status_code=504, detail="Region query timed out.") + except subprocess.CalledProcessError as e: + _LOGGER.error( + f"bigBedToBed failed for bed_id={bed_id}, chrom={chr_num}: " + f"returncode={e.returncode}, stderr={e.stderr}" + ) + raise HTTPException(status_code=502, detail="Failed to query bigBed file.") @router.get( @@ -513,8 +555,13 @@ def text_to_bed_search( query: str, genome: str | None = None, assay: str | None = None, - limit: int = 10, - offset: int = 0, + limit: int = Query( + 10, + ge=1, + le=MAX_SEARCH_LIMIT, + description=f"Limit (1-{MAX_SEARCH_LIMIT}), default 10", + ), + offset: int = Query(0, ge=0, description="Offset (>= 0)"), test_request: bool = test_query_parameter, # needed for usage tracking in @count_requests bbagent: BedBaseAgent = Depends(get_bbagent), ) -> BedListSearchResult: @@ -665,8 +712,13 @@ async def exact_search( query: str, genome: str | None = None, assay: str | None = None, - limit: int = 10, - offset: int = 0, + limit: int = Query( + 10, + ge=1, + le=MAX_SEARCH_LIMIT, + description=f"Limit (1-{MAX_SEARCH_LIMIT}), default 10", + ), + offset: int = Query(0, ge=0, description="Offset (>= 0)"), bbagent: BedBaseAgent = Depends(get_bbagent), ) -> BedListSearchResult: return bbagent.bed.sql_search( @@ -686,56 +738,20 @@ async def exact_search( response_model_by_alias=False, ) def bed_to_bed_search( - file: UploadFile = File(None), - limit: int = 10, - offset: int = 0, + file: UploadFile = File(...), + limit: int = Query( + 10, + ge=1, + le=MAX_SEARCH_LIMIT, + description=f"Limit (1-{MAX_SEARCH_LIMIT}), default 10", + ), + offset: int = Query(0, ge=0, description="Offset (>= 0)"), bbagent: BedBaseAgent = Depends(get_bbagent), ) -> BedListSearchResult: _LOGGER.info("Searching for bedfiles...") - print("file size {}", file.size) - if file.size > MAX_FILE_SIZE: - raise HTTPException( - status_code=413, - detail="File too large. Maximum file size is 20MB.", - ) - - if file is not None: - with tempfile.TemporaryDirectory() as dirpath: - file_path = os.path.join(dirpath, file.filename) - - with open(file_path, "wb") as bed_file: - shutil.copyfileobj(file.file, bed_file) - - try: - region_set = RegionSet(file_path) - except Exception as e: - _LOGGER.error(f"Error reading bed file: {e}") - raise HTTPException( - status_code=415, - detail="Error reading bed file. Please make sure the file is a valid BED file.", - ) - - if region_set.mean_region_width() < MIN_REGION_WIDTH: - raise HTTPException( - status_code=415, - detail="Mean region width is too small. Please provide a BED file with mean region width greater than 10.", - ) - - if len(region_set) > MAX_REGION_NUMBER: - raise HTTPException( - status_code=415, - detail="Too many regions in the BED file. Please provide a BED file with less than 1,000,000 regions.", - ) - - results = bbagent.bed.bed_to_bed_search( - region_set, limit=limit, offset=offset - ) - return results - - raise HTTPException( - status_code=404, - detail="Error occurred, please make sure file is correct and if issue persists, contact support.", - ) + _require_ml(bbagent, "b2b_search_interface") + with uploaded_region_set(file, validate_for_search=True) as region_set: + return bbagent.bed.bed_to_bed_search(region_set, limit=limit, offset=offset) @router.get( @@ -752,7 +768,6 @@ async def get_tokens( Return univers of bed file Example: bed: 0dcdf8986a72a3d85805bbc9493a1302 | universe: 58dee1672b7e581c8e1312bd4ca6b3c7 """ - _LOGGER.info(bbagent.config.config.s3) try: return bbagent.bed.get_tokenized(bed_id, universe_id) diff --git a/bedhost/routers/bedset_api.py b/bedhost/routers/bedset_api.py index dad7ad03..16d7e65d 100644 --- a/bedhost/routers/bedset_api.py +++ b/bedhost/routers/bedset_api.py @@ -9,10 +9,10 @@ BedSetPlots, BedSetStats, ) -from fastapi import APIRouter, Depends, HTTPException, Request, Response +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response from pephubclient.helpers import is_registry_path, unwrap_registry_path -from ..const import EXAMPLE_BEDSET, PKG_NAME +from ..const import EXAMPLE_BEDSET, MAX_LIST_LIMIT, PKG_NAME from ..data_models import CreateBEDsetRequest from ..dependencies import get_bbagent from ..helpers import count_requests, test_query_parameter @@ -48,13 +48,22 @@ async def get_example_bedset_record( def list_bedsets( request: Request, query: str | None = None, - limit: int = 1000, - offset: int = 0, + limit: int = Query( + 1000, + ge=1, + le=MAX_LIST_LIMIT, + description=f"Limit (1-{MAX_LIST_LIMIT}), default 1000", + ), + offset: int = Query(0, ge=0, description="Offset (>= 0)"), test_request: bool = test_query_parameter, bbagent: BedBaseAgent = Depends(get_bbagent), ) -> BedSetListResult: """ - Returns a list of BEDset records in the database with optional filters and search. + To enumerate everything, download the monthly Parquet snapshot from + `/v1/exports` (the `bedsets` file). + + Returns a list of BEDset records in the database with optional filters and + search. Results are ordered by id, so paging with `offset` is stable. """ return bbagent.bedset.get_ids_list(query=query, limit=limit, offset=offset) diff --git a/bedhost/uploads.py b/bedhost/uploads.py new file mode 100644 index 00000000..fd14ede6 --- /dev/null +++ b/bedhost/uploads.py @@ -0,0 +1,174 @@ +"""Safe handling of BED file uploads. + +The client-supplied filename is never used to build a path on disk. Uploads are +written to a fresh temporary directory under a server-chosen name, capped at +``const.MAX_FILE_SIZE`` while streaming (gzip uploads are decompressed on the way +in and capped at ``const.MAX_UNCOMPRESSED_SIZE``), and parsed into a ``RegionSet`` in one +place so every upload route gets the same validation. +""" + +import gzip +import os +import tempfile +import zlib +from contextlib import contextmanager +from typing import BinaryIO, Iterator, Optional + +from fastapi import HTTPException, Request, UploadFile +from fastapi.responses import JSONResponse +from gtars.models import RegionSet + +from . import _LOGGER, const + +GZIP_MAGIC = b"\x1f\x8b" + + +def _too_large_detail(max_bytes: int) -> str: + return f"File too large. Maximum file size is {max_bytes // (1024 * 1024)} MB." + + +class _CappedReader: + """File wrapper that raises HTTP 413 once more than ``max_bytes`` are read.""" + + def __init__(self, src: BinaryIO, max_bytes: int): + self._src = src + self._max_bytes = max_bytes + self.total = 0 + + def read(self, size: int = -1) -> bytes: + data = self._src.read(size) + self.total += len(data) + if self.total > self._max_bytes: + raise HTTPException( + status_code=413, detail=_too_large_detail(self._max_bytes) + ) + return data + + +def save_upload( + file: UploadFile, + dirpath: str, + max_bytes: Optional[int] = None, + max_uncompressed: Optional[int] = None, +) -> str: + """Copy an upload into ``dirpath`` as ``upload.bed``, a fixed, server-chosen name. + + Gzip uploads (detected by magic bytes) are decompressed while copying, so + what lands on disk is always plain text. Raises HTTP 413 as soon as more than + ``max_bytes`` of upload or more than ``max_uncompressed`` of decompressed data + have been read, so a small, highly compressed file cannot expand without + bound. Raises HTTP 415 for a corrupt gzip stream and HTTP 400 for an empty upload. + + :param file: the uploaded file + :param dirpath: directory to write into (should be a fresh temp dir) + :param max_bytes: upload size limit; defaults to ``const.MAX_FILE_SIZE`` read at call time + :param max_uncompressed: decompressed size limit for gzip uploads; defaults to + ``const.MAX_UNCOMPRESSED_SIZE`` read at call time + :return: path of the written file + """ + if max_bytes is None: + max_bytes = const.MAX_FILE_SIZE + if max_uncompressed is None: + max_uncompressed = const.MAX_UNCOMPRESSED_SIZE + + src = file.file + src.seek(0) + is_gzip = src.read(len(GZIP_MAGIC)) == GZIP_MAGIC + src.seek(0) + + reader = _CappedReader(src, max_bytes) + if is_gzip: + stream = gzip.GzipFile(fileobj=reader, mode="rb") + limit = max_uncompressed + else: + stream = reader + limit = max_bytes + + dest = os.path.join(dirpath, "upload.bed") + total = 0 + fd = os.open(dest, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "wb") as out: + try: + while chunk := stream.read(const.UPLOAD_CHUNK_SIZE): + total += len(chunk) + if total > limit: + raise HTTPException( + status_code=413, detail=_too_large_detail(limit) + ) + out.write(chunk) + except (OSError, EOFError, zlib.error) as e: + _LOGGER.warning(f"Error decompressing uploaded file: {e}") + raise HTTPException( + status_code=415, detail="Uploaded file is not a valid gzip file." + ) + + if total == 0: + raise HTTPException(status_code=400, detail="Uploaded file is empty.") + return dest + + +@contextmanager +def uploaded_region_set( + file: Optional[UploadFile], *, validate_for_search: bool = False +) -> Iterator[RegionSet]: + """Save an upload to a temp dir and yield a validated ``RegionSet``. + + The temp file exists for the lifetime of the ``with`` block, so callers can + pass the region set to code that may still read its path. + + :param file: the uploaded file + :param validate_for_search: also require mean region width >= MIN_REGION_WIDTH + """ + if file is None: + raise HTTPException(status_code=400, detail="No file uploaded.") + + with tempfile.TemporaryDirectory(prefix="bedhost-upload-") as dirpath: + path = save_upload(file, dirpath) + try: + region_set = RegionSet(path) + except Exception as e: + _LOGGER.warning(f"Error reading uploaded bed file: {e}") + raise HTTPException( + status_code=415, + detail="Error reading bed file. Please make sure the file is a valid BED file.", + ) + + n_regions = len(region_set) + if n_regions > const.MAX_REGION_NUMBER: + raise HTTPException( + status_code=413, + detail=f"Too many regions in the BED file. Maximum is {const.MAX_REGION_NUMBER:,} regions.", + ) + if n_regions == 0: + raise HTTPException(status_code=415, detail="BED file contains no regions.") + if ( + validate_for_search + and region_set.mean_region_width() < const.MIN_REGION_WIDTH + ): + raise HTTPException( + status_code=415, + detail=f"Mean region width is too small. Please provide a BED file with mean region width of at least {const.MIN_REGION_WIDTH}.", + ) + + yield region_set + + +async def upload_size_guard(request: Request, call_next): + """Reject oversized upload requests by Content-Length before the body is parsed. + + This is a cheap early check only; ``save_upload`` enforces the real limit + while streaming, since Content-Length can be wrong. + """ + if request.method == "POST" and request.url.path in const.UPLOAD_ROUTES: + cl = request.headers.get("content-length") + if cl is None: + return JSONResponse({"detail": "Content-Length required."}, status_code=411) + try: + n = int(cl) + except ValueError: + return JSONResponse({"detail": "Invalid Content-Length."}, status_code=400) + if n > const.MAX_FILE_SIZE + const.MULTIPART_OVERHEAD: + return JSONResponse( + {"detail": _too_large_detail(const.MAX_FILE_SIZE)}, status_code=413 + ) + return await call_next(request) diff --git a/deployment/Dockerfiles/dev1.Dockerfile b/deployment/Dockerfiles/dev1.Dockerfile index 30bc080a..18bc9cf0 100644 --- a/deployment/Dockerfiles/dev1.Dockerfile +++ b/deployment/Dockerfiles/dev1.Dockerfile @@ -1,9 +1,7 @@ -FROM databio/bedhost:dev +# The deploy workflow passes BASE_IMAGE built from this checkout's Dockerfile, +# so a deploy never depends on what was last published to Docker Hub. +ARG BASE_IMAGE=databio/bedhost:dev +FROM ${BASE_IMAGE} COPY deployment/config/api-dev.bedbase.org.yaml /bedbase.yaml ENV BEDBASE_CONFIG=/bedbase.yaml - -RUN mkdir -p /data/outputs/bedstat_output -RUN mkdir -p /data/outputs/bedbuncher_output - -ENTRYPOINT ["uvicorn", "bedhost.main:app", "--host", "0.0.0.0"] diff --git a/deployment/Dockerfiles/primary.Dockerfile b/deployment/Dockerfiles/primary.Dockerfile index f472de9a..663f3593 100644 --- a/deployment/Dockerfiles/primary.Dockerfile +++ b/deployment/Dockerfiles/primary.Dockerfile @@ -1,9 +1,7 @@ -FROM databio/bedhost:latest +# The deploy workflow passes BASE_IMAGE built from this checkout's Dockerfile, +# so a deploy never depends on what was last published to Docker Hub. +ARG BASE_IMAGE=databio/bedhost:latest +FROM ${BASE_IMAGE} COPY deployment/config/api.bedbase.org.yaml /bedbase.yaml ENV BEDBASE_CONFIG=/bedbase.yaml - -RUN mkdir -p /data/outputs/bedstat_output -RUN mkdir -p /data/outputs/bedbuncher_output - -ENTRYPOINT ["uvicorn", "bedhost.main:app", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/deployment/dev/README.md b/deployment/dev/README.md new file mode 100644 index 00000000..39f4bb90 --- /dev/null +++ b/deployment/dev/README.md @@ -0,0 +1,32 @@ +# Local dev environment + +Runs bedhost against a throwaway local Postgres, with ML, Qdrant, and S3 turned off. +Commands assume a venv at the repo root (`.venv`) with bedhost and bbconf installed. + +1. Start Postgres (host port 15443), from this directory: + + ``` + docker compose up + ``` + +2. Create the tables, from the repo root (expects bbconf cloned next to bedhost): + + ``` + .venv/bin/alembic -c deployment/dev/alembic.dev.ini upgrade head + ``` + +3. Optional: add fake usage data, from the repo root: + + ``` + .venv/bin/python deployment/dev/seed_usage.py + ``` + +4. Start the API on port 8104, from this directory: + + ``` + source dev.env + ../../.venv/bin/uvicorn bedhost.main:app --host 127.0.0.1 --port 8104 + ``` + +`bedbase-remote.yaml` points a read-only server at the production database instead. +It needs production credentials in the environment and never runs migrations. diff --git a/deployment/dev/alembic.dev.ini b/deployment/dev/alembic.dev.ini new file mode 100644 index 00000000..bc23b41f --- /dev/null +++ b/deployment/dev/alembic.dev.ini @@ -0,0 +1,46 @@ +# Alembic config for running bbconf migrations against the LOCAL dev Postgres. +# Usage (from the bedhost repo, with the dev venv): +# .venv/bin/alembic -c deployment/dev/alembic.dev.ini upgrade head +# Assumes bbconf is cloned next to bedhost (../bbconf). %(here)s is this file's +# directory, so this works from any working directory. +[alembic] +script_location = %(here)s/../../../bbconf/bbconf/alembic +prepend_sys_path = . +version_path_separator = os +sqlalchemy.url = postgresql+psycopg://bedbase:bedbase@127.0.0.1:15443/bedbase + +[post_write_hooks] + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/deployment/dev/bedbase-dev.yaml b/deployment/dev/bedbase-dev.yaml new file mode 100644 index 00000000..f5322c10 --- /dev/null +++ b/deployment/dev/bedbase-dev.yaml @@ -0,0 +1,38 @@ +# Local dev bbconf config for bedhost. +# +# Points at the local Docker Postgres (deployment/dev/docker-compose.yml). +# Qdrant and S3 sections are intentionally OMITTED: with no `qdrant` section, +# BedBaseConfig fails to build a qdrant client during construction, which the +# constructor catches and uses to set init_ml=False -- so no ML models are +# loaded and no live qdrant/s3 connection is needed. This keeps the dev env +# lightweight and sufficient for the read-only /v1/usage endpoint. +# +# Credentials are interpolated from the environment (see deployment/dev/dev.env). +database: + host: $BEDBASE_PG_HOST + port: $BEDBASE_PG_PORT + user: $BEDBASE_PG_USER + password: $BEDBASE_PG_PASSWORD + database: bedbase + dialect: postgresql + driver: psycopg + run_migrations: true +server: + host: 127.0.0.1 + port: 8104 +path: {} +# Needed to build file download links (/v1/bed/{id}/metadata/files, DRS). +# Copied from deployment/config/api.bedbase.org.yaml. +access_methods: + http: + type: "https" + description: HTTP compatible path + prefix: https://api.bedbase.org/v1/files/ + s3: + type: "s3" + description: S3 compatible path + prefix: s3://data2.bedbase.org/ + local: + type: "https" + description: How to serve local files. + prefix: /static/ diff --git a/deployment/dev/bedbase-remote.yaml b/deployment/dev/bedbase-remote.yaml new file mode 100644 index 00000000..7d155a73 --- /dev/null +++ b/deployment/dev/bedbase-remote.yaml @@ -0,0 +1,40 @@ +# READ-ONLY bbconf config pointing at the REAL production BEDbase Postgres. +# +# Mirrors the working local dev config (bedbase-dev.yaml): `path: {}` is present +# and the `qdrant` and `s3` sections are intentionally OMITTED. With no `qdrant` +# section, BedBaseConfig fails to build a qdrant client during construction, +# which the constructor catches and uses to force init_ml=False -- so no ML +# models load and no live qdrant/s3 connection is needed for the read-only +# /v1/usage and /v1/stats endpoints. +# +# The database section points at the production env vars, which are resolved by +# sourcing environment/production.env (which pulls creds from `pass`) at service +# launch. run_migrations is FALSE: never run migrations against production. +database: + host: $POSTGRES_HOST + port: 5432 + user: $POSTGRES_USER + password: $POSTGRES_PASSWORD + database: $POSTGRES_DB + dialect: postgresql + driver: psycopg + run_migrations: false +server: + host: 127.0.0.1 + port: 8125 +path: {} +# Needed to build file download links (/v1/bed/{id}/metadata/files, DRS). +# Copied from deployment/config/api.bedbase.org.yaml. +access_methods: + http: + type: "https" + description: HTTP compatible path + prefix: https://api.bedbase.org/v1/files/ + s3: + type: "s3" + description: S3 compatible path + prefix: s3://data2.bedbase.org/ + local: + type: "https" + description: How to serve local files. + prefix: /static/ diff --git a/deployment/dev/dev.env b/deployment/dev/dev.env new file mode 100644 index 00000000..8776a1b5 --- /dev/null +++ b/deployment/dev/dev.env @@ -0,0 +1,14 @@ +# Env for the local dev bedhost backend (loaded by the dash dashboard, which +# sources this file, so every var must be `export`ed to reach the process). +# Local dev database only -- these creds match deployment/dev/docker-compose.yml. +export BEDBASE_PG_HOST=127.0.0.1 +export BEDBASE_PG_PORT=15443 +export BEDBASE_PG_USER=bedbase +export BEDBASE_PG_PASSWORD=bedbase + +# Path to the bbconf config the server reads (bedhost/bedhost/main.py). +# Relative path: start the server from this directory (deployment/dev). +export BEDBASE_CONFIG=bedbase-dev.yaml + +# Skip the heavy ReferenceValidator / ML init for the lightweight dev server. +export BEDHOST_INIT_ML=false diff --git a/deployment/dev/docker-compose.yml b/deployment/dev/docker-compose.yml new file mode 100644 index 00000000..99b63852 --- /dev/null +++ b/deployment/dev/docker-compose.yml @@ -0,0 +1,22 @@ +# Local dev Postgres for bedhost/bbconf, managed by the "dash" dashboard. +# Isolated from any production DB. Exposes Postgres on host port 15443. +services: + postgres: + image: pgvector/pgvector:pg18 + container_name: bedbase-postgres-dev + environment: + POSTGRES_USER: bedbase + POSTGRES_PASSWORD: bedbase + POSTGRES_DB: bedbase + ports: + - "127.0.0.1:15443:5432" + volumes: + - bedbase_dev_pgdata:/var/lib/postgresql + healthcheck: + test: ["CMD-SHELL", "pg_isready -U bedbase -d bedbase"] + interval: 5s + timeout: 5s + retries: 5 + +volumes: + bedbase_dev_pgdata: diff --git a/deployment/dev/seed_usage.py b/deployment/dev/seed_usage.py new file mode 100644 index 00000000..c54ce8d9 --- /dev/null +++ b/deployment/dev/seed_usage.py @@ -0,0 +1,62 @@ +"""Seed synthetic usage rows into the local dev bedbase Postgres. + +Inserts >20 distinct usage_files rows (to prove the /v1/usage endpoint is not +capped at the top 20) spread across three date windows (to exercise the +date_from/date_to filter). + +usage_bed_meta / usage_bedset_meta are left empty on purpose: their key columns +are foreign keys into the bed / bedsets tables, so seeding them would require +first creating real BED records. usage_files has no such constraint and is +enough to exercise the new endpoint. + +Run with the dev venv, from the bedhost repo: + .venv/bin/python deployment/dev/seed_usage.py +""" + +import datetime + +from bbconf.db_utils import Base, UsageFiles +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + +DB_URL = "postgresql+psycopg://bedbase:bedbase@127.0.0.1:15443/bedbase" + +# Three non-overlapping monthly windows. +WINDOWS = [ + (datetime.datetime(2026, 1, 1), datetime.datetime(2026, 1, 31)), + (datetime.datetime(2026, 2, 1), datetime.datetime(2026, 2, 28)), + (datetime.datetime(2026, 3, 1), datetime.datetime(2026, 3, 31)), +] + + +def main() -> None: + engine = create_engine(DB_URL) + # Idempotent: make sure the schema exists (migrations also create it). + Base.metadata.create_all(engine) + + with Session(engine) as session: + # Clear any prior synthetic seed rows so re-running is idempotent. + session.query(UsageFiles).delete() + session.commit() + + files = [] + # 25 distinct file keys => proves no top-20 cap on `type=files`. + for i in range(25): + date_from, date_to = WINDOWS[i % len(WINDOWS)] + files.append( + UsageFiles( + file_path=f"s3://bedbase/files/GSM{1000 + i:04d}.bed.gz", + count=(i + 1) * 3, + date_from=date_from, + date_to=date_to, + ) + ) + session.add_all(files) + session.commit() + + n_files = session.query(UsageFiles).count() + print(f"Seeded {n_files} usage_files rows.") + + +if __name__ == "__main__": + main() diff --git a/deployment/task_defs/port81.json b/deployment/task_defs/port81.json index b7f2e6b3..ffbcb814 100644 --- a/deployment/task_defs/port81.json +++ b/deployment/task_defs/port81.json @@ -66,7 +66,7 @@ "hostname": null, "extraHosts": null, "pseudoTerminal": null, - "user": null, + "user": "10001:10001", "readonlyRootFilesystem": null, "dockerLabels": null, "systemControls": null, diff --git a/deployment/task_defs/port82.json b/deployment/task_defs/port82.json index fea1dd4c..166add0f 100644 --- a/deployment/task_defs/port82.json +++ b/deployment/task_defs/port82.json @@ -66,7 +66,7 @@ "hostname": null, "extraHosts": null, "pseudoTerminal": null, - "user": null, + "user": "10001:10001", "readonlyRootFilesystem": null, "dockerLabels": null, "systemControls": null, diff --git a/deployment/task_defs/port83.json b/deployment/task_defs/port83.json index 90a50a3b..e9e816d7 100644 --- a/deployment/task_defs/port83.json +++ b/deployment/task_defs/port83.json @@ -66,7 +66,7 @@ "hostname": null, "extraHosts": null, "pseudoTerminal": null, - "user": null, + "user": "10001:10001", "readonlyRootFilesystem": null, "dockerLabels": null, "systemControls": null, diff --git a/dev.Dockerfile b/dev.Dockerfile index 204d3975..dc12f5ed 100644 --- a/dev.Dockerfile +++ b/dev.Dockerfile @@ -1,42 +1,41 @@ FROM python:3.13-slim LABEL authors="Oleksandr Khoroshevskyi, Nathan Sheffield" -RUN apt-get update -RUN apt-get install -y gcc -RUN apt-get install -y libpq-dev -RUN apt-get install -y --no-install-recommends git +# gcc/build-essential/python3-dev: hnswlib build (geniml); libpq-dev: postgres; git: git deps +RUN apt-get update \ + && apt-get install -y --no-install-recommends gcc build-essential python3-dev libpq-dev git \ + && rm -rf /var/lib/apt/lists/* -# RUN apt-get update -y && \ -# apt-get install -y libpq-dev && \ # for postgres backend -# apt-get install -y gcc && \ -# apt-get install -y wget && \ -# apt-get install -y libcurl4 && \ -# cpp compiler required for hsnwlib +# UCSC bigBedToBed for the /v1/bed/{id}/regions route (needs only libc) +ADD --chmod=755 https://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/bigBedToBed /usr/local/bin/bigBedToBed -EXPOSE 80 -EXPOSE 6333 +ENV HNSWLIB_NO_NATIVE=1 \ + PYTHONDONTWRITEBYTECODE=1 -WORKDIR /app -COPY . /app - -RUN python -m pip install --upgrade pip - -# Need this command due to geniml dependency on hnswlib -ENV HNSWLIB_NO_NATIVE=1 -RUN apt-get install -y python3-dev -RUN apt-get install -y build-essential +RUN pip install --no-cache-dir --upgrade pip uv -RUN pip install uv - -# Install CPU-only pytorch, eliminating huge nvidia dependencies -#pip install torch==2.3.1+cpu -f https://download.pytorch.org/whl/torch_stable.html -RUN uv pip install torch --index-url https://download.pytorch.org/whl/cpu --system -# RUN uv pip install https://github.com/pepkit/pipestat/archive/refs/heads/dev.zip --system +WORKDIR /app +# Install deps before copying the full source so the dep layer caches +COPY pyproject.toml /app/pyproject.toml # Install dependencies only (from pyproject.toml), NOT bedhost itself: the app -# runs from the /app source tree copied above (uvicorn bedhost.main:app), and +# runs from the /app source tree copied below (uvicorn bedhost.main:app), and # the version is read from bedhost/_version.py, so bedhost is intentionally not # installed as a distribution here. -RUN uv pip install -r pyproject.toml --no-cache-dir --system +# --torch-backend cpu: without it, torch resolves to the CUDA build and pulls +# ~3 GB of nvidia-* libraries the server never uses. +RUN uv pip install -r pyproject.toml --torch-backend cpu --system --no-cache-dir --compile-bytecode -CMD ["uvicorn", "bedhost.main:app", "--host", "0.0.0.0", "--port", "80"] \ No newline at end of file +COPY . /app +RUN python -m compileall -q /app/bedhost + +# Unprivileged runtime user. /app and site-packages stay root-owned (read-only to the app). +RUN useradd --system --uid 10001 --user-group --create-home --home-dir /home/bedhost --shell /usr/sbin/nologin bedhost +ENV HOME=/home/bedhost \ + HF_HOME=/home/bedhost/.cache/huggingface \ + FASTEMBED_CACHE_PATH=/home/bedhost/.cache/fastembed \ + NUMBA_CACHE_DIR=/home/bedhost/.cache/numba +USER bedhost + +EXPOSE 8000 +CMD ["uvicorn", "bedhost.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 00000000..54d16f49 --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,25 @@ +# Changelog + +This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. + +## [0.14.0] - 2026-10-16 + +### Added +- `GET /v1/usage`: public per-key usage counts (bed/bedset views, file downloads, searches) for an optional date range, with paging. +- `GET /v1/ready` readiness probe, and `HEAD` on every `GET` route. +- `after=` keyset paging on `/v1/bed/list`. +- `BEDHOST_LOG_LEVEL` (default `INFO`) sets the log level; `BEDHOST_DISABLE_USAGE_SCHEDULER` turns off the background usage flush. +- The Docker image includes UCSC `bigBedToBed`, so `/v1/bed/{id}/regions/{chr}` works. + +### Changed +- **Breaking:** the container runs as non-root user `10001` and serves on port 8000. +- **Breaking:** input limits. `/v1/bed/list` `offset` is capped at 10000 and `count` is `null` after the first page; list `limit` is capped; search and neighbour `limit` is capped at 100; offsets must be `>= 0`; `/regions` `start`/`end` are integers with `end > start`. +- **Breaking:** uploads must send `Content-Length` (411 otherwise). Files over the size limit return 413, including gzip files that decompress past 200 MB; corrupt gzip returns 415; too many regions returns 413. +- Upload and ML routes return 503 when ML models are off. +- `/regions` output is streamed with a 20 MB cap (413) and a timeout (504). +- Deploys build the base image from the checkout and run a container smoke test before pushing. +- Requires bbconf 0.16.0. + +### Security +- Uploads are saved safely with size caps checked while streaming. +- bbconf and geniml no longer log at DEBUG level in production. diff --git a/docs/index.md b/docs/index.md index 493c31ae..e6d706a9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,3 +7,77 @@ Welcome to the BEDbase API. You might be looking for: - [Developer Guide and FAQ](https://docs.bedbase.org/bedhost) - [bedbase.org user interface](https://bedbase.org) - [Sheffield lab of computational biology](https://databio.org) + +## Bulk metadata export + +If you want metadata for every BED file or BEDset, do not crawl the list +endpoints. Download the monthly Parquet snapshot instead. It is one request, +it is served from a CDN, and it does not touch the database. + +- `/v1/exports` lists the published snapshots, newest first. Each entry has + a `file_path` (an HTTPS URL), `file_type`, `record_count`, `file_size`, + `checksum`, and `creation_date`. +- `/v1/objects/exports` lists the same files as GA4GH DRS objects, and + `/v1/objects/exports/{filename}` resolves one of them. +- [bedbase.org/downloads](https://bedbase.org/downloads) shows the same + index in the web interface. + +Each snapshot is three Parquet files plus a JSON manifest, all dated the same +day: + +- `bedbase_metadata_.parquet`: one row per BED file, with its + classification, stats, and annotations. This is what `/v1/bed/list` and + `/v1/bed/{id}/metadata` serve. +- `bedbase_bedsets_.parquet`: one row per BEDset. +- `bedbase_bedset_membership_.parquet`: one row per (bedset, bed) pair. +- `manifest_.json`: the file names, row counts, sizes, and sha256 sums. + +Pick the newest `metadata` entry and read it directly with pandas: + + import pandas as pd + import requests + + exports = requests.get("https://api.bedbase.org/v1/exports").json()["results"] + url = next(e["file_path"] for e in exports if e["file_type"] == "metadata") + df = pd.read_parquet(url) + +Resolve the file through `/v1/exports` rather than hardcoding a name. Files +are dated and immutable, so a new one appears each month. + +## Paging + +`/v1/bed/list` returns records ordered by id. Two ways to page: + +- `offset`: the usual `limit` and `offset` pair. It is capped at + `offset=10000` because Postgres has to read and throw away `offset` rows on + every page, so deep pages get slow. Requests above the cap return a 422. +- `after`: keyset paging. Pass the last id of the previous page as + `after=` and you get the next `limit` records with a greater id. Each + page costs the same no matter how far in you are. When `after` is set, + `offset` is ignored. + +The `count` field (the total number of matching records) is only filled in on +the first page (`offset=0` and no `after`). On every other page it is `null`. + +To walk every BED file for one genome: + + import requests + + base = "https://api.bedbase.org/v1/bed/list" + after = None + while True: + params = {"genome": "hg38", "limit": 1000} + if after: + params["after"] = after + page = requests.get(base, params=params).json() + if not page["results"]: + break + for record in page["results"]: + ... # do something with each record + after = page["results"][-1]["id"] + +If you find yourself doing this for the whole database, use the +[bulk export](#bulk-metadata-export) above instead. + +`/v1/bedset/list` is also ordered by id, and pages with `limit` (max 10000) +and `offset`. diff --git a/pyproject.toml b/pyproject.toml index 45d51a3b..f64bbc8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Bio-Informatics", ] dependencies = [ - "bbconf>=0.15.0", + "bbconf>=0.16.0", "fastapi>=0.103.0", "sentence_transformers>=5.0", "logmuse>=0.3.1", diff --git a/test.Dockerfile b/test.Dockerfile index 4f08d822..580e63c1 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -1,9 +1,4 @@ FROM bedhost -COPY config.yaml /config.yaml -ENV BEDBASE_CONFIG=/config.yaml - -RUN mkdir -p /data/outputs/bedstat_output -RUN mkdir -p /data/outputs/bedbuncher_output - -ENTRYPOINT ["uvicorn", "bedhost.main:app", "--host", "0.0.0.0", "--port", "80"] \ No newline at end of file +COPY config.yaml /bedbase.yaml +ENV BEDBASE_CONFIG=/bedbase.yaml diff --git a/tests/api/test_compliance.py b/tests/api/test_compliance.py index 18c9fd7f..b055d1c7 100644 --- a/tests/api/test_compliance.py +++ b/tests/api/test_compliance.py @@ -27,6 +27,14 @@ def test_service_info_returns_200(self, api_root): res = api_root.get("/v1/service-info") assert res.status_code == 200 + def test_ready_returns_200(self, api_root): + res = api_root.get("/v1/ready") + assert res.status_code == 200 + + def test_head_on_get_route(self, api_root): + res = api_root.head("/v1/service-info") + assert res.status_code == 200 + def test_service_info_has_required_fields(self, api_root): """Service info should contain required GA4GH fields.""" res = api_root.get("/v1/service-info") @@ -116,6 +124,19 @@ def test_bed_list_filters_by_genome(self, api_root): if "genome_alias" in bed: assert bed["genome_alias"] == "hg38" + def test_bed_list_after_pages_forward(self, api_root): + """after= returns only ids greater than it.""" + first = api_root.get("/v1/bed/list?limit=1").json()["results"] + if not first: + pytest.skip("no beds") + res = api_root.get(f"/v1/bed/list?after={first[0]['id']}") + assert res.status_code == 200 + assert all(b["id"] > first[0]["id"] for b in res.json()["results"]) + + def test_bed_list_offset_too_high_is_422(self, api_root): + res = api_root.get("/v1/bed/list?offset=10001") + assert res.status_code == 422 + @pytest.mark.require_service class TestBedExampleEndpoint: @@ -197,6 +218,30 @@ def test_bed_classification_returns_200(self, api_root, example_bed_id): assert res.status_code == 200 +@pytest.mark.require_service +class TestBedRegionsEndpoint: + """Test the /regions endpoint end to end (runs bigBedToBed on a real bigBed).""" + + def test_regions_returns_bed3_lines(self, api_root, example_bed_id): + """Regions query should return tab-separated chrom/start/end lines.""" + if not example_bed_id: + pytest.skip("No example BED ID available") + res = api_root.get( + f"/v1/bed/{example_bed_id}/regions/chr1?start=0&end=1000000", timeout=60 + ) + if res.status_code == 404 and "bigBed" in res.text: + pytest.skip("Example BED record has no bigBed file") + if res.status_code == 500 and "not installed" in res.text: + pytest.skip("bigBedToBed is not installed on the server's PATH") + assert res.status_code == 200 + lines = res.text.strip().splitlines() + assert lines, "expected at least one region in chr1:0-1000000" + for line in lines: + chrom, start, end = line.split("\t") + assert chrom == "chr1" + assert 0 <= int(start) < int(end) + + @pytest.mark.require_service class TestBedSearchEndpoints: """Test BED search endpoints.""" @@ -309,6 +354,76 @@ def test_openapi_json_available(self, api_root): assert "paths" in data +@pytest.mark.require_service +class TestInputBounds: + """Regression tests for bounded request inputs (security review O1).""" + + def test_bedset_list_limit_too_high_is_422(self, api_root): + res = api_root.get("/v1/bedset/list?limit=10001") + assert res.status_code == 422 + + def test_bedset_list_offset_negative_is_422(self, api_root): + res = api_root.get("/v1/bedset/list?offset=-1") + assert res.status_code == 422 + + def test_bedset_list_limit_at_max_is_200(self, api_root): + res = api_root.get("/v1/bedset/list?limit=10000") + assert res.status_code == 200 + + def test_usage_limit_too_high_is_422(self, api_root): + res = api_root.get("/v1/usage?type=files&limit=10001") + assert res.status_code == 422 + + def test_text_search_limit_too_high_is_422(self, api_root): + res = api_root.get("/v1/bed/search/text?query=x&limit=101") + assert res.status_code == 422 + + def test_text_search_offset_negative_is_422(self, api_root): + res = api_root.get("/v1/bed/search/text?query=x&offset=-1") + assert res.status_code == 422 + + def test_exact_search_limit_too_high_is_422(self, api_root): + res = api_root.get("/v1/bed/search/exact?query=x&limit=101") + assert res.status_code == 422 + + def test_neighbours_limit_too_high_is_422(self, api_root, example_bed_id): + if not example_bed_id: + pytest.skip("No example BED ID available") + res = api_root.get(f"/v1/bed/{example_bed_id}/neighbours?limit=101") + assert res.status_code == 422 + + def test_regions_start_negative_is_422(self, api_root, example_bed_id): + if not example_bed_id: + pytest.skip("No example BED ID available") + res = api_root.get(f"/v1/bed/{example_bed_id}/regions/chr1?start=-1") + assert res.status_code == 422 + + def test_regions_start_non_numeric_is_422(self, api_root, example_bed_id): + if not example_bed_id: + pytest.skip("No example BED ID available") + res = api_root.get(f"/v1/bed/{example_bed_id}/regions/chr1?start=abc") + assert res.status_code == 422 + + def test_regions_end_not_greater_than_start_is_400(self, api_root, example_bed_id): + if not example_bed_id: + pytest.skip("No example BED ID available") + res = api_root.get(f"/v1/bed/{example_bed_id}/regions/chr1?start=100&end=50") + assert res.status_code == 400 + + def test_analyze_genome_negative_length_is_422(self, api_root): + res = api_root.post("/v1/bed/analyze-genome", json={"bed_file": {"chr1": -5}}) + assert res.status_code == 422 + + def test_analyze_genome_empty_dict_is_422(self, api_root): + res = api_root.post("/v1/bed/analyze-genome", json={"bed_file": {}}) + assert res.status_code == 422 + + def test_analyze_genome_too_many_entries_is_422(self, api_root): + bed_file = {f"chr{i}": i for i in range(10001)} + res = api_root.post("/v1/bed/analyze-genome", json={"bed_file": bed_file}) + assert res.status_code == 422 + + @pytest.mark.require_service class TestMarkdownAndSchemaPages: """Regression tests for pages that broke under Starlette 1.0.""" diff --git a/tests/client.py b/tests/client.py index a54787c4..8fd5cd62 100644 --- a/tests/client.py +++ b/tests/client.py @@ -4,7 +4,7 @@ can run either against a live HTTP server (``RequestsClient``) or in-process via FastAPI's ``TestClient`` (``TestClientAdapter``). -Both ``.get`` / ``.post`` return a response object exposing +``.get`` / ``.post`` / ``.head`` return a response object exposing ``.status_code``, ``.json()``, ``.text``, and ``.headers`` -- the subset used by the compliance suite. """ @@ -26,6 +26,11 @@ def post(self, path, **kw): return requests.post(f"{self.base_url}{path}", **kw) + def head(self, path, **kw): + import requests + + return requests.head(f"{self.base_url}{path}", **kw) + class TestClientAdapter: """Wrap a ``fastapi.testclient.TestClient`` with the requests-like API. @@ -48,3 +53,6 @@ def get(self, path, **kw): def post(self, path, **kw): return self._client.post(path, **self._translate_kwargs(kw)) + + def head(self, path, **kw): + return self._client.head(path, **self._translate_kwargs(kw)) diff --git a/tests/fixtures/bedbase_test_config.template.yaml b/tests/fixtures/bedbase_test_config.template.yaml index 7a3ab31d..de2a908a 100644 --- a/tests/fixtures/bedbase_test_config.template.yaml +++ b/tests/fixtures/bedbase_test_config.template.yaml @@ -16,6 +16,19 @@ server: host: 0.0.0.0 port: 8000 path: - region2vec: databio/r2v_encoder-ChIP-atlas-hg38 + region2vec: databio/r2v-encode-hg38 text2vec: sentence-transformers/all-MiniLM-L6-v2 sparse_model: prithivida/Splade_PP_en_v2 +access_methods: + http: + type: "https" + description: HTTP compatible path + prefix: https://data2.bedbase.org/ + s3: + type: "s3" + description: S3 compatible path + prefix: s3://data2.bedbase.org/ + local: + type: "https" + description: How to serve local files. + prefix: /static/ diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 62a2b8e6..77f884f7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -124,13 +124,107 @@ def test_client(bedhost_app): yield c +# Seed data mirrors a real public bedbase record (hg38 CTCF ChIP-seq peaks), +# so its bigBed path resolves on the public CDN for /regions queries. +SEED_BED_ID = "bbad85f21962bb8d972444f7f9a3a932" +SEED_BEDSET_ID = "gse218680" +SEED_BED_FILES = { + "bed_file": ("BED file", "files/b/b/bbad85f21962bb8d972444f7f9a3a932.bed.gz"), + "bigbed_file": ( + "BigBed file", + "files/b/b/bbad85f21962bb8d972444f7f9a3a932.bigBed", + ), +} + + +@pytest.fixture(scope="session") +def seeded_db(test_client): + """Insert one BED record and one BEDset through bbconf's own API. + + Without this the ephemeral DB is empty and every test that needs an + example BED/BEDset ID skips. Files are inserted directly because + ``bed.add`` only records them when uploading to S3. + """ + from bbconf.db_utils import Files + from sqlalchemy.orm import Session + + bbagent = test_client.app.state.bbagent + bbagent.bed.add( + identifier=SEED_BED_ID, + stats={ + "number_of_regions": 57_281, + "gc_content": 0.52, + "median_tss_dist": 12_345, + "mean_region_width": 280.5, + "exon_frequency": 3_000, + "exon_percentage": 5.2, + "intron_frequency": 20_000, + "intron_percentage": 34.9, + "intergenic_percentage": 45.1, + "intergenic_frequency": 25_800, + "promotercore_frequency": 4_000, + "promotercore_percentage": 7.0, + "fiveutr_frequency": 500, + "fiveutr_percentage": 0.9, + "threeutr_frequency": 1_000, + "threeutr_percentage": 1.7, + "promoterprox_frequency": 2_981, + "promoterprox_percentage": 5.2, + }, + metadata={ + "species_name": "Homo sapiens", + "species_id": "9606", + "cell_type": "neural progenitor cell", + "assay": "ChIP-seq", + "antibody": "CTCF", + "target": "CTCF", + "global_experiment_id": ["geo:gse218680"], + }, + plots={}, + files={}, + classification={ + "name": "PM_137_NPC_CTCF_ChIP", + "genome_alias": "hg38", + "genome_digest": "Ba88PY52_qeifhJrgUXyin6UITdXNsg3", + "bed_compliance": "bed6+4", + "data_format": "encode_narrowpeak", + "compliant_columns": 6, + "non_compliant_columns": 4, + }, + upload_qdrant=False, + upload_s3=False, + ) + with Session(bbagent.config.db_engine.engine) as session: + for name, (title, path) in SEED_BED_FILES.items(): + session.add( + Files( + name=name, + title=title, + path=path, + type="file", + bedfile_id=SEED_BED_ID, + ) + ) + session.commit() + + bbagent.bedset.create( + identifier=SEED_BEDSET_ID, + name=SEED_BEDSET_ID, + bedid_list=[SEED_BED_ID], + description="Seeded test BEDset", + statistics=True, + ) + return {"bed_id": SEED_BED_ID, "bedset_id": SEED_BEDSET_ID} + + @pytest.fixture(scope="session") -def integration_api_root(test_client): +def integration_api_root(test_client, seeded_db): """The integration-mode ``api_root`` fixture value. Named distinctly from ``api_root`` to avoid a fixture name collision with the top-level ``tests/conftest.py``; the top-level fixture delegates via ``request.getfixturevalue`` when integration mode is on. + Depends on ``seeded_db`` so every API test sees the example records. """ from tests.client import TestClientAdapter diff --git a/tests/scripts/container-smoke.sh b/tests/scripts/container-smoke.sh new file mode 100755 index 00000000..62adf87e --- /dev/null +++ b/tests/scripts/container-smoke.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Smoke-test a built bedhost image before it is pushed or deployed. +# +# Checks the things the ECS task definitions rely on: the image runs as UID +# 10001, serves on port 8000, has writable cache/temp dirs, keeps the app code +# read-only, the app module imports, and bigBedToBed runs. No database is needed. +# +# Usage: ./tests/scripts/container-smoke.sh + +set -euo pipefail + +IMAGE="${1:?usage: $0 }" +FAILED=0 + +check() { + local name="$1" + shift + if "$@" >/dev/null 2>&1; then + echo "PASS $name" + else + echo "FAIL $name" + FAILED=1 + fi +} + +run() { docker run --rm --entrypoint "$1" "$IMAGE" "${@:2}"; } + +check "runs as UID 10001" test "$(run id -u)" = "10001" +check "CMD serves on port 8000" \ + sh -c "docker inspect -f '{{json .Config.Cmd}}' '$IMAGE' | grep -q '\"8000\"'" +check "cache and temp dirs are writable" run sh -c \ + 'mkdir -p "$HF_HOME" "$FASTEMBED_CACHE_PATH" "$NUMBA_CACHE_DIR" && touch "$HF_HOME/w" "$FASTEMBED_CACHE_PATH/w" "$NUMBA_CACHE_DIR/w" /tmp/w' +check "app code is read-only" run sh -c '! touch /app/bedhost/w' +check "app module imports" run python -c "import bedhost.main" +check "bigBedToBed is installed and runs" run sh -c 'bigBedToBed 2>&1 | grep -q "Convert from bigBed"' +check "no CUDA libraries (CPU-only torch)" run sh -c '! pip list 2>/dev/null | grep -qi "^nvidia-"' + +exit $FAILED diff --git a/tests/scripts/services.sh b/tests/scripts/services.sh index 197ded62..07c7ea40 100755 --- a/tests/scripts/services.sh +++ b/tests/scripts/services.sh @@ -50,8 +50,8 @@ start_postgres() { -e POSTGRES_PASSWORD="$DB_PASS" \ -e POSTGRES_DB="$DB_NAME" \ -p "${DB_PORT}:5432" \ - --tmpfs /var/lib/postgresql/data \ - postgres:17 + --tmpfs /var/lib/postgresql \ + pgvector/pgvector:pg18 # Wait for healthy status (up to 30 seconds) echo "Waiting for PostgreSQL to be ready..." diff --git a/tests/scripts/test-integration.sh b/tests/scripts/test-integration.sh index 20416ac7..07cbd69b 100755 --- a/tests/scripts/test-integration.sh +++ b/tests/scripts/test-integration.sh @@ -9,6 +9,14 @@ # Usage: ./tests/scripts/test-integration.sh [pytest args] # Example: ./tests/scripts/test-integration.sh -v -k "test_service_info" # +# ML models are off by default. To also exercise text search (downloads +# HuggingFace models on first use, needs a few GB of RAM): +# BEDHOST_INIT_ML=true ./tests/scripts/test-integration.sh +# If torch fails with a CUDA error (GPU too old for the installed build), +# force CPU with CUDA_VISIBLE_DEVICES= in front of that command. +# +# The /regions test also needs UCSC bigBedToBed on PATH (it skips otherwise). +# # For manual service control (debugging): # ./tests/scripts/services.sh start # RUN_INTEGRATION_TESTS=true pytest tests/integration/ @@ -87,7 +95,7 @@ SERVICES_STARTED=true export RUN_INTEGRATION_TESTS=true export TEST_DB_URL="postgresql+psycopg://testuser:testpass@localhost:${BEDHOST_TEST_DB_PORT}/bedbase_test" export TEST_QDRANT_URL="http://localhost:${BEDHOST_TEST_QDRANT_PORT}" -export BEDHOST_INIT_ML=false +export BEDHOST_INIT_ML="${BEDHOST_INIT_ML:-false}" # Run integration tests echo "" diff --git a/tests/test_regions_helper.py b/tests/test_regions_helper.py new file mode 100644 index 00000000..7da779e0 --- /dev/null +++ b/tests/test_regions_helper.py @@ -0,0 +1,83 @@ +"""Tests for the bigBedToBed subprocess helper (no running services needed). + +Covers ``run_bigbed_to_bed``: it waits for the child process (no zombie/fd +leak), enforces a timeout, surfaces non-zero exits, and trims output to the +first three BED columns, and stops the child once output passes a byte ceiling. +""" + +import subprocess +import sys + +import pytest + +from bedhost.helpers import RegionOutputTooLarge, run_bigbed_to_bed + +MAX = 1024 * 1024 + + +def _py_cmd(code: str) -> list[str]: + return [sys.executable, "-c", code] + + +def test_returns_first_three_columns(): + cmd = _py_cmd("import sys; sys.stdout.write('chr1\\t10\\t20\\tname\\t0\\t+\\n')") + result = run_bigbed_to_bed(cmd, timeout=5, max_bytes=MAX) + assert result == "chr1\t10\t20\n" + + +def test_multiple_lines(): + cmd = _py_cmd( + "import sys; sys.stdout.write(" + "'chr1\\t10\\t20\\tname\\t0\\t+\\n'" + "'chr1\\t30\\t40\\tname2\\t0\\t-\\n'" + ")" + ) + result = run_bigbed_to_bed(cmd, timeout=5, max_bytes=MAX) + assert result == "chr1\t10\t20\nchr1\t30\t40\n" + + +def test_empty_output_returns_empty_string(): + cmd = _py_cmd("pass") + result = run_bigbed_to_bed(cmd, timeout=5, max_bytes=MAX) + assert result == "" + + +def test_nonzero_exit_raises_called_process_error(): + cmd = _py_cmd("import sys; sys.exit(1)") + with pytest.raises(subprocess.CalledProcessError): + run_bigbed_to_bed(cmd, timeout=5, max_bytes=MAX) + + +def test_timeout_raises_timeout_expired(): + cmd = _py_cmd("import time; time.sleep(5)") + with pytest.raises(subprocess.TimeoutExpired): + run_bigbed_to_bed(cmd, timeout=1, max_bytes=MAX) + + +def test_missing_binary_raises_file_not_found_error(): + with pytest.raises(FileNotFoundError): + run_bigbed_to_bed( + ["definitely-not-a-real-binary-xyz"], timeout=5, max_bytes=MAX + ) + + +def test_output_over_ceiling_kills_child_and_raises(): + # Endless output: without the ceiling this would never finish. + cmd = _py_cmd( + "import sys\nwhile True: sys.stdout.write('chr1\\t10\\t20\\tname\\n')" + ) + with pytest.raises(RegionOutputTooLarge): + run_bigbed_to_bed(cmd, timeout=30, max_bytes=10_000) + + +def test_output_at_ceiling_is_allowed(): + cmd = _py_cmd("import sys; sys.stdout.write('chr1\\t10\\t20\\tname\\n' * 10)") + result = run_bigbed_to_bed(cmd, timeout=5, max_bytes=len("chr1\t10\t20\n") * 10) + assert result == "chr1\t10\t20\n" * 10 + + +def test_nonzero_exit_includes_stderr(): + cmd = _py_cmd("import sys; sys.stderr.write('boom'); sys.exit(2)") + with pytest.raises(subprocess.CalledProcessError) as exc: + run_bigbed_to_bed(cmd, timeout=5, max_bytes=MAX) + assert "boom" in exc.value.stderr diff --git a/tests/test_uploads.py b/tests/test_uploads.py new file mode 100644 index 00000000..9eee9576 --- /dev/null +++ b/tests/test_uploads.py @@ -0,0 +1,293 @@ +"""Tests for safe BED upload handling (no running services needed). + +Covers the three upload routes (/v1/bed/embed, /v1/bed/umap, +/v1/bed/search/bed): client filenames are never used as paths, uploads are +size-limited, and parse/validation errors map to clean 4xx responses. +""" + +import gzip +import io +import os +import tempfile +import uuid +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +pytest.importorskip("gtars") +pytest.importorskip("bedboss") +pytest.importorskip("bbconf") + +import numpy as np # noqa: E402 +from bbconf.models.bed_models import BedListSearchResult # noqa: E402 +from fastapi import FastAPI, HTTPException, UploadFile # noqa: E402 +from fastapi.middleware.cors import CORSMiddleware # noqa: E402 +from fastapi.testclient import TestClient # noqa: E402 +from gtars.models import RegionSet # noqa: E402 + +from bedhost import const # noqa: E402 +from bedhost.dependencies import get_bbagent # noqa: E402 +from bedhost.routers import bed_api # noqa: E402 +from bedhost.uploads import save_upload, upload_size_guard # noqa: E402 + +# Captured at import, before any test redirects tempfile.tempdir. +REAL_TMP = os.path.realpath(tempfile.gettempdir()) + +ROUTES = ["/v1/bed/embed", "/v1/bed/umap", "/v1/bed/search/bed"] + +VALID_BED = b"chr1\t100\t200\nchr1\t300\t450\nchr2\t1000\t1500\n" +NARROW_BED = b"chr1\t100\t101\nchr1\t200\t202\nchr2\t300\t303\n" + + +@pytest.fixture +def fake_agent(): + bed = MagicMock() + bed._embed_file.return_value = np.array([[0.1, 0.2]]) + bed._get_umap_file.return_value = np.array([[0.1, 0.2]]) + bed.bed_to_bed_search.return_value = BedListSearchResult( + count=0, limit=10, offset=0, results=[] + ) + config = SimpleNamespace( + qdrant_file_backend=object(), + r2v_encoder=object(), + umap_encoder=object(), + b2b_search_interface=object(), + ) + return SimpleNamespace(bed=bed, config=config) + + +def _build_app(fake_agent) -> FastAPI: + app = FastAPI() + app.include_router(bed_api.router) + # Same order as bedhost.main: guard first, then CORS (so CORS is outermost). + app.middleware("http")(upload_size_guard) + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], + ) + app.dependency_overrides[get_bbagent] = lambda: fake_agent + return app + + +@pytest.fixture +def client(fake_agent): + return TestClient(_build_app(fake_agent)) + + +@pytest.fixture +def upload_tmp(tmp_path, monkeypatch): + """Point tempfile at a dedicated, initially empty directory.""" + tt = tmp_path / "tt" + tt.mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(tt)) + return tt + + +def _agent_called(fake_agent) -> bool: + b = fake_agent.bed + return b._embed_file.called or b._get_umap_file.called or b.bed_to_bed_search.called + + +# --- path traversal ------------------------------------------------------- + + +EVIL_NAMES = [ + "../../{s}", + "../../../../tmp/{s}", + "{abs}", + "..\\..\\{s}", + "sentinel/../../{s}", +] + + +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize("pattern", EVIL_NAMES) +def test_client_filename_is_ignored(client, route, pattern, tmp_path, upload_tmp): + sentinel = f"sentinel_{uuid.uuid4().hex}.bed" + evil = pattern.format(s=sentinel, abs=str(tmp_path / sentinel)) + + res = client.post(route, files={"file": (evil, VALID_BED, "text/plain")}) + assert res.status_code == 200, res.text + + cwd = os.getcwd() + places = { + str(tmp_path), + str(upload_tmp), + os.path.dirname(str(upload_tmp)), + cwd, + os.path.dirname(cwd), + REAL_TMP, + os.path.dirname(REAL_TMP), + } + for place in places: + assert not os.path.exists(os.path.join(place, sentinel)), place + # Upload temp dir was cleaned up, and nothing sentinel-like was written. + assert list(upload_tmp.iterdir()) == [] + assert not list(tmp_path.rglob("sentinel_*")) + + +def test_save_upload_uses_fixed_name(tmp_path): + f = UploadFile(file=io.BytesIO(VALID_BED), filename="/etc/passwd") + assert save_upload(f, str(tmp_path)) == os.path.join(str(tmp_path), "upload.bed") + + +def test_save_upload_decompresses_gzip(tmp_path): + plain_dir = tmp_path / "plain" + gz_dir = tmp_path / "gz" + plain_dir.mkdir() + gz_dir.mkdir() + + plain = save_upload( + UploadFile(file=io.BytesIO(VALID_BED), filename="x.bed.gz"), str(plain_dir) + ) + gz = save_upload( + UploadFile(file=io.BytesIO(gzip.compress(VALID_BED)), filename="x.bed"), + str(gz_dir), + ) + assert plain == os.path.join(str(plain_dir), "upload.bed") + assert gz == os.path.join(str(gz_dir), "upload.bed") + assert open(gz, "rb").read() == VALID_BED + assert len(RegionSet(gz)) == len(RegionSet(plain)) == 3 + + +def test_save_upload_gzip_bomb_is_413(tmp_path): + # ~10 KB of gzip that expands to 10 MB: under the upload cap, over the + # (patched) decompressed cap. Must stop at the cap, not decompress it all. + bomb = gzip.compress(b"chr1\t100\t200\n" * (10 * 1024 * 1024 // 13)) + assert len(bomb) < 64 * 1024 + with pytest.raises(HTTPException) as exc: + save_upload( + UploadFile(file=io.BytesIO(bomb), filename="x.bed.gz"), + str(tmp_path), + max_uncompressed=1024 * 1024, + ) + assert exc.value.status_code == 413 + assert (tmp_path / "upload.bed").stat().st_size <= 1024 * 1024 + + +def test_save_upload_corrupt_gzip_is_415(tmp_path): + corrupt = gzip.compress(VALID_BED * 100)[:-20] + with pytest.raises(HTTPException) as exc: + save_upload(UploadFile(file=io.BytesIO(corrupt), filename="x"), str(tmp_path)) + assert exc.value.status_code == 415 + + +@pytest.mark.parametrize("route", ROUTES) +def test_gzip_bomb_rejected_by_routes( + client, fake_agent, route, monkeypatch, upload_tmp +): + monkeypatch.setattr(const, "MAX_UNCOMPRESSED_SIZE", 1024 * 1024) + bomb = gzip.compress(b"chr1\t100\t200\n" * (10 * 1024 * 1024 // 13)) + res = client.post(route, files={"file": ("a.bed.gz", bomb, "application/gzip")}) + assert res.status_code == 413, res.text + assert not _agent_called(fake_agent) + assert list(upload_tmp.iterdir()) == [] + + +def test_save_upload_refuses_existing_file(tmp_path): + (tmp_path / "upload.bed").write_bytes(b"keep") + with pytest.raises(FileExistsError): + save_upload(UploadFile(file=io.BytesIO(VALID_BED), filename="a"), str(tmp_path)) + assert (tmp_path / "upload.bed").read_bytes() == b"keep" + + +# --- size limits ---------------------------------------------------------- + + +@pytest.mark.parametrize("route", ROUTES) +def test_streaming_size_limit(client, fake_agent, route, monkeypatch, upload_tmp): + monkeypatch.setattr(const, "MAX_FILE_SIZE", 1024) + # Large slack so the middleware lets it through and streaming is what trips. + monkeypatch.setattr(const, "MULTIPART_OVERHEAD", 10 * 1024 * 1024) + body = VALID_BED * (2048 // len(VALID_BED) + 1) + res = client.post(route, files={"file": ("a.bed", body, "text/plain")}) + assert res.status_code == 413, res.text + assert not _agent_called(fake_agent) + assert list(upload_tmp.iterdir()) == [] + + +@pytest.mark.parametrize("route", ROUTES) +def test_content_length_guard(client, fake_agent, route): + claimed = const.MAX_FILE_SIZE + const.MULTIPART_OVERHEAD + 1 + res = client.post( + route, + content=b"x", + headers={ + "content-type": "multipart/form-data; boundary=x", + "content-length": str(claimed), + "origin": "https://bedbase.org", + }, + ) + assert res.status_code == 413, res.text + assert not _agent_called(fake_agent) + # Browser clients must still be able to read the error. + assert "access-control-allow-origin" in res.headers + + +# --- validation ----------------------------------------------------------- + + +@pytest.mark.parametrize("route", ROUTES) +def test_garbage_file_is_415(client, route): + res = client.post( + route, files={"file": ("a.bed", b"this is not a bed file\n", "text/plain")} + ) + assert res.status_code == 415, res.text + + +@pytest.mark.parametrize("route", ROUTES) +def test_empty_file_is_400(client, route): + res = client.post(route, files={"file": ("a.bed", b"", "text/plain")}) + assert res.status_code == 400, res.text + + +@pytest.mark.parametrize("route", ROUTES) +def test_missing_file_is_422(client, route): + res = client.post(route, data={"other": "x"}) + assert res.status_code == 422, res.text + + +@pytest.mark.parametrize("route", ROUTES) +def test_too_many_regions_is_413(client, route, monkeypatch): + monkeypatch.setattr(const, "MAX_REGION_NUMBER", 2) + res = client.post(route, files={"file": ("a.bed", VALID_BED, "text/plain")}) + assert res.status_code == 413, res.text + assert "2 regions" in res.json()["detail"] + + +def test_narrow_regions_rejected_only_for_search(client): + res = client.post( + "/v1/bed/search/bed", files={"file": ("a.bed", NARROW_BED, "text/plain")} + ) + assert res.status_code == 415, res.text + res = client.post( + "/v1/bed/embed", files={"file": ("a.bed", NARROW_BED, "text/plain")} + ) + assert res.status_code == 200, res.text + + +def test_valid_upload_returns_results(client, fake_agent): + res = client.post("/v1/bed/embed", files={"file": ("a.bed", VALID_BED)}) + assert res.status_code == 200 + assert res.json() == [0.1, 0.2] + res = client.post("/v1/bed/search/bed", files={"file": ("a.bed", VALID_BED)}) + assert res.status_code == 200 + assert res.json()["count"] == 0 + + +@pytest.mark.parametrize( + "route, attr", + [ + ("/v1/bed/embed", "r2v_encoder"), + ("/v1/bed/umap", "umap_encoder"), + ("/v1/bed/search/bed", "b2b_search_interface"), + ], +) +def test_ml_disabled_returns_503(fake_agent, route, attr): + setattr(fake_agent.config, attr, None) + client = TestClient(_build_app(fake_agent)) + res = client.post(route, files={"file": ("x.bed", NARROW_BED)}) + assert res.status_code == 503 diff --git a/ui/src/components/bed-splash-components/refgenome-modal.tsx b/ui/src/components/bed-splash-components/refgenome-modal.tsx index 7e796c31..be7b7a12 100644 --- a/ui/src/components/bed-splash-components/refgenome-modal.tsx +++ b/ui/src/components/bed-splash-components/refgenome-modal.tsx @@ -75,7 +75,7 @@ export const RefGenomeModal = (props: Props) => { ) .map((genome) => ( {
-

{convertStatusCodeToMessage(errorCode)}

+

{errorCode === 413 ? 'File too large' : convertStatusCodeToMessage(errorCode)}

{error.message && ( diff --git a/ui/wrangler.jsonc b/ui/wrangler.jsonc index 27c9aa8c..1327ffe1 100644 --- a/ui/wrangler.jsonc +++ b/ui/wrangler.jsonc @@ -1,5 +1,6 @@ { "name": "bedhost-ui", + "workers_dev": false, "main": "worker.ts", "compatibility_date": "2026-02-20", "assets": {