Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
312 changes: 210 additions & 102 deletions .github/security/container-vulnerability-baseline.json

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ either registry; both names resolve to the same signed digest.

| Provider | GHCR | Docker Hub |
| --- | --- | --- |
| GetComics | `ghcr.io/pullboxapp/pullbox-provider-getcomics:1.0.3` | `docker.io/pullbox/pullbox-provider-getcomics:1.0.3` |
| GetComics | `ghcr.io/pullboxapp/pullbox-provider-getcomics:1.0.4` | `docker.io/pullbox/pullbox-provider-getcomics:1.0.4` |
| Anna's Archive | `ghcr.io/pullboxapp/pullbox-provider-annas-archive:1.0.4` | `docker.io/pullbox/pullbox-provider-annas-archive:1.0.4` |
| LibGen | `ghcr.io/pullboxapp/pullbox-provider-libgen:1.0.2` | `docker.io/pullbox/pullbox-provider-libgen:1.0.2` |

Expand Down
9 changes: 5 additions & 4 deletions docker/Dockerfile.annas-archive
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
ARG PYTHON_BASE=python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
ARG PYTHON_BASE=python:3.14.8-slim-trixie@sha256:a2b82f3c48559aa0a8446d9af49826b6e2b2016f4cd2afabfe6013ec53729170

FROM ${PYTHON_BASE} AS security-patched

RUN apt-get update && \
apt-get install --yes --no-install-recommends \
libssl3t64=3.5.7-1~deb13u2 \
openssl=3.5.7-1~deb13u2 \
openssl-provider-legacy=3.5.7-1~deb13u2 && \
libssl3t64=3.5.7-1~deb13u3 \
openssl=3.5.7-1~deb13u3 \
openssl-provider-legacy=3.5.7-1~deb13u3 \
libpcre2-8-0=10.46-1~deb13u3 && \
rm -rf /var/lib/apt/lists/*

FROM security-patched AS build
Expand Down
9 changes: 5 additions & 4 deletions docker/Dockerfile.getcomics
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
ARG PYTHON_BASE=python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
ARG PYTHON_BASE=python:3.14.8-slim-trixie@sha256:a2b82f3c48559aa0a8446d9af49826b6e2b2016f4cd2afabfe6013ec53729170

FROM ${PYTHON_BASE} AS security-patched

RUN apt-get update && \
apt-get install --yes --no-install-recommends \
libssl3t64=3.5.7-1~deb13u2 \
openssl=3.5.7-1~deb13u2 \
openssl-provider-legacy=3.5.7-1~deb13u2 && \
libssl3t64=3.5.7-1~deb13u3 \
openssl=3.5.7-1~deb13u3 \
openssl-provider-legacy=3.5.7-1~deb13u3 \
libpcre2-8-0=10.46-1~deb13u3 && \
rm -rf /var/lib/apt/lists/*

FROM security-patched AS build
Expand Down
9 changes: 5 additions & 4 deletions docker/Dockerfile.libgen
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
ARG PYTHON_BASE=python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
ARG PYTHON_BASE=python:3.14.8-slim-trixie@sha256:a2b82f3c48559aa0a8446d9af49826b6e2b2016f4cd2afabfe6013ec53729170

FROM ${PYTHON_BASE} AS security-patched

RUN apt-get update && \
apt-get install --yes --no-install-recommends \
libssl3t64=3.5.7-1~deb13u2 \
openssl=3.5.7-1~deb13u2 \
openssl-provider-legacy=3.5.7-1~deb13u2 && \
libssl3t64=3.5.7-1~deb13u3 \
openssl=3.5.7-1~deb13u3 \
openssl-provider-legacy=3.5.7-1~deb13u3 \
libpcre2-8-0=10.46-1~deb13u3 && \
rm -rf /var/lib/apt/lists/*

FROM security-patched AS build
Expand Down
9 changes: 5 additions & 4 deletions docker/Dockerfile.provider-smoke
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
ARG PYTHON_BASE=python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
ARG PYTHON_BASE=python:3.14.8-slim-trixie@sha256:a2b82f3c48559aa0a8446d9af49826b6e2b2016f4cd2afabfe6013ec53729170
FROM ${PYTHON_BASE} AS runtime

RUN apt-get update && \
apt-get install --yes --no-install-recommends \
libssl3t64=3.5.7-1~deb13u2 \
openssl=3.5.7-1~deb13u2 \
openssl-provider-legacy=3.5.7-1~deb13u2 && \
libssl3t64=3.5.7-1~deb13u3 \
openssl=3.5.7-1~deb13u3 \
openssl-provider-legacy=3.5.7-1~deb13u3 \
libpcre2-8-0=10.46-1~deb13u3 && \
rm -rf /var/lib/apt/lists/*

ENV PYTHONDONTWRITEBYTECODE=1 \
Expand Down
9 changes: 5 additions & 4 deletions docker/Dockerfile.synthetic
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
ARG PYTHON_BASE=python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
ARG PYTHON_BASE=python:3.14.8-slim-trixie@sha256:a2b82f3c48559aa0a8446d9af49826b6e2b2016f4cd2afabfe6013ec53729170

FROM ${PYTHON_BASE} AS security-patched

RUN apt-get update && \
apt-get install --yes --no-install-recommends \
libssl3t64=3.5.7-1~deb13u2 \
openssl=3.5.7-1~deb13u2 \
openssl-provider-legacy=3.5.7-1~deb13u2 && \
libssl3t64=3.5.7-1~deb13u3 \
openssl=3.5.7-1~deb13u3 \
openssl-provider-legacy=3.5.7-1~deb13u3 \
libpcre2-8-0=10.46-1~deb13u3 && \
rm -rf /var/lib/apt/lists/*

FROM security-patched AS build
Expand Down
8 changes: 4 additions & 4 deletions docker/compose.providers-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ services:
context: ..
dockerfile: docker/Dockerfile.getcomics
args:
PYTHON_BASE: public.ecr.aws/docker/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
PYTHON_BASE: public.ecr.aws/docker/library/python:3.14.8-slim-trixie@sha256:a2b82f3c48559aa0a8446d9af49826b6e2b2016f4cd2afabfe6013ec53729170
environment:
PULLBOX_PROVIDER_TOKEN: ${PULLBOX_PROVIDER_TOKEN:-test-provider-token-with-sufficient-entropy}
read_only: true
Expand All @@ -23,7 +23,7 @@ services:
context: ..
dockerfile: docker/Dockerfile.annas-archive
args:
PYTHON_BASE: public.ecr.aws/docker/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
PYTHON_BASE: public.ecr.aws/docker/library/python:3.14.8-slim-trixie@sha256:a2b82f3c48559aa0a8446d9af49826b6e2b2016f4cd2afabfe6013ec53729170
environment:
PULLBOX_PROVIDER_TOKEN: ${PULLBOX_PROVIDER_TOKEN:-test-provider-token-with-sufficient-entropy}
read_only: true
Expand All @@ -42,7 +42,7 @@ services:
context: ..
dockerfile: docker/Dockerfile.libgen
args:
PYTHON_BASE: public.ecr.aws/docker/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
PYTHON_BASE: public.ecr.aws/docker/library/python:3.14.8-slim-trixie@sha256:a2b82f3c48559aa0a8446d9af49826b6e2b2016f4cd2afabfe6013ec53729170
environment:
PULLBOX_PROVIDER_TOKEN: ${PULLBOX_PROVIDER_TOKEN:-test-provider-token-with-sufficient-entropy}
read_only: true
Expand All @@ -61,7 +61,7 @@ services:
context: ..
dockerfile: docker/Dockerfile.provider-smoke
args:
PYTHON_BASE: public.ecr.aws/docker/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
PYTHON_BASE: public.ecr.aws/docker/library/python:3.14.8-slim-trixie@sha256:a2b82f3c48559aa0a8446d9af49826b6e2b2016f4cd2afabfe6013ec53729170
depends_on:
getcomics:
condition: service_healthy
Expand Down
4 changes: 2 additions & 2 deletions docker/compose.synthetic-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ services:
dockerfile: docker/Dockerfile.synthetic
target: runtime
args:
PYTHON_BASE: ${PYTHON_BASE:-mirror.gcr.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d}
PYTHON_BASE: ${PYTHON_BASE:-mirror.gcr.io/library/python:3.14.8-slim-trixie@sha256:a2b82f3c48559aa0a8446d9af49826b6e2b2016f4cd2afabfe6013ec53729170}
image: pullbox-provider-synthetic:dd0-test
environment:
PULLBOX_PROVIDER_TOKEN: ${PULLBOX_PROVIDER_TOKEN:-synthetic-conformance-token-00000000}
Expand All @@ -26,7 +26,7 @@ services:
dockerfile: docker/Dockerfile.synthetic
target: conformance
args:
PYTHON_BASE: ${PYTHON_BASE:-mirror.gcr.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d}
PYTHON_BASE: ${PYTHON_BASE:-mirror.gcr.io/library/python:3.14.8-slim-trixie@sha256:a2b82f3c48559aa0a8446d9af49826b6e2b2016f4cd2afabfe6013ec53729170}
environment:
PULLBOX_PROVIDER_BASE_URL: http://synthetic:8780
PULLBOX_PROVIDER_TOKEN: ${PULLBOX_PROVIDER_TOKEN:-synthetic-conformance-token-00000000}
Expand Down
2 changes: 1 addition & 1 deletion providers/getcomics/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "hatchling.build"

[project]
name = "pullbox-provider-getcomics"
version = "1.0.3"
version = "1.0.4"
description = "Optional GetComics discovery provider for Pullbox"
requires-python = ">=3.12"
license = "GPL-3.0-or-later"
Expand Down
8 changes: 5 additions & 3 deletions providers/getcomics/src/pullbox_provider_getcomics/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

_SIZE = re.compile(r"\bSize\s*:\s*(\d+(?:\.\d+)?)\s*(KB|MB|GB)\b", re.IGNORECASE)
_QUALITY_VARIANT = re.compile(r"\s*\((?:HD|SD)[-\s]*Digital\)\s*", re.IGNORECASE)
_DOWNLOAD_SUFFIX = re.compile(r"\s+Free\s+Download\s*$", re.IGNORECASE)
_CONTROL_PREFIX = re.compile(
r"^(?:(?:DOWNLOAD NOW|READ ONLINE|PIXELDRAIN|MEGA|MEGANZ|ROOTZ|MEDIAFIRE|"
r"TERABOX|DATANODES|VIKINGFILE)\s+)+",
Expand Down Expand Up @@ -201,7 +202,7 @@ def parse_release_html(
continue
group_text = " ".join(group.text_parts)
title = _release_title(group.text_parts, source_url)
evidence = parse_comic_title(_QUALITY_VARIANT.sub(" ", title))
evidence = parse_comic_title(_DOWNLOAD_SUFFIX.sub("", _QUALITY_VARIANT.sub(" ", title)))
size_bytes = _parse_size(group_text)
artifacts.append(
Artifact(
Expand Down Expand Up @@ -317,9 +318,10 @@ def _release_title(text_parts: list[str], source_url: str) -> str:


def _looks_like_release_metadata(text: str) -> bool:
evidence = parse_comic_title(text)
evidence = parse_comic_title(_DOWNLOAD_SUFFIX.sub("", text))
return bool(
_SIZE.search(text)
_DOWNLOAD_SUFFIX.search(text)
or _SIZE.search(text)
or evidence.issue_numbers
or evidence.volume
or evidence.year is not None
Expand Down
15 changes: 8 additions & 7 deletions tests/unit/test_container_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def test_runtime_uses_pinned_python_314_and_non_root_identity() -> None:
dockerfile = DOCKERFILE.read_text(encoding="utf-8")

assert (
"python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d"
"python:3.14.8-slim-trixie@sha256:a2b82f3c48559aa0a8446d9af49826b6e2b2016f4cd2afabfe6013ec53729170"
in dockerfile
)
assert "AS build" in dockerfile
Expand Down Expand Up @@ -79,8 +79,8 @@ def test_source_provider_images_are_self_contained_hardened_python_314_services(
dockerfile = path.read_text(encoding="utf-8")

assert (
"python:3.14-slim-trixie@sha256:"
"51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d" in dockerfile
"python:3.14.8-slim-trixie@sha256:"
"a2b82f3c48559aa0a8446d9af49826b6e2b2016f4cd2afabfe6013ec53729170" in dockerfile
)
assert "USER 65532:65532" in dockerfile
assert "EXPOSE 8780" in dockerfile
Expand All @@ -95,13 +95,14 @@ def test_source_provider_images_are_self_contained_hardened_python_314_services(
assert all(source not in dockerfile for source in other_source_paths)


def test_runtime_images_install_pinned_openssl_security_update() -> None:
def test_runtime_images_install_pinned_openssl_and_pcre_security_updates() -> None:
for path in ALL_RUNTIME_DOCKERFILES:
dockerfile = path.read_text(encoding="utf-8")

assert "libssl3t64=3.5.7-1~deb13u2" in dockerfile
assert "openssl=3.5.7-1~deb13u2" in dockerfile
assert "openssl-provider-legacy=3.5.7-1~deb13u2" in dockerfile
assert "libssl3t64=3.5.7-1~deb13u3" in dockerfile
assert "openssl=3.5.7-1~deb13u3" in dockerfile
assert "openssl-provider-legacy=3.5.7-1~deb13u3" in dockerfile
assert "libpcre2-8-0=10.46-1~deb13u3" in dockerfile
assert "rm -rf /var/lib/apt/lists/*" in dockerfile


Expand Down
51 changes: 51 additions & 0 deletions tests/unit/test_getcomics_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,57 @@
FIXTURES = Path(__file__).parents[1] / "fixtures" / "getcomics"


@pytest.mark.parametrize("dash", ["-", "\u2013"])
def test_download_suffix_keeps_adjacent_pack_groups_separate(dash: str) -> None:
html = f"""<section class="post-contents">
<h1>House of M #1-8 (2005)</h1>
<p>House of M #1{dash}4 Free Download</p>
<a class="aio-red" href="https://files.example.test/first.cbz">DOWNLOAD NOW</a>
<p>House of M #5{dash}8 Free Download</p>
<a class="aio-red" href="https://files.example.test/second.cbz">DOWNLOAD NOW</a>
</section>"""
artifacts = parse_release_html(html, source_url="https://getcomics.org/marvel/pack/")
assert [a.coverage.issue_numbers for a in artifacts] == [
["1", "2", "3", "4"],
["5", "6", "7", "8"],
]
assert [len(a.mirrors) for a in artifacts] == [1, 1]


@pytest.mark.parametrize(
"heading",
[
"House of M Free Download",
"House of M #1-8 Free Download unavailable",
"House of M #1-8 and other comics Free Download",
],
)
def test_download_suffix_never_borrows_page_coverage(heading: str) -> None:
html = f"""<section class="post-contents"><h1>House of M #1-8 (2005)</h1>
<p>{heading}<br>Size : 400 MB</p>
<a class="aio-red" href="https://files.example.test/pack.cbz">DOWNLOAD NOW</a>
</section>"""
artifacts = parse_release_html(html, source_url="https://getcomics.org/marvel/pack/")
assert len(artifacts) == 1
assert artifacts[0].coverage.issue_numbers == []


@pytest.mark.parametrize(
"heading",
[
"House of M Free Download",
"House of M #1-8 and other comics Free Download",
],
)
def test_download_group_without_size_still_requires_its_own_issue_evidence(heading: str) -> None:
html = f"""<section class="post-contents"><h1>House of M #1-8 (2005)</h1>
<p>{heading}</p>
<a class="aio-red" href="https://files.example.test/pack.cbz">DOWNLOAD NOW</a>
</section>"""
artifacts = parse_release_html(html, source_url="https://getcomics.org/marvel/pack/")
assert artifacts[0].coverage.issue_numbers == []


def test_search_parser_normalizes_issue_and_collection_candidates() -> None:
candidates = parse_search_html(
(FIXTURES / "search-results.html").read_text(encoding="utf-8"),
Expand Down
41 changes: 41 additions & 0 deletions tests/unit/test_getcomics_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,47 @@
FIXTURES = Path(__file__).parents[1] / "fixtures" / "getcomics"


@pytest.mark.parametrize(
("series", "year", "count", "target"),
[("House of M", 2005, 8, n) for n in (3, 7)]
+ [("House of M \u2013 Masters of Evil", 2009, 4, n) for n in range(1, 5)],
)
async def test_house_of_m_search_and_resolve_preserve_pack_coverage(
series: str, year: int, count: int, target: int
) -> None:
async def pages(url: str, **_kwargs: object) -> str:
if urlsplit(url).query:
return f"""<h1 class="search-title">Search Result</h1>
<article class="post-title"><a href="https://getcomics.org/marvel/pack/">
{series} #1 \u2013 {count} ({year})</a></article>"""
return f"""<section class="post-contents">
<p><strong>{series} #1-{count} Free Download</strong><br>
Language : English | Year : {year} | Size : 400 MB</p>
<a class="aio-red" title="DOWNLOAD NOW"
href="https://files.example.test/pack.cbz">DOWNLOAD NOW</a>
<a class="aio-blue" title="MEGA"
href="https://mega.nz/file/pack#key">MEGA</a></section>"""

service = GetComicsProviderService(page_fetcher=pages)
candidates = await service.search(
SearchIntent(
series_title=series,
normalized_title=series.casefold(),
issue_number=str(target),
year=year,
),
limit=1,
)
expected = [str(n) for n in range(1, count + 1)]
assert candidates[0].parsed.issue_numbers == expected
artifacts = await service.resolve(candidates[0].provider_candidate_id)
assert len(artifacts) == 1
assert artifacts[0].coverage.issue_numbers == expected
assert artifacts[0].coverage.description == series
assert str(target) in artifacts[0].coverage.issue_numbers
assert [m.host_kind for m in artifacts[0].mirrors] == ["generic_https", "mega"]


class _Pages:
def __init__(self) -> None:
self.urls: list[str] = []
Expand Down
8 changes: 4 additions & 4 deletions tests/unit/test_provider_release_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ def test_security_release_versions_and_bundled_libgen_dependency_agree() -> None
name: tomllib.loads((ROOT / "providers" / name / "pyproject.toml").read_text())["project"]
for name in ("getcomics", "annas_archive", "libgen")
}
assert projects["getcomics"]["version"] == "1.0.3"
assert projects["getcomics"]["version"] == "1.0.4"
assert projects["annas_archive"]["version"] == "1.0.4"
assert projects["libgen"]["version"] == "1.0.2"
assert "pullbox-provider-libgen==1.0.2" in projects["annas_archive"]["dependencies"]
Expand All @@ -43,12 +43,12 @@ def _load_latest_module():
def test_tag_release_maps_getcomics_to_both_registry_names() -> None:
release = _load_module().resolve_release(
repository_owner="pullboxapp",
tag="getcomics-v1.0.3",
tag="getcomics-v1.0.4",
)

assert release.provider == "getcomics"
assert release.version == "1.0.3"
assert release.release_tag == "getcomics-v1.0.3"
assert release.version == "1.0.4"
assert release.release_tag == "getcomics-v1.0.4"
assert release.is_release is True
assert release.is_prerelease is False
assert release.dockerfile == "docker/Dockerfile.getcomics"
Expand Down
Loading
Loading