Distrust compromised eMudhra certificate chains

.dockerignore

@@ -36,6 +36,8 @@
 
 # Mix artifacts
 /_build/
+**/_build/
+/vendor/certifi/ebin/
 /deps/
 *.ez
 

.github/workflows/security.yml

@@ -37,7 +37,7 @@ jobs:
           trap 'sudo rm -rf "$artifacts"' EXIT
           docker run --rm \
             -v "$PWD/deploy/encoding-booster/encoder/sandbox/broker:/src:ro" \
-            -v "$artifacts:/out" -w /src golang:1.27.2-alpine3.23 \
+            -v "$artifacts:/out" -w /src golang:1.27.3-alpine3.23 \
             sh -c 'go test ./... && go vet ./... && CGO_ENABLED=0 go build -trimpath -o /out/mave-media-broker .'
           cc -O2 -Wall -Wextra -Werror deploy/encoding-booster/encoder/sandbox/media_sandbox.c \
             -lseccomp -o "$artifacts/mave-media-sandbox"

Dockerfile

@@ -26,7 +26,7 @@ ARG RUNNER_IMAGE="docker.io/ubuntu:26.04@sha256:da6fc2be547864451aa253836dd926da
 
 FROM node:24.21.0-bookworm-slim@sha256:0e0ff40c39bc087845bfb27465a0df4ea419520094bc35842ff83dd8cbe6f9b6 AS node_build
 
-FROM golang:1.27.2-alpine3.23 AS media_broker_build
+FROM golang:1.27.3-alpine3.23 AS media_broker_build
 WORKDIR /src
 COPY deploy/encoding-booster/encoder/sandbox/broker/ ./
 RUN go test ./... && CGO_ENABLED=0 go build -trimpath -o /mave-media-broker .
@@ -54,6 +54,9 @@ RUN apt-get update \
     pkg-config \
     xz-utils \
     zlib1g-dev \
+  && sed -i -e '/^mozilla\/emSign_ECC_Root_CA_-_G3.crt$/s/^/!/' \
+    -e '/^mozilla\/Comodo_AAA_Services_root.crt$/s/^/!/' /etc/ca-certificates.conf \
+  && update-ca-certificates --fresh \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
@@ -105,7 +108,10 @@ FROM ${BUILDER_IMAGE} AS builder_base
 
 # install build dependencies
 RUN apt-get update \
-  && apt-get install -y --no-install-recommends build-essential git cargo rustc libseccomp-dev \
+  && apt-get install -y --no-install-recommends build-essential git cargo rustc libseccomp-dev ca-certificates \
+  && sed -i -e '/^mozilla\/emSign_ECC_Root_CA_-_G3.crt$/s/^/!/' \
+    -e '/^mozilla\/Comodo_AAA_Services_root.crt$/s/^/!/' /etc/ca-certificates.conf \
+  && update-ca-certificates --fresh \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
@@ -120,7 +126,7 @@ RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \
 WORKDIR /app
 
 # Disable Erlang JIT (Fixes QEMU crash during cross-compile)
-ENV ERL_AFLAGS="+JMsingle true"
+ENV ERL_AFLAGS="+JMsingle true" NODE_OPTIONS="--use-openssl-ca"
 
 # install hex + rebar
 RUN mix local.hex --force \
@@ -142,6 +148,7 @@ FROM builder_base AS builder
 
 # install mix dependencies
 COPY mix.exs mix.lock ./
+COPY vendor vendor
 RUN mix deps.get --only $MIX_ENV
 RUN mkdir config
 
@@ -184,6 +191,9 @@ ENV MIX_ENV="dev" MAVE_MEDIA_SANDBOX="required"
 
 RUN apt-get update \
   && apt-get install -y --no-install-recommends ffmpeg procps inotify-tools \
+  && sed -i -e '/^mozilla\/emSign_ECC_Root_CA_-_G3.crt$/s/^/!/' \
+    -e '/^mozilla\/Comodo_AAA_Services_root.crt$/s/^/!/' /etc/ca-certificates.conf \
+  && update-ca-certificates --fresh \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* \
   && ffmpeg -version \
@@ -223,6 +233,9 @@ RUN apt-get update \
     procps \
     tini \
     zlib1g \
+  && sed -i -e '/^mozilla\/emSign_ECC_Root_CA_-_G3.crt$/s/^/!/' \
+    -e '/^mozilla\/Comodo_AAA_Services_root.crt$/s/^/!/' /etc/ca-certificates.conf \
+  && update-ca-certificates --fresh \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 

deploy/encoding-booster/encoder/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.27.2-alpine3.23 AS app-build
+FROM golang:1.27.3-alpine3.23 AS app-build
 WORKDIR /src
 COPY go.mod *.go ./
 COPY assets/play.png ./assets/play.png
@@ -73,7 +73,10 @@ RUN apk add --no-cache \
       libwebpmux \
       x264-libs \
       x265-libs \
-      zlib
+      zlib \
+  && sed -i -e '/^mozilla\/emSign_ECC_Root_CA_-_G3.crt$/s/^/!/' \
+    -e '/^mozilla\/Comodo_AAA_Services_root.crt$/s/^/!/' /etc/ca-certificates.conf \
+  && update-ca-certificates --fresh
 COPY --from=ffmpeg-build /opt/ffmpeg /opt/ffmpeg
 ENV PATH="/opt/ffmpeg/bin:${PATH}"
 RUN command -v ffmpeg | grep -q '^/opt/ffmpeg/bin/ffmpeg$' \

deploy/encoding-booster/encoder/Dockerfile.gpu

@@ -1,4 +1,4 @@
-FROM golang:1.27.2-alpine3.23 AS app-build
+FROM golang:1.27.3-alpine3.23 AS app-build
 
 WORKDIR /src
 COPY go.mod *.go ./
@@ -60,6 +60,9 @@ RUN apt-get update \
     && apt-get install -y --no-install-recommends \
       ca-certificates libdav1d7 libgnutls30t64 libmp3lame0 libopus0 libseccomp2 \
       libsvtav1enc1d1 libwebp7 libwebpmux3 libx264-164 libx265-199 zlib1g \
+    && sed -i -e '/^mozilla\/emSign_ECC_Root_CA_-_G3.crt$/s/^/!/' \
+    -e '/^mozilla\/Comodo_AAA_Services_root.crt$/s/^/!/' /etc/ca-certificates.conf \
+    && update-ca-certificates --fresh \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 

deploy/licenses/collect.sh

@@ -27,7 +27,7 @@ cp -R deploy/licenses/device-detector "$output_dir/data/"
 
 # Keep original relative paths so identically named notices cannot overwrite
 # each other. Include transitive dependencies, not just direct package names.
-find deps assets/node_modules \
+find deps vendor assets/node_modules \
   -type d \( -name .git -o -name _build -o -name target \) -prune -o \
   -type f \( -iname 'license*' -o -iname 'licence*' -o -iname 'copying*' \
     -o -iname 'notice*' -o -iname 'copyright*' -o -name Cargo.lock \

deploy/minio/Dockerfile

@@ -1,15 +1,18 @@
 # The community images are no longer publicly pullable. Build the same pinned
 # releases from upstream source; Go's checksum database verifies module contents.
-FROM golang:1.27.2-alpine3.23 AS server-build
+FROM golang:1.27.3-alpine3.23 AS server-build
 ENV CGO_ENABLED=0 GOTOOLCHAIN=local
 RUN go install -trimpath github.com/minio/minio@v0.0.0-20250907161309-07c3a429bfed
 
-FROM golang:1.27.2-alpine3.23 AS client-build
+FROM golang:1.27.3-alpine3.23 AS client-build
 ENV CGO_ENABLED=0 GOTOOLCHAIN=local
 RUN go install -trimpath github.com/minio/mc@v0.0.0-20250813083541-7394ce0dd2a8
 
 FROM alpine:3.23 AS base
-RUN apk add --no-cache ca-certificates curl
+RUN apk add --no-cache ca-certificates curl \
+  && sed -i -e '/^mozilla\/emSign_ECC_Root_CA_-_G3.crt$/s/^/!/' \
+    -e '/^mozilla\/Comodo_AAA_Services_root.crt$/s/^/!/' /etc/ca-certificates.conf \
+  && update-ca-certificates --fresh
 
 FROM base AS client
 LABEL org.opencontainers.image.source="https://github.com/minio/mc" \

mix.exs

@@ -74,6 +74,9 @@ defmodule MaveCore.MixProject do
       {:finch, "~> 0.24.0"},
       # Keep consuming applications above the patched HTTP/1 and HTTP/2 baseline.
       {:mint, ">= 1.11.0 and < 2.0.0"},
+      {:castore, "~> 1.0.22"},
+      # Temporary CA distrust until upstream certifi ships the removal.
+      {:certifi, path: "vendor/certifi", override: true},
       {:req_s3, "~> 0.2.5"},
       {:telemetry_metrics, "~> 1.0"},
       {:telemetry_poller, "~> 1.0"},

mix.lock

@@ -2,8 +2,8 @@
   "bandit": {:hex, :bandit, "1.13.1", "1ce25a7beb750ed3fa2e905962f1f9a679cb585f1da0ae45637d06e385ce2eae", [:mix], [{:hpax, "~> 1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.6", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "39028f0b5a21404661410a6c19275060a9822c9dcca70b7c3e4e4fa50b11da58"},
   "broadway": {:hex, :broadway, "1.2.1", "83a1567423c26885e15f6cd8670ca790370af2fcff2ede7fa88c5ea793087a67", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.7 or ~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "68ae63d83b55bdca0f95cd49feee5fb74c5a6bec557caf940860fe07dbc8a4fb"},
   "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
+  "castore": {:hex, :castore, "1.0.22", "a01978e6a5d42fa3a34143630588f92036e7d90e2309ecdd466722d1279a2213", [:mix], [], "hexpm", "c2a4d42723b4affbc1c3e15d8f6df3f7d8f1f627c84de643383634cce58493d5"},
   "cc_precompiler": {:hex, :cc_precompiler, "0.1.11", "8c844d0b9fb98a3edea067f94f616b3f6b29b959b6b3bf25fee94ffe34364768", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "3427232caf0835f94680e5bcf082408a70b48ad68a5f5c0b02a3bea9f3a075b9"},
-  "certifi": {:hex, :certifi, "2.17.0", "835748414307e15e05b17d0e518190228ce648b08d569a5cc93a85a40f3e5c9b", [:rebar3], [], "hexpm", "8122798a17f0293c80daada25d0f81c7f4d708c73fef782c7c9b1950e26e4d21"},
   "ch": {:hex, :ch, "0.8.3", "c23a8200e63a430c833848f92b31b85ec235fe406f38f532d31144152293b974", [:mix], [{:db_connection, "~> 2.10.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.13.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "3666babdd2130c3731422b7387b6d7a591f501681e3b840c49d44207772ca954"},
   "cors_plug": {:hex, :cors_plug, "3.0.3", "7c3ac52b39624bc616db2e937c282f3f623f25f8d550068b6710e58d04a0e330", [:mix], [{:plug, "~> 1.13", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "3f2d759e8c272ed3835fab2ef11b46bddab8c1ab9528167bd463b6452edf830d"},
   "credo": {:hex, :credo, "1.7.18", "5c5596bf7aedf9c8c227f13272ac499fe8eae6237bd326f2f07dfc173786f042", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a189d164685fd945809e862fe76a7420c4398fa288d76257662aecb909d6b3e5"},

scripts/check_certificate_trust.exs

@@ -0,0 +1,69 @@
+# Run inside the candidate image/release environment, without starting Mave:
+#   mix run --no-start scripts/check_certificate_trust.exs
+#   mix run --no-start scripts/check_certificate_trust.exs --network
+# External probes are opt-in and require the incident test service to be online.
+
+blocked = ~w(
+  86A1ECBA089C4A8D3BBE2734C612BA341D813E043CF9E8A862CD5C57A36BBE6B
+  D7A7A0FB5D7E2731D771E9484EBCDEF71D5F0C3E0A2948782BC83EE0EA699EF4
+  7DD78D5F4F13459A83DFF9ABBBA62EDBAF6F2D102BF257FD712F4D9F2746ED8D
+  F2EDAAFF3D9B701FB4D14C9F285F8C0BAD90FC3BCC1FA07D47F4718EDABA34A5
+)
+
+read_pem = fn path ->
+  for {:Certificate, der, :not_encrypted} <- :public_key.pem_decode(File.read!(path)),
+      do: der
+end
+
+# OTP 29 exposes the DER as the second member of each combined certificate.
+system_certs = Enum.map(:public_key.cacerts_get(), &elem(&1, 1))
+
+for {name, certs} <- [
+      {"certifi compiled", :certifi.cacerts()},
+      {"certifi PEM", read_pem.(:certifi.cacertfile())},
+      {"CAStore PEM", read_pem.(CAStore.file_path())},
+      {"OS/OTP", system_certs}
+    ] do
+  fingerprints = Enum.map(certs, &Base.encode16(:crypto.hash(:sha256, &1)))
+
+  if Enum.any?(blocked, &(&1 in fingerprints)) or length(fingerprints) < 100 do
+    raise "Unsafe or incomplete trust store: #{name}"
+  end
+
+  IO.puts("PASS #{name}: affected roots absent")
+end
+
+if "--network" in System.argv() do
+  {:ok, _} = Application.ensure_all_started(:req)
+  {:ok, _} = Application.ensure_all_started(:hackney)
+
+  clients = [
+    {"Req/Finch", fn url -> Req.get(url, retry: false, receive_timeout: 10_000) end},
+    {"Hackney",
+     fn url ->
+       case :hackney.request(:get, url, [], "", connect_timeout: 10_000, with_body: true) do
+         {:ok, status, _headers, _body} -> {:ok, status}
+         {:error, _reason} = error -> error
+       end
+     end}
+  ]
+
+  for {name, request} <- clients do
+    {:ok, _response} = request.("https://example.com/")
+    IO.puts("PASS #{name}: ordinary HTTPS works")
+
+    for kind <- ["dv", "ov", "ev"], region <- ["g3b", "g3c"], cross_sign <- ["", "-aaa"] do
+      host = "#{kind}-#{region}#{cross_sign}-emudhra.we-have-a-problem.com"
+
+      case request.("https://#{host}/") do
+        {:error, reason} ->
+          # Timeouts, DNS errors and expired certificates are not proof of distrust.
+          unless inspect(reason) =~ "unknown_ca", do: raise("Inconclusive TLS rejection: #{host}")
+          IO.puts("PASS #{name}: #{host} rejected (unknown CA)")
+
+        {:ok, _response} ->
+          raise "Compromised certificate accepted: #{name}, #{host}"
+      end
+    end
+  end
+end

test/mave_core/certificate_trust_test.exs

@@ -0,0 +1,41 @@
+defmodule MaveCore.CertificateTrustTest do
+  use ExUnit.Case, async: true
+
+  # The incident includes cross-signs through AAA and the older emSign G3 root.
+  @distrusted ~w(
+    86A1ECBA089C4A8D3BBE2734C612BA341D813E043CF9E8A862CD5C57A36BBE6B
+    D7A7A0FB5D7E2731D771E9484EBCDEF71D5F0C3E0A2948782BC83EE0EA699EF4
+    7DD78D5F4F13459A83DFF9ABBBA62EDBAF6F2D102BF257FD712F4D9F2746ED8D
+    F2EDAAFF3D9B701FB4D14C9F285F8C0BAD90FC3BCC1FA07D47F4718EDABA34A5
+  )
+  @isrg_root_x1 "96BCEC06264976F37460779ACF28C5A7CFE8A3C0AAE11A8FFCEE05C0BDDF08C6"
+
+  test "certifi's compiled trust matches its PEM and rejects compromised CA paths" do
+    embedded = MapSet.new(:certifi.cacerts())
+    pem = :certifi.cacertfile() |> read_certificates() |> MapSet.new()
+
+    assert embedded == pem
+    assert_safe_trust(embedded)
+  end
+
+  test "CAStore rejects the compromised roots while retaining ordinary HTTPS trust" do
+    CAStore.file_path() |> read_certificates() |> assert_safe_trust()
+  end
+
+  defp read_certificates(path) do
+    for {:Certificate, der, :not_encrypted} <- :public_key.pem_decode(File.read!(path)),
+        do: der
+  end
+
+  defp assert_safe_trust(certificates) do
+    fingerprints =
+      MapSet.new(certificates, fn der -> Base.encode16(:crypto.hash(:sha256, der)) end)
+
+    for fingerprint <- @distrusted do
+      refute MapSet.member?(fingerprints, fingerprint)
+    end
+
+    assert MapSet.member?(fingerprints, @isrg_root_x1)
+    assert MapSet.size(fingerprints) > 100
+  end
+end

test/mave_core/distribution_notices_test.exs

@@ -19,6 +19,7 @@ defmodule MaveCore.DistributionNoticesTest do
       "deploy/licenses/device-detector/LICENSE" => "Data licence",
       "deploy/licenses/device-detector/NOTICE" => "Data attribution",
       "deps/example/LICENSE.md" => "Example licence",
+      "vendor/certifi/LICENSE" => "Vendored CA bundle licence",
       "deps/example/hex_metadata.config" => "Package metadata",
       "deps/example/native/Cargo.lock" => "Native dependency versions",
       "deps/example/third party/NOTICE" => "Nested attribution",
@@ -45,6 +46,7 @@ defmodule MaveCore.DistributionNoticesTest do
           "THIRD_PARTY_NOTICES.md",
           "Dockerfile",
           "deps/example/LICENSE.md",
+          "vendor/certifi/LICENSE",
           "deps/example/hex_metadata.config",
           "deps/example/native/Cargo.lock",
           "deps/example/third party/NOTICE",

vendor/certifi/.gitignore

@@ -0,0 +1,2 @@
+/_build/
+/ebin/

vendor/certifi/LICENSE

@@ -0,0 +1,29 @@
+Copyright (c) 2015-2025, Benoit Chesneau <bchesneau@gmail.com>.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+* The names of its contributors may not be used to endorse or promote
+  products derived from this software without specific prior written
+  permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/certifi/PATCH.md

@@ -0,0 +1,25 @@
+# Temporary certificate distrust
+
+This directory vendors the `certifi` Hex package version 2.17.0. Its Erlang
+implementation and public API are unchanged. The application version is marked
+`2.17.0-mave.1` to distinguish this build from the upstream release.
+
+The sole trust-store change removes **emSign ECC Root CA - G3** (SHA-256
+`86A1ECBA089C4A8D3BBE2734C612BA341D813E043CF9E8A862CD5C57A36BBE6B`)
+following the October 2026 intermediate-key compromise:
+<https://emudhra.we-have-a-problem.com/>.
+
+The other affected roots, including AAA Certificate Services, are already absent
+from this upstream bundle. All other certificates are preserved. The original
+BSD license and the Mozilla bundle's license notice are retained.
+
+Vendoring is necessary because `certifi_pt` embeds the bundle into
+`certifi:cacerts/0` during compilation. Editing a PEM file after compilation or
+changing only an HTTP client's configuration does not protect every caller.
+Both the compiled list and `certifi:cacertfile/0` must use the filtered bundle.
+
+Remove this override once an upstream release contains the same distrust and
+passes `test/mave_core/certificate_trust_test.exs`. Consuming applications that
+also depend directly on HTTP clients may need the same top-level path override.
+Rebuild releases and restart them; existing processes retain their old trust
+stores and connections.