From 5a5ac8d459536c322fdc8fba7956bae8ea370087 Mon Sep 17 00:00:00 2001 From: Christoph Fabianek Date: Tue, 20 Oct 2026 12:54:10 +0200 Subject: [PATCH] Pin the image by digest, spread replicas, check SOyA for readiness The deployment names the image by its digest instead of latest with imagePullPolicy Always, so that all replicas run the same image and a restart never picks up a newer one. DOCKERHUB=1 ./build.sh writes the digest of the pushed image into kubernetes/deployment.yaml. kubernetes/ is no longer part of the image, so writing the digest does not change the image. The two replicas are spread over nodes where possible (topologySpreadConstraints over kubernetes.io/hostname, maxSkew 1, ScheduleAnyway). A pod disruption budget keeps at least one of them running while nodes are drained. GET /ready answers 200 when the SOyA web-cli and the SOyA repository with the structures of the image answer, and 503 with the reason when not; it is the readiness probe now. The result is held and renewed at most every 30 s, every second while not ready, with local requests of at most 0.75 s each, so the probe starts no process. GET /version is unchanged. --- .dockerignore | 1 + README.md | 34 +++-- app/controllers/application_controller.rb | 8 ++ app/services/readiness.rb | 92 ++++++++++++++ build.sh | 13 +- config/initializers/dpplint.rb | 9 +- config/routes.rb | 1 + kubernetes/deployment.yaml | 13 +- kubernetes/pdb.yaml | 9 ++ swagger/v1/swagger.yaml | 12 ++ test/integration/ready_test.rb | 43 +++++++ test/services/readiness_test.rb | 147 ++++++++++++++++++++++ 12 files changed, 366 insertions(+), 16 deletions(-) create mode 100644 app/services/readiness.rb create mode 100644 kubernetes/pdb.yaml create mode 100644 test/integration/ready_test.rb create mode 100644 test/services/readiness_test.rb diff --git a/.dockerignore b/.dockerignore index 13fea92..583bd73 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,4 @@ tmp/* criteria structures CRITERIA_REF +kubernetes diff --git a/README.md b/README.md index dce1748..3d0a872 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,11 @@ complete and are not a legal assessment. private or link-local ranges are not retrieved. `DPPLINT_ALLOW_PRIVATE_NETWORKS=1` lifts this for local development. - API documentation: `/api-docs`. +- `GET /version` names the versions of dpplint, dpp-criteria and the SOyA + web-cli. `GET /ready` answers 200 when the instance can check passports + (the SOyA web-cli and the SOyA repository with the structures of the image + answer) and 503 with the reason when not; the result is held and renewed at + most every 30 s, every second while not ready. The image contains everything it needs at run time: the Rails API, the SOyA web-cli (`oydeu/soya-web-cli`) and the criteria and SOyA structures of @@ -146,15 +151,26 @@ for linux/amd64, like the SOyA web-cli it contains. ## Deployment `kubernetes/` holds the manifests for dpplint.ownyourdata.eu: deployment, -service, certificate (cert-manager, ClusterIssuer `letsencrypt-prod`) and ingress -(nginx). - -The deployment runs two replicas with 16 threads each (`RAILS_MAX_THREADS`). -Replicas keep no state of their own: every check is answered by one replica -from the criteria and structures in the image. A replica whose threads are all -busy fails its readiness probe (`/version`) and is taken out of the ingress -until it answers again; the liveness probe only checks that the port is open, so -a busy replica is not restarted. +pod disruption budget, service, certificate (cert-manager, ClusterIssuer +`letsencrypt-prod`) and ingress (nginx). + +The deployment runs two replicas with 16 threads each (`RAILS_MAX_THREADS`), +spread over different nodes where possible (`topologySpreadConstraints`, +`ScheduleAnyway`). The pod disruption budget keeps at least one replica +running while nodes are drained. Replicas keep no state of their own: every +check is answered by one replica from the criteria and structures in the +image. A replica whose threads are all busy, or whose SOyA web-cli does not +answer, fails its readiness probe (`/ready`) and is taken out of the ingress +until it answers again; the liveness probe only checks that the port is open, +so a busy replica is not restarted. + +The deployment names the image by its digest (`oydeu/dpplint@sha256:…`), so +that all replicas run the same image and a restart never picks up a newer +one. The digest of a new image gets into the manifest when the image is +pushed: `DOCKERHUB=1 ./build.sh` builds and pushes it and writes its digest +into `kubernetes/deployment.yaml`. The changed manifest is committed and +applied like any other change. `kubernetes/` is not part of the image, so +writing the digest does not change the image. ## Tests diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 9b65a85..22a1ae0 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -8,4 +8,12 @@ def version "soya-context": SoyaContext.new.version_info } end + + # For the readiness probe: 200 when this replica can check passports, 503 + # with the reason when the SOyA web-cli or its repository does not answer. + def ready + result = Readiness.current.status + body = result.ready ? { ready: true } : { ready: false, reason: result.reason } + render json: body, status: result.ready ? :ok : :service_unavailable + end end diff --git a/app/services/readiness.rb b/app/services/readiness.rb new file mode 100644 index 0000000..815291e --- /dev/null +++ b/app/services/readiness.rb @@ -0,0 +1,92 @@ +require "net/http" + +# Whether this replica can check passports: the SOyA web-cli and the local +# SOyA repository with the structures of the image (both started by +# docker/init.sh) answer on their ports. GET /ready reports it for the +# readiness probe. +# +# The result is held, so that the probe (every 5 s, 2 s time limit) stays +# cheap and starts no process: once both have answered, they are asked again +# at most every RECHECK seconds; until then at most every RETRY seconds. Each +# question is one local request with short time limits (at most 1.5 s for +# both together). Only one question runs at a time; a caller that comes while +# one runs gets the held result. +class Readiness + RECHECK = 30 + RETRY = 1 + OPEN_TIMEOUT = 0.25 + READ_TIMEOUT = 0.5 + + Result = Struct.new(:ready, :reason, keyword_init: true) + + NOT_CHECKED = Result.new(ready: false, reason: "not checked yet").freeze + + CURRENT_LOCK = Mutex.new + + def self.current = CURRENT_LOCK.synchronize { @current ||= new } + + def initialize(web_cli: nil, repository: nil, structures_dir: nil, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }) + config = Rails.configuration.x.dpplint + @web_cli = (web_cli || config.soya_web_cli).chomp("/") + @repository = (repository || config.soya_repository).chomp("/") + @structures_dir = structures_dir || config.structures_dir + @clock = clock + @lock = Mutex.new + @result = NOT_CHECKED + @checked_at = nil + end + + def status + if due? && @lock.try_lock + begin + @result = check + @checked_at = @clock.call + ensure + @lock.unlock + end + end + @result + end + + private + + def due? + return true if @checked_at.nil? + + @clock.call - @checked_at >= (@result.ready ? RECHECK : RETRY) + end + + def check + reason = web_cli_problem || repository_problem + reason ? Result.new(ready: false, reason: reason) : Result.new(ready: true) + end + + def web_cli_problem + res = request(Net::HTTP::Get, "#{@web_cli}/api/v1/version") + return "SOyA web-cli answered HTTP #{res.code}" unless res.is_a?(Net::HTTPSuccess) + return "SOyA web-cli answered without its version" unless res["Content-Type"].to_s.match?(/json/i) && JSON.parse(res.body)["version"].present? + + nil + rescue JSON::ParserError + "SOyA web-cli answered without its version" + rescue StandardError + "SOyA web-cli not reachable" + end + + def repository_problem + structure = Dir.exist?(@structures_dir) && Dir.children(@structures_dir).min + return "no SOyA structures in this image" unless structure + + res = request(Net::HTTP::Head, "#{@repository}/#{ERB::Util.url_encode(structure)}") + res.is_a?(Net::HTTPSuccess) ? nil : "SOyA repository answered HTTP #{res.code} for #{structure}" + rescue StandardError + "SOyA repository not reachable" + end + + def request(method, url) + uri = URI(url) + Net::HTTP.start(uri.host, uri.port, open_timeout: OPEN_TIMEOUT, read_timeout: READ_TIMEOUT, max_retries: 0) do |http| + http.request(method.new(uri.request_uri)) + end + end +end diff --git a/build.sh b/build.sh index de26704..c8a22dd 100755 --- a/build.sh +++ b/build.sh @@ -2,7 +2,8 @@ # Builds oydeu/dpplint (linux/amd64, like the SOyA web-cli it contains). # ./build.sh build with dpp-criteria main # ./build.sh build with a given commit or tag -# DOCKERHUB=1 ./build.sh build and push +# DOCKERHUB=1 ./build.sh build, push and write the pushed digest +# into kubernetes/deployment.yaml set -eu REF="${1:-main}" # A branch or tag is resolved to its commit, so that the Docker build cache @@ -14,4 +15,14 @@ echo "dpp-criteria: $REF" docker build --platform linux/amd64 --build-arg DPP_CRITERIA_REF="$REF" -f docker/Dockerfile -t oydeu/dpplint:latest . if [ "${DOCKERHUB:-0}" = "1" ]; then docker push oydeu/dpplint:latest + # The deployment runs the image by its digest, so that all replicas run the + # same image until the manifest names a new one. + DIGEST=$(docker inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' oydeu/dpplint:latest | + grep '^oydeu/dpplint@sha256:' | head -n 1) + [ -n "$DIGEST" ] || { echo "no digest for oydeu/dpplint:latest after the push" >&2; exit 1; } + sed -i.bak "s#image: oydeu/dpplint[@:].*#image: $DIGEST#" kubernetes/deployment.yaml + rm -f kubernetes/deployment.yaml.bak + grep -q "image: $DIGEST\$" kubernetes/deployment.yaml || + { echo "kubernetes/deployment.yaml does not name $DIGEST" >&2; exit 1; } + echo "kubernetes/deployment.yaml: image $DIGEST" fi diff --git a/config/initializers/dpplint.rb b/config/initializers/dpplint.rb index 1b485f6..f676e72 100644 --- a/config/initializers/dpplint.rb +++ b/config/initializers/dpplint.rb @@ -1,6 +1,8 @@ -# Locations and services dpplint depends on. All of them live inside the image, -# except didlint, which checks DIDs (DIDLINT_URL). A check waits for didlint at -# most DIDLINT_TIMEOUT seconds per request and DIDLINT_BUDGET seconds in total. +# Locations and services dpplint depends on. All of them live inside the image +# (docker/init.sh starts the SOyA web-cli and the SOyA repository it reads the +# structures from), except didlint, which checks DIDs (DIDLINT_URL). A check +# waits for didlint at most DIDLINT_TIMEOUT seconds per request and +# DIDLINT_BUDGET seconds in total. # SOYA_CONTEXT.json records which SOyA context the structures of the image were # built with (fetched or a cached copy, and from when). # @@ -10,6 +12,7 @@ # (default: CHECK_DEADLINE plus 5 s for the transfer, so 40). Rails.application.config.x.dpplint = ActiveSupport::OrderedOptions.new.tap do |c| c.soya_web_cli = ENV.fetch("SOYA_WEB_CLI", "http://127.0.0.1:8080") + c.soya_repository = ENV.fetch("SOYA_REPOSITORY", "http://127.0.0.1:9000") c.didlint = ENV.fetch("DIDLINT_URL", "https://didlint.ownyourdata.eu") c.didlint_timeout = ENV.fetch("DIDLINT_TIMEOUT", "10").to_f c.didlint_budget = ENV.fetch("DIDLINT_BUDGET", "20").to_f diff --git a/config/routes.rb b/config/routes.rb index 3d5bdab..7d240ef 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -11,5 +11,6 @@ root "home#show" get "version", to: "application#version" + get "ready", to: "application#ready" get "up", to: "rails/health#show", as: :rails_health_check end diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml index 32af619..444eb9f 100644 --- a/kubernetes/deployment.yaml +++ b/kubernetes/deployment.yaml @@ -17,10 +17,17 @@ spec: labels: app: dpplint spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: dpplint containers: - name: dpplint - image: oydeu/dpplint:latest - imagePullPolicy: Always + image: oydeu/dpplint@sha256:9a967537cfba81b96ad377bff375fa2ccfe63c7d72a97dbbf5cd22f08798a1ce + imagePullPolicy: IfNotPresent ports: - containerPort: 3000 env: @@ -40,7 +47,7 @@ spec: memory: 1Gi readinessProbe: httpGet: - path: /version + path: /ready port: 3000 initialDelaySeconds: 10 periodSeconds: 5 diff --git a/kubernetes/pdb.yaml b/kubernetes/pdb.yaml new file mode 100644 index 0000000..80b12fd --- /dev/null +++ b/kubernetes/pdb.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: dpplint +spec: + minAvailable: 1 + selector: + matchLabels: + app: dpplint diff --git a/swagger/v1/swagger.yaml b/swagger/v1/swagger.yaml index 88b2fed..dde3122 100644 --- a/swagger/v1/swagger.yaml +++ b/swagger/v1/swagger.yaml @@ -59,6 +59,18 @@ paths: responses: "200": description: Version information + /ready: + get: + summary: Whether this instance can check passports + description: >- + Used as readiness probe. Ready when the SOyA web-cli and the SOyA + repository with the structures of the image answer. The result is + held and renewed at most every 30 s (every second while not ready). + responses: + "200": + description: Ready + "503": + description: Not ready, with the reason components: schemas: Report: diff --git a/test/integration/ready_test.rb b/test/integration/ready_test.rb new file mode 100644 index 0000000..c664cba --- /dev/null +++ b/test/integration/ready_test.rb @@ -0,0 +1,43 @@ +require "test_helper" +require "minitest/mock" +require "socket" + +# GET /ready is the readiness probe of a replica; GET /version answers as +# before, whether or not the SOyA web-cli answers. +class ReadyTest < ActionDispatch::IntegrationTest + wait_for_web_cli + + def closed_url + tcp = TCPServer.new("127.0.0.1", 0) + port = tcp.addr[1] + tcp.close + "http://127.0.0.1:#{port}" + end + + test "ready: 200 when the SOyA web-cli and its repository answer" do + get "/ready" + assert_response :ok + assert_equal({ "ready" => true }, response.parsed_body) + end + + test "not ready: 503 with the reason when the SOyA web-cli does not answer" do + Readiness.stub(:current, Readiness.new(web_cli: closed_url)) do + get "/ready" + end + assert_response :service_unavailable + assert_equal({ "ready" => false, "reason" => "SOyA web-cli not reachable" }, response.parsed_body) + end + + test "version: 200 with the same fields, also when the SOyA web-cli does not answer" do + config = Rails.configuration.x.dpplint + previous = config.soya_web_cli + config.soya_web_cli = closed_url + get "/version" + assert_response :ok + body = response.parsed_body + assert_equal %w[service version dpp-criteria soya-web-cli soya-context], body.keys + assert_nil body["soya-web-cli"] + ensure + config.soya_web_cli = previous + end +end diff --git a/test/services/readiness_test.rb b/test/services/readiness_test.rb new file mode 100644 index 0000000..f8f11d5 --- /dev/null +++ b/test/services/readiness_test.rb @@ -0,0 +1,147 @@ +require "test_helper" +require "socket" +require "tmpdir" + +# Readiness of a replica: the SOyA web-cli and its repository answer. The +# result is held, so that the readiness probe stays cheap. +class ReadinessTest < ActiveSupport::TestCase + WEB_CLI_VERSION = SlowServer.http(200, '{"name":"soya-web-cli","version":"1.0.0"}', "Content-Type" => "application/json") + + class Clock + attr_accessor :now + + def initialize = @now = 100.0 + def call = @now + end + + setup do + @dir = Dir.mktmpdir + File.write(File.join(@dir, "DigitalProductPassport"), "{}") + File.write(File.join(@dir, "BatteryPassport"), "{}") + @clock = Clock.new + @servers = [] + end + + teardown do + @servers.each(&:stop) + FileUtils.remove_entry(@dir) + end + + def server(&handler) = SlowServer.new(&handler).tap { |s| @servers << s } + + def web_cli(answer = WEB_CLI_VERSION) = server { [0, answer] } + + def repository = server { [0, SlowServer.http(200)] } + + def closed_url + tcp = TCPServer.new("127.0.0.1", 0) + port = tcp.addr[1] + tcp.close + "http://127.0.0.1:#{port}" + end + + def readiness(web_cli_url, repository_url, dir: @dir) + Readiness.new(web_cli: web_cli_url, repository: repository_url, structures_dir: dir, clock: @clock) + end + + def timed + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + value = yield + [value, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started] + end + + test "web-cli and repository answer: ready, after one request to each" do + cli = web_cli + repo = repository + result = readiness(cli.url, repo.url).status + assert result.ready + assert_nil result.reason + assert_equal ["GET /api/v1/version HTTP/1.1"], cli.requests.map(&:first) + assert_equal ["HEAD /BatteryPassport HTTP/1.1"], repo.requests.map(&:first), "one structure of the image, the first by name" + end + + test "web-cli not running: not ready, and the repository is not asked" do + repo = repository + result = readiness(closed_url, repo.url).status + assert_not result.ready + assert_equal "SOyA web-cli not reachable", result.reason + assert_empty repo.requests + end + + test "web-cli answers with an error or without its version: not ready" do + assert_equal "SOyA web-cli answered HTTP 500", + readiness(web_cli(SlowServer.http(500, "Error")).url, repository.url).status.reason + assert_equal "SOyA web-cli answered without its version", + readiness(web_cli(SlowServer.http(200, "", "Content-Type" => "text/html")).url, repository.url).status.reason + assert_equal "SOyA web-cli answered without its version", + readiness(web_cli(SlowServer.http(200, "{", "Content-Type" => "application/json")).url, repository.url).status.reason + end + + test "web-cli hangs: the question ends well within the 2 s of the probe, and no request stays open" do + cli = server { [0, nil] } + result, took = timed { readiness(cli.url, repository.url).status } + assert_not result.ready + assert_equal "SOyA web-cli not reachable", result.reason + assert_operator took, :<, 1.0 + assert cli.all_given_up? + end + + test "repository missing, without the structure or without structures: not ready" do + assert_equal "SOyA repository not reachable", readiness(web_cli.url, closed_url).status.reason + assert_equal "SOyA repository answered HTTP 404 for BatteryPassport", + readiness(web_cli.url, server { [0, SlowServer.http(404)] }.url).status.reason + Dir.mktmpdir do |empty| + assert_equal "no SOyA structures in this image", readiness(web_cli.url, repository.url, dir: empty).status.reason + end + end + + test "ready: the result is held and the web-cli is asked again only after 30 s" do + cli = web_cli + check = readiness(cli.url, repository.url) + assert check.status.ready + @clock.now += 29 + assert check.status.ready + assert_equal 1, cli.requests.size + @clock.now += 1 + assert check.status.ready + assert_equal 2, cli.requests.size + end + + test "not ready: asked again at most every second, and ready once the web-cli answers" do + answers = [SlowServer.http(503, "starting"), SlowServer.http(503, "starting"), WEB_CLI_VERSION] + cli = server { [0, answers.shift || WEB_CLI_VERSION] } + check = readiness(cli.url, repository.url) + assert_not check.status.ready + @clock.now += 0.5 + assert_not check.status.ready + assert_equal 1, cli.requests.size, "no new request within a second" + @clock.now += 0.5 + assert_not check.status.ready + @clock.now += 1 + assert check.status.ready + assert_equal 3, cli.requests.size + end + + test "web-cli stops after it was ready: not ready at the next question" do + cli = web_cli + check = readiness(cli.url, repository.url) + assert check.status.ready + cli.stop + @clock.now += 30 + result = check.status + assert_not result.ready + assert_equal "SOyA web-cli not reachable", result.reason + end + + test "one question at a time: a caller during a question gets the held result at once" do + cli = server { [0.4, WEB_CLI_VERSION] } + check = readiness(cli.url, repository.url) + first = Thread.new { check.status } + sleep 0.1 + second, took = timed { check.status } + assert_equal "not checked yet", second.reason + assert_operator took, :<, 0.1 + assert first.value.ready + assert_equal 1, cli.requests.size + end +end