diff --git a/.dockerignore b/.dockerignore index 13fea92..583bd73 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,4 @@ tmp/* criteria structures CRITERIA_REF +kubernetes diff --git a/README.md b/README.md index dce1748..3d0a872 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,11 @@ complete and are not a legal assessment. private or link-local ranges are not retrieved. `DPPLINT_ALLOW_PRIVATE_NETWORKS=1` lifts this for local development. - API documentation: `/api-docs`. +- `GET /version` names the versions of dpplint, dpp-criteria and the SOyA + web-cli. `GET /ready` answers 200 when the instance can check passports + (the SOyA web-cli and the SOyA repository with the structures of the image + answer) and 503 with the reason when not; the result is held and renewed at + most every 30 s, every second while not ready. The image contains everything it needs at run time: the Rails API, the SOyA web-cli (`oydeu/soya-web-cli`) and the criteria and SOyA structures of @@ -146,15 +151,26 @@ for linux/amd64, like the SOyA web-cli it contains. ## Deployment `kubernetes/` holds the manifests for dpplint.ownyourdata.eu: deployment, -service, certificate (cert-manager, ClusterIssuer `letsencrypt-prod`) and ingress -(nginx). - -The deployment runs two replicas with 16 threads each (`RAILS_MAX_THREADS`). -Replicas keep no state of their own: every check is answered by one replica -from the criteria and structures in the image. A replica whose threads are all -busy fails its readiness probe (`/version`) and is taken out of the ingress -until it answers again; the liveness probe only checks that the port is open, so -a busy replica is not restarted. +pod disruption budget, service, certificate (cert-manager, ClusterIssuer +`letsencrypt-prod`) and ingress (nginx). + +The deployment runs two replicas with 16 threads each (`RAILS_MAX_THREADS`), +spread over different nodes where possible (`topologySpreadConstraints`, +`ScheduleAnyway`). The pod disruption budget keeps at least one replica +running while nodes are drained. Replicas keep no state of their own: every +check is answered by one replica from the criteria and structures in the +image. A replica whose threads are all busy, or whose SOyA web-cli does not +answer, fails its readiness probe (`/ready`) and is taken out of the ingress +until it answers again; the liveness probe only checks that the port is open, +so a busy replica is not restarted. + +The deployment names the image by its digest (`oydeu/dpplint@sha256:…`), so +that all replicas run the same image and a restart never picks up a newer +one. The digest of a new image gets into the manifest when the image is +pushed: `DOCKERHUB=1 ./build.sh` builds and pushes it and writes its digest +into `kubernetes/deployment.yaml`. The changed manifest is committed and +applied like any other change. `kubernetes/` is not part of the image, so +writing the digest does not change the image. ## Tests diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 9b65a85..22a1ae0 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -8,4 +8,12 @@ def version "soya-context": SoyaContext.new.version_info } end + + # For the readiness probe: 200 when this replica can check passports, 503 + # with the reason when the SOyA web-cli or its repository does not answer. + def ready + result = Readiness.current.status + body = result.ready ? { ready: true } : { ready: false, reason: result.reason } + render json: body, status: result.ready ? :ok : :service_unavailable + end end diff --git a/app/services/readiness.rb b/app/services/readiness.rb new file mode 100644 index 0000000..815291e --- /dev/null +++ b/app/services/readiness.rb @@ -0,0 +1,92 @@ +require "net/http" + +# Whether this replica can check passports: the SOyA web-cli and the local +# SOyA repository with the structures of the image (both started by +# docker/init.sh) answer on their ports. GET /ready reports it for the +# readiness probe. +# +# The result is held, so that the probe (every 5 s, 2 s time limit) stays +# cheap and starts no process: once both have answered, they are asked again +# at most every RECHECK seconds; until then at most every RETRY seconds. Each +# question is one local request with short time limits (at most 1.5 s for +# both together). Only one question runs at a time; a caller that comes while +# one runs gets the held result. +class Readiness + RECHECK = 30 + RETRY = 1 + OPEN_TIMEOUT = 0.25 + READ_TIMEOUT = 0.5 + + Result = Struct.new(:ready, :reason, keyword_init: true) + + NOT_CHECKED = Result.new(ready: false, reason: "not checked yet").freeze + + CURRENT_LOCK = Mutex.new + + def self.current = CURRENT_LOCK.synchronize { @current ||= new } + + def initialize(web_cli: nil, repository: nil, structures_dir: nil, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }) + config = Rails.configuration.x.dpplint + @web_cli = (web_cli || config.soya_web_cli).chomp("/") + @repository = (repository || config.soya_repository).chomp("/") + @structures_dir = structures_dir || config.structures_dir + @clock = clock + @lock = Mutex.new + @result = NOT_CHECKED + @checked_at = nil + end + + def status + if due? && @lock.try_lock + begin + @result = check + @checked_at = @clock.call + ensure + @lock.unlock + end + end + @result + end + + private + + def due? + return true if @checked_at.nil? + + @clock.call - @checked_at >= (@result.ready ? RECHECK : RETRY) + end + + def check + reason = web_cli_problem || repository_problem + reason ? Result.new(ready: false, reason: reason) : Result.new(ready: true) + end + + def web_cli_problem + res = request(Net::HTTP::Get, "#{@web_cli}/api/v1/version") + return "SOyA web-cli answered HTTP #{res.code}" unless res.is_a?(Net::HTTPSuccess) + return "SOyA web-cli answered without its version" unless res["Content-Type"].to_s.match?(/json/i) && JSON.parse(res.body)["version"].present? + + nil + rescue JSON::ParserError + "SOyA web-cli answered without its version" + rescue StandardError + "SOyA web-cli not reachable" + end + + def repository_problem + structure = Dir.exist?(@structures_dir) && Dir.children(@structures_dir).min + return "no SOyA structures in this image" unless structure + + res = request(Net::HTTP::Head, "#{@repository}/#{ERB::Util.url_encode(structure)}") + res.is_a?(Net::HTTPSuccess) ? nil : "SOyA repository answered HTTP #{res.code} for #{structure}" + rescue StandardError + "SOyA repository not reachable" + end + + def request(method, url) + uri = URI(url) + Net::HTTP.start(uri.host, uri.port, open_timeout: OPEN_TIMEOUT, read_timeout: READ_TIMEOUT, max_retries: 0) do |http| + http.request(method.new(uri.request_uri)) + end + end +end diff --git a/build.sh b/build.sh index de26704..c8a22dd 100755 --- a/build.sh +++ b/build.sh @@ -2,7 +2,8 @@ # Builds oydeu/dpplint (linux/amd64, like the SOyA web-cli it contains). # ./build.sh build with dpp-criteria main # ./build.sh build with a given commit or tag -# DOCKERHUB=1 ./build.sh build and push +# DOCKERHUB=1 ./build.sh build, push and write the pushed digest +# into kubernetes/deployment.yaml set -eu REF="${1:-main}" # A branch or tag is resolved to its commit, so that the Docker build cache @@ -14,4 +15,14 @@ echo "dpp-criteria: $REF" docker build --platform linux/amd64 --build-arg DPP_CRITERIA_REF="$REF" -f docker/Dockerfile -t oydeu/dpplint:latest . if [ "${DOCKERHUB:-0}" = "1" ]; then docker push oydeu/dpplint:latest + # The deployment runs the image by its digest, so that all replicas run the + # same image until the manifest names a new one. + DIGEST=$(docker inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' oydeu/dpplint:latest | + grep '^oydeu/dpplint@sha256:' | head -n 1) + [ -n "$DIGEST" ] || { echo "no digest for oydeu/dpplint:latest after the push" >&2; exit 1; } + sed -i.bak "s#image: oydeu/dpplint[@:].*#image: $DIGEST#" kubernetes/deployment.yaml + rm -f kubernetes/deployment.yaml.bak + grep -q "image: $DIGEST\$" kubernetes/deployment.yaml || + { echo "kubernetes/deployment.yaml does not name $DIGEST" >&2; exit 1; } + echo "kubernetes/deployment.yaml: image $DIGEST" fi diff --git a/config/initializers/dpplint.rb b/config/initializers/dpplint.rb index 1b485f6..f676e72 100644 --- a/config/initializers/dpplint.rb +++ b/config/initializers/dpplint.rb @@ -1,6 +1,8 @@ -# Locations and services dpplint depends on. All of them live inside the image, -# except didlint, which checks DIDs (DIDLINT_URL). A check waits for didlint at -# most DIDLINT_TIMEOUT seconds per request and DIDLINT_BUDGET seconds in total. +# Locations and services dpplint depends on. All of them live inside the image +# (docker/init.sh starts the SOyA web-cli and the SOyA repository it reads the +# structures from), except didlint, which checks DIDs (DIDLINT_URL). A check +# waits for didlint at most DIDLINT_TIMEOUT seconds per request and +# DIDLINT_BUDGET seconds in total. # SOYA_CONTEXT.json records which SOyA context the structures of the image were # built with (fetched or a cached copy, and from when). # @@ -10,6 +12,7 @@ # (default: CHECK_DEADLINE plus 5 s for the transfer, so 40). Rails.application.config.x.dpplint = ActiveSupport::OrderedOptions.new.tap do |c| c.soya_web_cli = ENV.fetch("SOYA_WEB_CLI", "http://127.0.0.1:8080") + c.soya_repository = ENV.fetch("SOYA_REPOSITORY", "http://127.0.0.1:9000") c.didlint = ENV.fetch("DIDLINT_URL", "https://didlint.ownyourdata.eu") c.didlint_timeout = ENV.fetch("DIDLINT_TIMEOUT", "10").to_f c.didlint_budget = ENV.fetch("DIDLINT_BUDGET", "20").to_f diff --git a/config/routes.rb b/config/routes.rb index 3d5bdab..7d240ef 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -11,5 +11,6 @@ root "home#show" get "version", to: "application#version" + get "ready", to: "application#ready" get "up", to: "rails/health#show", as: :rails_health_check end diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml index 32af619..444eb9f 100644 --- a/kubernetes/deployment.yaml +++ b/kubernetes/deployment.yaml @@ -17,10 +17,17 @@ spec: labels: app: dpplint spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: dpplint containers: - name: dpplint - image: oydeu/dpplint:latest - imagePullPolicy: Always + image: oydeu/dpplint@sha256:9a967537cfba81b96ad377bff375fa2ccfe63c7d72a97dbbf5cd22f08798a1ce + imagePullPolicy: IfNotPresent ports: - containerPort: 3000 env: @@ -40,7 +47,7 @@ spec: memory: 1Gi readinessProbe: httpGet: - path: /version + path: /ready port: 3000 initialDelaySeconds: 10 periodSeconds: 5 diff --git a/kubernetes/pdb.yaml b/kubernetes/pdb.yaml new file mode 100644 index 0000000..80b12fd --- /dev/null +++ b/kubernetes/pdb.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: dpplint +spec: + minAvailable: 1 + selector: + matchLabels: + app: dpplint diff --git a/swagger/v1/swagger.yaml b/swagger/v1/swagger.yaml index 88b2fed..dde3122 100644 --- a/swagger/v1/swagger.yaml +++ b/swagger/v1/swagger.yaml @@ -59,6 +59,18 @@ paths: responses: "200": description: Version information + /ready: + get: + summary: Whether this instance can check passports + description: >- + Used as readiness probe. Ready when the SOyA web-cli and the SOyA + repository with the structures of the image answer. The result is + held and renewed at most every 30 s (every second while not ready). + responses: + "200": + description: Ready + "503": + description: Not ready, with the reason components: schemas: Report: diff --git a/test/integration/ready_test.rb b/test/integration/ready_test.rb new file mode 100644 index 0000000..c664cba --- /dev/null +++ b/test/integration/ready_test.rb @@ -0,0 +1,43 @@ +require "test_helper" +require "minitest/mock" +require "socket" + +# GET /ready is the readiness probe of a replica; GET /version answers as +# before, whether or not the SOyA web-cli answers. +class ReadyTest < ActionDispatch::IntegrationTest + wait_for_web_cli + + def closed_url + tcp = TCPServer.new("127.0.0.1", 0) + port = tcp.addr[1] + tcp.close + "http://127.0.0.1:#{port}" + end + + test "ready: 200 when the SOyA web-cli and its repository answer" do + get "/ready" + assert_response :ok + assert_equal({ "ready" => true }, response.parsed_body) + end + + test "not ready: 503 with the reason when the SOyA web-cli does not answer" do + Readiness.stub(:current, Readiness.new(web_cli: closed_url)) do + get "/ready" + end + assert_response :service_unavailable + assert_equal({ "ready" => false, "reason" => "SOyA web-cli not reachable" }, response.parsed_body) + end + + test "version: 200 with the same fields, also when the SOyA web-cli does not answer" do + config = Rails.configuration.x.dpplint + previous = config.soya_web_cli + config.soya_web_cli = closed_url + get "/version" + assert_response :ok + body = response.parsed_body + assert_equal %w[service version dpp-criteria soya-web-cli soya-context], body.keys + assert_nil body["soya-web-cli"] + ensure + config.soya_web_cli = previous + end +end diff --git a/test/services/readiness_test.rb b/test/services/readiness_test.rb new file mode 100644 index 0000000..f8f11d5 --- /dev/null +++ b/test/services/readiness_test.rb @@ -0,0 +1,147 @@ +require "test_helper" +require "socket" +require "tmpdir" + +# Readiness of a replica: the SOyA web-cli and its repository answer. The +# result is held, so that the readiness probe stays cheap. +class ReadinessTest < ActiveSupport::TestCase + WEB_CLI_VERSION = SlowServer.http(200, '{"name":"soya-web-cli","version":"1.0.0"}', "Content-Type" => "application/json") + + class Clock + attr_accessor :now + + def initialize = @now = 100.0 + def call = @now + end + + setup do + @dir = Dir.mktmpdir + File.write(File.join(@dir, "DigitalProductPassport"), "{}") + File.write(File.join(@dir, "BatteryPassport"), "{}") + @clock = Clock.new + @servers = [] + end + + teardown do + @servers.each(&:stop) + FileUtils.remove_entry(@dir) + end + + def server(&handler) = SlowServer.new(&handler).tap { |s| @servers << s } + + def web_cli(answer = WEB_CLI_VERSION) = server { [0, answer] } + + def repository = server { [0, SlowServer.http(200)] } + + def closed_url + tcp = TCPServer.new("127.0.0.1", 0) + port = tcp.addr[1] + tcp.close + "http://127.0.0.1:#{port}" + end + + def readiness(web_cli_url, repository_url, dir: @dir) + Readiness.new(web_cli: web_cli_url, repository: repository_url, structures_dir: dir, clock: @clock) + end + + def timed + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + value = yield + [value, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started] + end + + test "web-cli and repository answer: ready, after one request to each" do + cli = web_cli + repo = repository + result = readiness(cli.url, repo.url).status + assert result.ready + assert_nil result.reason + assert_equal ["GET /api/v1/version HTTP/1.1"], cli.requests.map(&:first) + assert_equal ["HEAD /BatteryPassport HTTP/1.1"], repo.requests.map(&:first), "one structure of the image, the first by name" + end + + test "web-cli not running: not ready, and the repository is not asked" do + repo = repository + result = readiness(closed_url, repo.url).status + assert_not result.ready + assert_equal "SOyA web-cli not reachable", result.reason + assert_empty repo.requests + end + + test "web-cli answers with an error or without its version: not ready" do + assert_equal "SOyA web-cli answered HTTP 500", + readiness(web_cli(SlowServer.http(500, "Error")).url, repository.url).status.reason + assert_equal "SOyA web-cli answered without its version", + readiness(web_cli(SlowServer.http(200, "", "Content-Type" => "text/html")).url, repository.url).status.reason + assert_equal "SOyA web-cli answered without its version", + readiness(web_cli(SlowServer.http(200, "{", "Content-Type" => "application/json")).url, repository.url).status.reason + end + + test "web-cli hangs: the question ends well within the 2 s of the probe, and no request stays open" do + cli = server { [0, nil] } + result, took = timed { readiness(cli.url, repository.url).status } + assert_not result.ready + assert_equal "SOyA web-cli not reachable", result.reason + assert_operator took, :<, 1.0 + assert cli.all_given_up? + end + + test "repository missing, without the structure or without structures: not ready" do + assert_equal "SOyA repository not reachable", readiness(web_cli.url, closed_url).status.reason + assert_equal "SOyA repository answered HTTP 404 for BatteryPassport", + readiness(web_cli.url, server { [0, SlowServer.http(404)] }.url).status.reason + Dir.mktmpdir do |empty| + assert_equal "no SOyA structures in this image", readiness(web_cli.url, repository.url, dir: empty).status.reason + end + end + + test "ready: the result is held and the web-cli is asked again only after 30 s" do + cli = web_cli + check = readiness(cli.url, repository.url) + assert check.status.ready + @clock.now += 29 + assert check.status.ready + assert_equal 1, cli.requests.size + @clock.now += 1 + assert check.status.ready + assert_equal 2, cli.requests.size + end + + test "not ready: asked again at most every second, and ready once the web-cli answers" do + answers = [SlowServer.http(503, "starting"), SlowServer.http(503, "starting"), WEB_CLI_VERSION] + cli = server { [0, answers.shift || WEB_CLI_VERSION] } + check = readiness(cli.url, repository.url) + assert_not check.status.ready + @clock.now += 0.5 + assert_not check.status.ready + assert_equal 1, cli.requests.size, "no new request within a second" + @clock.now += 0.5 + assert_not check.status.ready + @clock.now += 1 + assert check.status.ready + assert_equal 3, cli.requests.size + end + + test "web-cli stops after it was ready: not ready at the next question" do + cli = web_cli + check = readiness(cli.url, repository.url) + assert check.status.ready + cli.stop + @clock.now += 30 + result = check.status + assert_not result.ready + assert_equal "SOyA web-cli not reachable", result.reason + end + + test "one question at a time: a caller during a question gets the held result at once" do + cli = server { [0.4, WEB_CLI_VERSION] } + check = readiness(cli.url, repository.url) + first = Thread.new { check.status } + sleep 0.1 + second, took = timed { check.status } + assert_equal "not checked yet", second.reason + assert_operator took, :<, 0.1 + assert first.value.ready + assert_equal 1, cli.requests.size + end +end