From f9227907f2d78809985a3ef37b52def15f057c90 Mon Sep 17 00:00:00 2001 From: Christoph Fabianek Date: Sun, 25 Oct 2026 13:51:58 +0100 Subject: [PATCH] Check unique elementIds and device functions in the HTML view element_ids (DPP-DAT-017, DPP-DAT-021): in the expanded full serialization, elementIds are compared within each array of data elements, or across the whole passport with the items of a MultiValuedDataElement counted as one place. In the compressed serialization the level check reads the passport as delivered (the response, or the posted body) and reports member names repeated within one JSON object; the passport-wide check is skipped there. html with device_features: forbidden (DPP-SEC-021): without running any script, the inline scripts, event handler attributes and javascript: URLs of the HTML view and the scripts it loads from its own host are searched for browser interfaces that use functions of the reader's device, and the markup for frames granted device permissions, file inputs with capture and permission elements. Test vectors for the three criteria under test/fixtures/files; the image is built with the dpp-criteria commit that adds them. --- README.md | 35 ++++- .../api/v1/validations_controller.rb | 2 +- app/services/element_ids_check.rb | 137 ++++++++++++++++++ app/services/html_check.rb | 99 +++++++++++++ app/services/json_duplicates.rb | 105 ++++++++++++++ app/services/passport_linter.rb | 18 ++- kubernetes/deployment.yaml | 2 +- .../invalid/compressed-repeated-member.json | 1 + .../full-collection-language-variants.json | 29 ++++ .../full-multi-valued-item-repeated.json | 41 ++++++ .../invalid/full-top-level-repeated.json | 23 +++ .../DPP-DAT-017/valid/compressed-unique.json | 27 ++++ .../valid/full-items-repeat-structure.json | 53 +++++++ .../full-same-id-in-two-collections.json | 35 +++++ .../DPP-DAT-017/valid/full-unique.json | 101 +++++++++++++ .../valid/full-items-repeat-structure.json | 53 +++++++ .../DPP-DAT-021/valid/full-unique.json | 101 +++++++++++++ .../warning/full-same-id-at-two-depths.json | 29 ++++ .../full-same-id-in-two-collections.json | 35 +++++ .../invalid/event-handler-notifications.html | 11 ++ .../invalid/file-input-capture.html | 11 ++ .../invalid/iframe-allow-camera.html | 11 ++ .../invalid/inline-script-location.html | 11 ++ .../invalid/javascript-url-vibration.html | 11 ++ .../invalid/modulepreload-sensors.html | 11 ++ .../invalid/own-script-camera.html | 11 ++ .../invalid/permission-element.html | 11 ++ .../files/html/DPP-SEC-021/scripts/render.js | 1 + .../files/html/DPP-SEC-021/scripts/scanner.js | 4 + .../files/html/DPP-SEC-021/scripts/sensors.js | 1 + .../DPP-SEC-021/valid/json-ld-data-block.html | 11 ++ .../html/DPP-SEC-021/valid/no-scripts.html | 11 ++ .../valid/own-script-without-device-use.html | 11 ++ .../script-from-other-host-not-fetched.html | 11 ++ .../DPP-SEC-021/valid/tel-and-sms-links.html | 11 ++ test/integration/validations_test.rb | 35 ++++- test/services/element_ids_check_test.rb | 98 +++++++++++++ test/services/html_check_device_test.rb | 122 ++++++++++++++++ test/services/json_duplicates_test.rb | 39 +++++ test/services/passport_linter_test.rb | 21 +++ 40 files changed, 1376 insertions(+), 14 deletions(-) create mode 100644 app/services/element_ids_check.rb create mode 100644 app/services/json_duplicates.rb create mode 100644 test/fixtures/files/element_ids/DPP-DAT-017/invalid/compressed-repeated-member.json create mode 100644 test/fixtures/files/element_ids/DPP-DAT-017/invalid/full-collection-language-variants.json create mode 100644 test/fixtures/files/element_ids/DPP-DAT-017/invalid/full-multi-valued-item-repeated.json create mode 100644 test/fixtures/files/element_ids/DPP-DAT-017/invalid/full-top-level-repeated.json create mode 100644 test/fixtures/files/element_ids/DPP-DAT-017/valid/compressed-unique.json create mode 100644 test/fixtures/files/element_ids/DPP-DAT-017/valid/full-items-repeat-structure.json create mode 100644 test/fixtures/files/element_ids/DPP-DAT-017/valid/full-same-id-in-two-collections.json create mode 100644 test/fixtures/files/element_ids/DPP-DAT-017/valid/full-unique.json create mode 100644 test/fixtures/files/element_ids/DPP-DAT-021/valid/full-items-repeat-structure.json create mode 100644 test/fixtures/files/element_ids/DPP-DAT-021/valid/full-unique.json create mode 100644 test/fixtures/files/element_ids/DPP-DAT-021/warning/full-same-id-at-two-depths.json create mode 100644 test/fixtures/files/element_ids/DPP-DAT-021/warning/full-same-id-in-two-collections.json create mode 100644 test/fixtures/files/html/DPP-SEC-021/invalid/event-handler-notifications.html create mode 100644 test/fixtures/files/html/DPP-SEC-021/invalid/file-input-capture.html create mode 100644 test/fixtures/files/html/DPP-SEC-021/invalid/iframe-allow-camera.html create mode 100644 test/fixtures/files/html/DPP-SEC-021/invalid/inline-script-location.html create mode 100644 test/fixtures/files/html/DPP-SEC-021/invalid/javascript-url-vibration.html create mode 100644 test/fixtures/files/html/DPP-SEC-021/invalid/modulepreload-sensors.html create mode 100644 test/fixtures/files/html/DPP-SEC-021/invalid/own-script-camera.html create mode 100644 test/fixtures/files/html/DPP-SEC-021/invalid/permission-element.html create mode 100644 test/fixtures/files/html/DPP-SEC-021/scripts/render.js create mode 100644 test/fixtures/files/html/DPP-SEC-021/scripts/scanner.js create mode 100644 test/fixtures/files/html/DPP-SEC-021/scripts/sensors.js create mode 100644 test/fixtures/files/html/DPP-SEC-021/valid/json-ld-data-block.html create mode 100644 test/fixtures/files/html/DPP-SEC-021/valid/no-scripts.html create mode 100644 test/fixtures/files/html/DPP-SEC-021/valid/own-script-without-device-use.html create mode 100644 test/fixtures/files/html/DPP-SEC-021/valid/script-from-other-host-not-fetched.html create mode 100644 test/fixtures/files/html/DPP-SEC-021/valid/tel-and-sms-links.html create mode 100644 test/services/element_ids_check_test.rb create mode 100644 test/services/html_check_device_test.rb create mode 100644 test/services/json_duplicates_test.rb diff --git a/README.md b/README.md index 3d0a872..e9eee9c 100644 --- a/README.md +++ b/README.md @@ -103,14 +103,33 @@ complete and are not a legal assessment. - Criteria with `check.type: links` check every `RelatedResource` of the passport for the required attributes and send a HEAD request to its URL (at most 20 URLs per passport). A URL that does not answer gives a warning. -- Criteria with `check.type: html` (DPP-SEC-014) request the product - identifier with `Accept: text/html` and list the URLs a browser loads without - user action while it displays the HTML view (scripts, style sheets, fonts, - images, frames, preloads, CSS `url()` and `@import`, meta refresh). With - `external_resources: forbidden` every such URL on another host than the - passport's fails the criterion. Without an HTML view the result is - `skipped` (`no_evidence`). Resources added by scripts at run time are not - seen. +- Criteria with `check.type: html` (DPP-SEC-014, DPP-SEC-021) request the + product identifier with `Accept: text/html`. Without an HTML view the result + is `skipped` (`no_evidence`). + - With `external_resources: forbidden` dpplint lists the URLs a browser loads + without user action while it displays the HTML view (scripts, style + sheets, fonts, images, frames, preloads, CSS `url()` and `@import`, meta + refresh); every such URL on another host than the passport's fails the + criterion. Resources added by scripts at run time are not seen. + - With `device_features: forbidden` dpplint searches, without running any + script, the inline scripts, event handler attributes and `javascript:` + URLs of the HTML view and the scripts it loads from its own host (at most + 10, the first 2 MB of each) for browser interfaces that use functions of + the reader's device (location, camera, microphone, notifications, + contacts, sensors, network information and others), and the markup for + frames granted such permissions, file inputs with `capture` and permission + elements. Every finding fails the criterion. Code assembled at run time, + obfuscated or loaded by a script, and scripts from other hosts, are not + seen; an interface name in a comment or string counts. +- Criteria with `check.type: element_ids` (DPP-DAT-017, DPP-DAT-021) check + that elementIds are unique. In the expanded full serialization dpplint + compares the elementIds within every array of data elements (`unique: + level`, a repeated elementId fails) or across the whole passport (`unique: + passport`, an elementId at two places gives a warning; the items of a + MultiValuedDataElement count as one place). In the compressed serialization + `level` looks for a member name repeated within one JSON object of the + passport as delivered (the response, or the posted body), and `passport` is + `skipped`. - Criteria whose condition does not hold are reported as `skipped`. - Every `skipped` criterion carries a `reason` and a `reason_code` (`not_applicable`, `no_evidence`, `not_evaluated`, … as defined under diff --git a/app/controllers/api/v1/validations_controller.rb b/app/controllers/api/v1/validations_controller.rb index b17855b..3b609a1 100644 --- a/app/controllers/api/v1/validations_controller.rb +++ b/app/controllers/api/v1/validations_controller.rb @@ -24,7 +24,7 @@ def create passport = JSON.parse(request.raw_post) return render(json: { error: "body is not a JSON object" }, status: :unprocessable_entity) unless passport.is_a?(Hash) - render json: PassportLinter.new.run(passport: passport) + render json: PassportLinter.new.run(passport: passport, source: request.raw_post) rescue JSON::ParserError => e render json: { error: "invalid JSON: #{e.message}" }, status: :unprocessable_entity end diff --git a/app/services/element_ids_check.rb b/app/services/element_ids_check.rb new file mode 100644 index 0000000..9352779 --- /dev/null +++ b/app/services/element_ids_check.rb @@ -0,0 +1,137 @@ +# Runs a criterion with check.type element_ids: whether the elementIds of the +# passport are unique (EN 18223 4.1.2.3, Table 2; "Check types", +# element_ids, in CRITERIA-FORMAT.md of dpp-criteria). +# +# In the expanded full serialization (EN 18223 Annex A; an `elements` array +# at the top level) the data elements are the objects with an elementId in +# the `elements` arrays of the passport, of its DataElementCollections and +# MultiValuedDataElements, and in the `value` array of a MultiValuedDataElement +# whose items are such objects. +# +# unique: level - every elementId once per array of data elements (a +# violation for each one repeated). In the compressed serialization the +# elementIds are JSON member names, so no JSON object of the passport as +# delivered (`source`, the bytes of the response or of the posted body) may +# contain a name twice; without those bytes the criterion is skipped. +# unique: passport - every elementId at one place of the passport (a warning +# for each one used at several places). The items of a MultiValuedDataElement +# count as one place, because they repeat one structure. Only the expanded +# full serialization. +class ElementIdsCheck + Outcome = Struct.new(:skipped, :code, :messages, keyword_init: true) + + Element = Struct.new(:id, :place, keyword_init: true) + + MAX_LISTED = 10 + + def initialize(check, passport, source: nil) + @check = check + @passport = passport + @source = source + end + + def outcome + if full? + @check["unique"] == "passport" ? Outcome.new(messages: passport_messages) : Outcome.new(messages: level_messages) + elsif @check["unique"] == "passport" + Outcome.new(skipped: "the passport is in the compressed serialization, where elementIds cannot be told apart from " \ + "attribute names without the data dictionary", code: "not_evaluated") + elsif @source.nil? + Outcome.new(skipped: "the passport is in the compressed serialization and is not available as delivered, so " \ + "repeated member names cannot be seen", code: "not_evaluated") + else + compressed_level_outcome + end + end + + # The full serialization has an `elements` array at the top level. + def full? = @passport.is_a?(Hash) && @passport["elements"].is_a?(Array) + + private + + def level_messages + levels = [] + walk(@passport["elements"], [], [], levels: levels) + repeated = levels.flat_map do |parent, ids| + ids.tally.select { |_, n| n > 1 }.map { |id, n| "#{id} #{n} times #{where(parent)}" } + end + return [] if repeated.empty? + + [{ severity: "violation", message: "elementId repeated within one level: #{listed(repeated)}" }] + end + + def passport_messages + elements = [] + walk(@passport["elements"], [], [], elements: elements) + places = elements.group_by(&:id).transform_values { |list| list.map(&:place).uniq } + shared = places.select { |_, list| list.size > 1 } + return [] if shared.empty? + + texts = shared.map { |id, list| "#{id} at #{list.size} places (#{list.map { |p| p.join('/') }.join(', ')})" } + [{ severity: "warning", message: "elementId used at more than one place of the passport: #{listed(texts)}" }] + end + + # Walks one array of data elements. path: the elementIds above it; place: + # the same with "*" for the items of a MultiValuedDataElement. + def walk(array, path, place, levels: nil, elements: nil) + items = array.select { |e| e.is_a?(Hash) && e["elementId"].is_a?(String) } + levels&.push([path, items.map { |e| e["elementId"] }]) + items.each do |e| + id = e["elementId"] + elements&.push(Element.new(id: id, place: place + [id])) + multi = e["objectType"] == "MultiValuedDataElement" + children(e).each do |list| + if multi + walk_items(list, path + [id], place + [id], levels: levels, elements: elements) + else + walk(list, path + [id], place + [id], levels: levels, elements: elements) + end + end + end + end + + # The items of a MultiValuedDataElement: one level for the repeated-id check; + # below them, every item counts as the same place ("*"). + def walk_items(array, path, place, levels:, elements:) + items = array.select { |e| e.is_a?(Hash) && e["elementId"].is_a?(String) } + levels&.push([path, items.map { |e| e["elementId"] }]) + items.each do |e| + id = e["elementId"] + elements&.push(Element.new(id: id, place: place + ["*"])) + multi = e["objectType"] == "MultiValuedDataElement" + children(e).each do |list| + if multi + walk_items(list, path + [id], place + ["*"], levels: levels, elements: elements) + else + walk(list, path + [id], place + ["*"], levels: levels, elements: elements) + end + end + end + end + + # Arrays of data elements below an element: `elements`, and `value` if it + # holds objects with an elementId (MultiValuedDataElement items, Annex A). + def children(element) + %w[elements value].filter_map do |key| + list = element[key] + list if list.is_a?(Array) && list.any? { |i| i.is_a?(Hash) && i["elementId"].is_a?(String) } + end + end + + def compressed_level_outcome + duplicates = JsonDuplicates.find(@source) + return Outcome.new(messages: []) if duplicates.empty? + + texts = duplicates.map { |d| "#{d[:name]} #{d[:count]} times in the object at #{d[:path]}" } + Outcome.new(messages: [{ severity: "violation", message: "member name repeated within one JSON object: #{listed(texts)}" }]) + rescue JsonDuplicates::Invalid => e + Outcome.new(skipped: "the passport as delivered cannot be read: #{e.message}", code: "not_evaluated") + end + + def where(parent) = parent.empty? ? "at the top level of the passport" : "in #{parent.join('/')}" + + def listed(texts) + more = texts.size > MAX_LISTED ? " and #{texts.size - MAX_LISTED} more" : "" + "#{texts.first(MAX_LISTED).join('; ')}#{more}" + end +end diff --git a/app/services/html_check.rb b/app/services/html_check.rb index 9d0c5c0..aaf00bb 100644 --- a/app/services/html_check.rb +++ b/app/services/html_check.rb @@ -12,9 +12,39 @@ # differs from the host of the final response URL. Links the user has to # follow are not counted; resources added by scripts at run time cannot be # seen in the document. +# +# device_features: forbidden looks, without running any script, for browser +# interfaces that use functions of the reader's device (DEVICE_INTERFACES) in +# the inline scripts, event handler attributes and javascript: URLs of the +# document and in the scripts it loads from its own host (at most MAX_SCRIPTS, +# the first MAX_SCRIPT_BYTES of each; a script that cannot be retrieved is not +# checked), and in the markup for frames granted device permissions +# (DEVICE_PERMISSIONS), file inputs with capture and permission elements. class HtmlCheck Outcome = Struct.new(:skipped, :code, :messages, keyword_init: true) + DEVICE_INTERFACES = [ + ["location", /\.geolocation\b|\bgetCurrentPosition\b|\bwatchPosition\b/], + ["camera, microphone or screen", /\bgetUserMedia\b|\bgetDisplayMedia\b|\.mediaDevices\b/], + ["notifications", /\bNotification\s*\.\s*requestPermission\b|\bnew\s+Notification\s*\(|\.showNotification\s*\(|\b[Pp]ushManager\b/], + ["contacts", /\bnavigator\s*\.\s*contacts\b/], + ["Bluetooth, USB, serial, HID or NFC", /\bnavigator\s*\.\s*(?:bluetooth|usb|serial|hid)\b|\bNDEFReader\b/], + ["motion or orientation sensors", + /\b(?:on)?(?:devicemotion|deviceorientation(?:absolute)?)\b|\bDevice(?:Motion|Orientation)Event\s*\.\s*requestPermission\b|\bnew\s+(?:Accelerometer|Gyroscope|Magnetometer|AbsoluteOrientationSensor|RelativeOrientationSensor|LinearAccelerationSensor|GravitySensor|AmbientLightSensor)\s*\(/], + ["context or network information", /\bnavigator\s*\.\s*connection\b|\bgetBattery\s*\(|\bIdleDetector\b/], + ["one-time codes from SMS", /\bOTPCredential\b|\btransport\s*:\s*\[\s*["']sms["']/], + ["app installation", /\bbeforeinstallprompt\b|\bgetInstalledRelatedApps\b/], + ["device behaviour", /\bnavigator\s*\.\s*(?:wakeLock|vibrate)\b/] + ].freeze + DEVICE_PERMISSIONS = %w[camera microphone geolocation display-capture bluetooth usb serial hid accelerometer gyroscope + magnetometer ambient-light-sensor idle-detection screen-wake-lock].freeze + JS_TYPES = %w[text/javascript application/javascript application/ecmascript text/ecmascript module + application/x-javascript text/jscript text/x-javascript].freeze + URL_ATTRIBUTES = %w[href src action formaction].freeze + MAX_SCRIPTS = 10 + MAX_SCRIPT_BYTES = 2 * 1024 * 1024 + MAX_FINDINGS = 10 + SRC_ELEMENTS = %w[script img iframe frame embed audio video source track].freeze SRCSET_ELEMENTS = %w[img source].freeze LINK_RELS = %w[stylesheet icon apple-touch-icon mask-icon manifest preload modulepreload @@ -37,9 +67,23 @@ def outcome messages = [] messages << external_message(res) if @check["external_resources"] == "forbidden" + messages << device_message(res) if @check["device_features"] == "forbidden" Outcome.new(messages: messages.compact) end + # Device interfaces found in a piece of script: [[label, matched text], …]. + def self.device_uses(code) + DEVICE_INTERFACES.filter_map do |label, pattern| + match = code.to_s.match(pattern) + [label, match[0].gsub(/\s+/, " ")] if match + end + end + + def self.javascript_type?(type) + type = type.to_s.split(";").first.to_s.strip.downcase + type.empty? || JS_TYPES.include?(type) + end + # The URLs a browser loads without user action, resolved, with the element # or construct they come from: [[url, source], …]. def self.loaded_urls(html, page_url) @@ -121,6 +165,61 @@ def external_message(res) message: "the HTML view loads #{external.size} resource(s) from other hosts than #{own}: #{listed.join(', ')}#{more}" } end + def device_message(res) + doc = Nokogiri::HTML5(res.body.to_s) + findings = markup_findings(doc) + inline_findings(doc) + script_findings(doc, res.url.to_s) + return nil if findings.empty? + + listed = findings.first(MAX_FINDINGS) + more = findings.size > MAX_FINDINGS ? " and #{findings.size - MAX_FINDINGS} more" : "" + { severity: "violation", message: "the HTML view uses functions of the reader's device: #{listed.join('; ')}#{more}" } + end + + def markup_findings(doc) + found = [] + doc.css("iframe[allow]").each do |e| + granted = e["allow"].to_s.split(";").map { |d| d.strip.split(/\s+/).first.to_s.downcase } & DEVICE_PERMISSIONS + found << "iframe allow grants #{granted.join(', ')}" if granted.any? + end + found << "file input with capture" if doc.css("input[capture]").any? + %w[permission geolocation].each { |name| found << "#{name} element" if doc.css(name).any? } + found + end + + def inline_findings(doc) + found = [] + doc.css("script").each_with_index do |e, i| + next if e["src"] || !self.class.javascript_type?(e["type"]) + + self.class.device_uses(e.content).each { |label, use| found << "#{label} (#{use} in inline script #{i + 1})" } + end + doc.xpath("//@*").each do |a| + name = a.name.downcase + code = if name.start_with?("on") then a.value + elsif URL_ATTRIBUTES.include?(name) && a.value.to_s.strip.downcase.start_with?("javascript:") then a.value + end + next unless code + + self.class.device_uses(code).each { |label, use| found << "#{label} (#{use} in attribute #{name} of #{a.parent.name})" } + end + found + end + + # Scripts loaded from the host of the page, fetched as text and searched. + def script_findings(doc, page_url) + own = self.class.web_host(page_url) + base = self.class.absolute(doc.at_css("base[href]")&.[]("href"), page_url) || page_url + urls = doc.css("script[src]").select { |e| self.class.javascript_type?(e["type"]) }.map { |e| e["src"] } + urls += doc.css("link[href]").select { |e| e["rel"].to_s.downcase.split.include?("modulepreload") }.map { |e| e["href"] } + urls = urls.filter_map { |u| self.class.absolute(u, base) }.uniq.select { |u| self.class.web_host(u) == own } + urls.first(MAX_SCRIPTS).flat_map do |url| + res = @resolver.get(url, accept: "*/*") + next [] unless res.success? + + self.class.device_uses(res.body.to_s.byteslice(0, MAX_SCRIPT_BYTES).to_s.dup.force_encoding(Encoding::UTF_8).scrub).map { |label, use| "#{label} (#{use} in #{url})" } + end + end + def describe(res) return res.error if res.error diff --git a/app/services/json_duplicates.rb b/app/services/json_duplicates.rb new file mode 100644 index 0000000..5c922d5 --- /dev/null +++ b/app/services/json_duplicates.rb @@ -0,0 +1,105 @@ +require "strscan" + +# Finds member names that occur more than once in one JSON object, in JSON +# text as it was delivered. JSON parsers keep only one of the values of such a +# name, so the repetition cannot be seen in the parsed passport. +# +# JsonDuplicates.find('{"a":1,"b":{"c":1,"c":2}}') +# # => [{ path: "$['b']", name: "c", count: 2 }] +# +# Names are compared after their escapes are resolved ("a" and "a" are +# the same name). Paths are normalized JSONPaths (RFC 9535 2.7). Raises +# JsonDuplicates::Invalid if the text is not a single JSON value or nests +# deeper than MAX_DEPTH. +class JsonDuplicates + class Invalid < StandardError; end + + MAX_DEPTH = 512 + WS = /[ \t\n\r]*/ + STRING = /"(?:[^"\\\u0000-\u001f]|\\(?:["\\\/bfnrt]|u\h{4}))*"/ + NUMBER = /-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/ + LITERAL = /true|false|null/ + + def self.find(text) + new(text).find + end + + def initialize(text) + source = text.to_s.dup.force_encoding(Encoding::UTF_8) + raise Invalid, "the text is not valid UTF-8" unless source.valid_encoding? + + @s = StringScanner.new(source.delete_prefix("")) + @found = [] + end + + def find + @s.skip(WS) + value("$", 0) + @s.skip(WS) + raise Invalid, "unexpected text after the JSON value at character #{@s.charpos}" unless @s.eos? + + @found + end + + private + + def value(path, depth) + raise Invalid, "nested deeper than #{MAX_DEPTH} levels" if depth > MAX_DEPTH + + if @s.skip(/\{/) then object(path, depth) + elsif @s.skip(/\[/) then array(path, depth) + elsif @s.scan(STRING) || @s.scan(NUMBER) || @s.scan(LITERAL) then nil + else raise Invalid, "unexpected character at #{@s.charpos}" + end + end + + def object(path, depth) + counts = Hash.new(0) + @s.skip(WS) + unless @s.skip(/\}/) + loop do + @s.skip(WS) + token = @s.scan(STRING) or raise Invalid, "member name expected at character #{@s.charpos}" + name = JSON.parse(token) + counts[name] += 1 + @s.skip(WS) + @s.skip(/:/) or raise Invalid, "':' expected at character #{@s.charpos}" + @s.skip(WS) + value("#{path}[#{self.class.quote(name)}]", depth + 1) + @s.skip(WS) + next if @s.skip(/,/) + break if @s.skip(/\}/) + + raise Invalid, "',' or '}' expected at character #{@s.charpos}" + end + end + counts.each { |name, count| @found << { path: path, name: name, count: count } if count > 1 } + end + + def array(path, depth) + @s.skip(WS) + return if @s.skip(/\]/) + + index = 0 + loop do + @s.skip(WS) + value("#{path}[#{index}]", depth + 1) + index += 1 + @s.skip(WS) + next if @s.skip(/,/) + break if @s.skip(/\]/) + + raise Invalid, "',' or ']' expected at character #{@s.charpos}" + end + end + + SHORT_ESCAPES = { "\b" => "\\b", "\f" => "\\f", "\n" => "\\n", "\r" => "\\r", "\t" => "\\t" }.freeze + + # A member name in single quotes, escaped as a normalized path requires + # (RFC 9535 2.7). + def self.quote(name) + escaped = name.gsub(/[\\']/) { |c| "\\#{c}" } + .gsub(/[\u0000-\u001f]/) { |c| SHORT_ESCAPES[c] || format("\\u%04x", c.ord) } + "'#{escaped}'" + end +end diff --git a/app/services/passport_linter.rb b/app/services/passport_linter.rb index d03ac9d..3c1aecd 100644 --- a/app/services/passport_linter.rb +++ b/app/services/passport_linter.rb @@ -9,7 +9,10 @@ # check.type proof verify an integrity proof in the passport; criteria with # check.type links check the related resources it links to; criteria with # check.type html check the HTML view of the passport (needs a product -# identifier, like resolve). +# identifier, like resolve); criteria with check.type element_ids check that +# the elementIds of the passport are unique (in the compressed serialization +# on `source`, the passport as delivered: the bytes of the response or of the +# posted body). # # A check ends within its time limit (CheckDeadline): a criterion whose # requests are cut off by it, or that would need a request after it, is @@ -28,9 +31,10 @@ def initialize(catalogue: CriteriaCatalogue.new, deadline: CheckDeadline.new, we @didlint = didlint end - def run(passport:, product_id: nil, retrieval: nil, jws: nil, jws_only: false, raw: nil, jws_unchecked: nil) + def run(passport:, product_id: nil, retrieval: nil, jws: nil, jws_only: false, raw: nil, jws_unchecked: nil, source: nil) @product_id = product_id @raw = raw + @source = source || raw @jws = jws @jws_only = jws_only @jws_unchecked = jws_unchecked @@ -106,7 +110,7 @@ def evaluate(criterion, passport, reports) return resolve(base, check) if check["type"] == "resolve" return html(base, check) if check["type"] == "html" - unless %w[shacl did proof links].include?(check["type"]) + unless %w[shacl did proof links element_ids].include?(check["type"]) return skip(base, "check type #{check['type']} is not implemented in this version", "not_evaluated") end return skip(base, "passport could not be retrieved", "not_evaluated") if passport.nil? @@ -119,6 +123,7 @@ def evaluate(criterion, passport, reports) return did(base, check, passport) if check["type"] == "did" return proof(base, check, passport) if check["type"] == "proof" return links(base, check, passport) if check["type"] == "links" + return element_ids(base, check, passport) if check["type"] == "element_ids" return skip(base, "no shapes available for #{check['shapes_select']}", "not_evaluated") unless check["structure"] @@ -176,6 +181,13 @@ def proof(base, check, passport) skip(base, e.message, "not_evaluated") end + def element_ids(base, check, passport) + outcome = ElementIdsCheck.new(check, passport, source: @source).outcome + return skip(base, outcome.skipped, outcome.code) if outcome.skipped + + base.merge(result: result_for(outcome.messages), messages: outcome.messages) + end + def links(base, check, passport) links_check = LinksCheck.new(check, passport, resolver: @resolver) return skip(base, "no #{check['element_type']} elements", "not_applicable") if links_check.resources.empty? diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml index 2ebc053..8fe73e2 100644 --- a/kubernetes/deployment.yaml +++ b/kubernetes/deployment.yaml @@ -26,7 +26,7 @@ spec: app: dpplint containers: - name: dpplint - image: oydeu/dpplint@sha256:bf49962f61eea1998486462af28dc504a0b1f09143554ecf518eeb9298f59094 + image: oydeu/dpplint@sha256:c7cc08a926cf261d3fd3321aaa851343118f3e810ebd0a763312ec21050e1db8 imagePullPolicy: IfNotPresent ports: - containerPort: 3000 diff --git a/test/fixtures/files/element_ids/DPP-DAT-017/invalid/compressed-repeated-member.json b/test/fixtures/files/element_ids/DPP-DAT-017/invalid/compressed-repeated-member.json new file mode 100644 index 0000000..f394e5b --- /dev/null +++ b/test/fixtures/files/element_ids/DPP-DAT-017/invalid/compressed-repeated-member.json @@ -0,0 +1 @@ +{"digitalProductPassportId": "https://dpp.example.org/dpp/1", "uniqueProductIdentifier": "https://dpp.example.org/01/09520123456788/21/1", "granularity": "item", "dppSchemaVersion": "EN 18223:2026", "dppStatus": "Active", "lastUpdated": "2026-10-25T00:00:00Z", "economicOperatorId": "did:web:operator.example.org", "performanceMetrics": {"maxPressure": 750.0, "maxPressure": 700.0, "isFireResistant": true}} diff --git a/test/fixtures/files/element_ids/DPP-DAT-017/invalid/full-collection-language-variants.json b/test/fixtures/files/element_ids/DPP-DAT-017/invalid/full-collection-language-variants.json new file mode 100644 index 0000000..a7999f0 --- /dev/null +++ b/test/fixtures/files/element_ids/DPP-DAT-017/invalid/full-collection-language-variants.json @@ -0,0 +1,29 @@ +{ + "digitalProductPassportId": "https://dpp.example.org/dpp/1", + "uniqueProductIdentifier": "https://dpp.example.org/01/09520123456788/21/1", + "granularity": "item", + "dppSchemaVersion": "EN 18223:2026", + "dppStatus": "Active", + "lastUpdated": "2026-10-25T00:00:00Z", + "economicOperatorId": "did:web:operator.example.org", + "elements": [ + { + "elementId": "ProductInformation", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "SafetyInstructions", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "Wear eye protection." + }, + { + "elementId": "SafetyInstructions", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "Schutzbrille tragen." + } + ] + } + ] +} diff --git a/test/fixtures/files/element_ids/DPP-DAT-017/invalid/full-multi-valued-item-repeated.json b/test/fixtures/files/element_ids/DPP-DAT-017/invalid/full-multi-valued-item-repeated.json new file mode 100644 index 0000000..62cea31 --- /dev/null +++ b/test/fixtures/files/element_ids/DPP-DAT-017/invalid/full-multi-valued-item-repeated.json @@ -0,0 +1,41 @@ +{ + "digitalProductPassportId": "https://dpp.example.org/dpp/1", + "uniqueProductIdentifier": "https://dpp.example.org/01/09520123456788/21/1", + "granularity": "item", + "dppSchemaVersion": "EN 18223:2026", + "dppStatus": "Active", + "lastUpdated": "2026-10-25T00:00:00Z", + "economicOperatorId": "did:web:operator.example.org", + "elements": [ + { + "elementId": "EnergyPerformance", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "efficiencyRatings", + "objectType": "MultiValuedDataElement", + "value": [ + { + "elementId": "efficiencyRating1", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:decimal", + "value": 0.95 + }, + { + "elementId": "efficiencyRating2", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:decimal", + "value": 0.92 + }, + { + "elementId": "efficiencyRating2", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:decimal", + "value": 0.88 + } + ] + } + ] + } + ] +} diff --git a/test/fixtures/files/element_ids/DPP-DAT-017/invalid/full-top-level-repeated.json b/test/fixtures/files/element_ids/DPP-DAT-017/invalid/full-top-level-repeated.json new file mode 100644 index 0000000..f912677 --- /dev/null +++ b/test/fixtures/files/element_ids/DPP-DAT-017/invalid/full-top-level-repeated.json @@ -0,0 +1,23 @@ +{ + "digitalProductPassportId": "https://dpp.example.org/dpp/1", + "uniqueProductIdentifier": "https://dpp.example.org/01/09520123456788/21/1", + "granularity": "item", + "dppSchemaVersion": "EN 18223:2026", + "dppStatus": "Active", + "lastUpdated": "2026-10-25T00:00:00Z", + "economicOperatorId": "did:web:operator.example.org", + "elements": [ + { + "elementId": "ModelIdentifier", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "M-1" + }, + { + "elementId": "ModelIdentifier", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "M-2" + } + ] +} diff --git a/test/fixtures/files/element_ids/DPP-DAT-017/valid/compressed-unique.json b/test/fixtures/files/element_ids/DPP-DAT-017/valid/compressed-unique.json new file mode 100644 index 0000000..7fdb79f --- /dev/null +++ b/test/fixtures/files/element_ids/DPP-DAT-017/valid/compressed-unique.json @@ -0,0 +1,27 @@ +{ + "digitalProductPassportId": "https://dpp.example.org/dpp/1", + "uniqueProductIdentifier": "https://dpp.example.org/01/09520123456788/21/1", + "granularity": "item", + "dppSchemaVersion": "EN 18223:2026", + "dppStatus": "Active", + "lastUpdated": "2026-10-25T00:00:00Z", + "economicOperatorId": "did:web:operator.example.org", + "performanceMetrics": { + "maxPressure": 750.0, + "isFireResistant": true, + "efficiencyRatings": [ + 0.95, + 0.92 + ] + }, + "componentParts": [ + { + "partId": "A-101", + "material": "Steel" + }, + { + "partId": "B-202", + "material": "Recycled plastic" + } + ] +} diff --git a/test/fixtures/files/element_ids/DPP-DAT-017/valid/full-items-repeat-structure.json b/test/fixtures/files/element_ids/DPP-DAT-017/valid/full-items-repeat-structure.json new file mode 100644 index 0000000..ec8e5ac --- /dev/null +++ b/test/fixtures/files/element_ids/DPP-DAT-017/valid/full-items-repeat-structure.json @@ -0,0 +1,53 @@ +{ + "digitalProductPassportId": "https://dpp.example.org/dpp/1", + "uniqueProductIdentifier": "https://dpp.example.org/01/09520123456788/21/1", + "granularity": "item", + "dppSchemaVersion": "EN 18223:2026", + "dppStatus": "Active", + "lastUpdated": "2026-10-25T00:00:00Z", + "economicOperatorId": "did:web:operator.example.org", + "elements": [ + { + "elementId": "componentParts", + "objectType": "MultiValuedDataElement", + "elements": [ + { + "elementId": "part1", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "partId", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "A-101" + }, + { + "elementId": "material", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "Steel" + } + ] + }, + { + "elementId": "part2", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "partId", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "B-202" + }, + { + "elementId": "material", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "Recycled plastic" + } + ] + } + ] + } + ] +} diff --git a/test/fixtures/files/element_ids/DPP-DAT-017/valid/full-same-id-in-two-collections.json b/test/fixtures/files/element_ids/DPP-DAT-017/valid/full-same-id-in-two-collections.json new file mode 100644 index 0000000..be7936d --- /dev/null +++ b/test/fixtures/files/element_ids/DPP-DAT-017/valid/full-same-id-in-two-collections.json @@ -0,0 +1,35 @@ +{ + "digitalProductPassportId": "https://dpp.example.org/dpp/1", + "uniqueProductIdentifier": "https://dpp.example.org/01/09520123456788/21/1", + "granularity": "item", + "dppSchemaVersion": "EN 18223:2026", + "dppStatus": "Active", + "lastUpdated": "2026-10-25T00:00:00Z", + "economicOperatorId": "did:web:operator.example.org", + "elements": [ + { + "elementId": "Manufacturer", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "name", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "Example Corp" + } + ] + }, + { + "elementId": "Importer", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "name", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "Import Ltd" + } + ] + } + ] +} diff --git a/test/fixtures/files/element_ids/DPP-DAT-017/valid/full-unique.json b/test/fixtures/files/element_ids/DPP-DAT-017/valid/full-unique.json new file mode 100644 index 0000000..f184af0 --- /dev/null +++ b/test/fixtures/files/element_ids/DPP-DAT-017/valid/full-unique.json @@ -0,0 +1,101 @@ +{ + "digitalProductPassportId": "https://dpp.example.org/dpp/1", + "uniqueProductIdentifier": "https://dpp.example.org/01/09520123456788/21/1", + "granularity": "item", + "dppSchemaVersion": "EN 18223:2026", + "dppStatus": "Active", + "lastUpdated": "2026-10-25T00:00:00Z", + "economicOperatorId": "did:web:operator.example.org", + "elements": [ + { + "elementId": "ProductIdentification", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "ModelIdentifier", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "M-1" + }, + { + "elementId": "SerialNumber", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "0001" + } + ] + }, + { + "elementId": "EnergyPerformance", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "OnModePower", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:decimal", + "value": 8.5 + }, + { + "elementId": "efficiencyRatings", + "objectType": "MultiValuedDataElement", + "value": [ + { + "elementId": "efficiencyRating1", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:decimal", + "value": 0.95 + }, + { + "elementId": "efficiencyRating2", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:decimal", + "value": 0.92 + } + ] + } + ] + }, + { + "elementId": "componentParts", + "objectType": "MultiValuedDataElement", + "elements": [ + { + "elementId": "part1", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "partId", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "A-101" + }, + { + "elementId": "material", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "Steel" + } + ] + }, + { + "elementId": "part2", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "partId", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "B-202" + }, + { + "elementId": "material", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "Recycled plastic" + } + ] + } + ] + } + ] +} diff --git a/test/fixtures/files/element_ids/DPP-DAT-021/valid/full-items-repeat-structure.json b/test/fixtures/files/element_ids/DPP-DAT-021/valid/full-items-repeat-structure.json new file mode 100644 index 0000000..ec8e5ac --- /dev/null +++ b/test/fixtures/files/element_ids/DPP-DAT-021/valid/full-items-repeat-structure.json @@ -0,0 +1,53 @@ +{ + "digitalProductPassportId": "https://dpp.example.org/dpp/1", + "uniqueProductIdentifier": "https://dpp.example.org/01/09520123456788/21/1", + "granularity": "item", + "dppSchemaVersion": "EN 18223:2026", + "dppStatus": "Active", + "lastUpdated": "2026-10-25T00:00:00Z", + "economicOperatorId": "did:web:operator.example.org", + "elements": [ + { + "elementId": "componentParts", + "objectType": "MultiValuedDataElement", + "elements": [ + { + "elementId": "part1", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "partId", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "A-101" + }, + { + "elementId": "material", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "Steel" + } + ] + }, + { + "elementId": "part2", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "partId", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "B-202" + }, + { + "elementId": "material", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "Recycled plastic" + } + ] + } + ] + } + ] +} diff --git a/test/fixtures/files/element_ids/DPP-DAT-021/valid/full-unique.json b/test/fixtures/files/element_ids/DPP-DAT-021/valid/full-unique.json new file mode 100644 index 0000000..f184af0 --- /dev/null +++ b/test/fixtures/files/element_ids/DPP-DAT-021/valid/full-unique.json @@ -0,0 +1,101 @@ +{ + "digitalProductPassportId": "https://dpp.example.org/dpp/1", + "uniqueProductIdentifier": "https://dpp.example.org/01/09520123456788/21/1", + "granularity": "item", + "dppSchemaVersion": "EN 18223:2026", + "dppStatus": "Active", + "lastUpdated": "2026-10-25T00:00:00Z", + "economicOperatorId": "did:web:operator.example.org", + "elements": [ + { + "elementId": "ProductIdentification", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "ModelIdentifier", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "M-1" + }, + { + "elementId": "SerialNumber", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "0001" + } + ] + }, + { + "elementId": "EnergyPerformance", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "OnModePower", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:decimal", + "value": 8.5 + }, + { + "elementId": "efficiencyRatings", + "objectType": "MultiValuedDataElement", + "value": [ + { + "elementId": "efficiencyRating1", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:decimal", + "value": 0.95 + }, + { + "elementId": "efficiencyRating2", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:decimal", + "value": 0.92 + } + ] + } + ] + }, + { + "elementId": "componentParts", + "objectType": "MultiValuedDataElement", + "elements": [ + { + "elementId": "part1", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "partId", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "A-101" + }, + { + "elementId": "material", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "Steel" + } + ] + }, + { + "elementId": "part2", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "partId", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "B-202" + }, + { + "elementId": "material", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "Recycled plastic" + } + ] + } + ] + } + ] +} diff --git a/test/fixtures/files/element_ids/DPP-DAT-021/warning/full-same-id-at-two-depths.json b/test/fixtures/files/element_ids/DPP-DAT-021/warning/full-same-id-at-two-depths.json new file mode 100644 index 0000000..a0ab6d2 --- /dev/null +++ b/test/fixtures/files/element_ids/DPP-DAT-021/warning/full-same-id-at-two-depths.json @@ -0,0 +1,29 @@ +{ + "digitalProductPassportId": "https://dpp.example.org/dpp/1", + "uniqueProductIdentifier": "https://dpp.example.org/01/09520123456788/21/1", + "granularity": "item", + "dppSchemaVersion": "EN 18223:2026", + "dppStatus": "Active", + "lastUpdated": "2026-10-25T00:00:00Z", + "economicOperatorId": "did:web:operator.example.org", + "elements": [ + { + "elementId": "weight", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:decimal", + "value": 1.2 + }, + { + "elementId": "Packaging", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "weight", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:decimal", + "value": 0.1 + } + ] + } + ] +} diff --git a/test/fixtures/files/element_ids/DPP-DAT-021/warning/full-same-id-in-two-collections.json b/test/fixtures/files/element_ids/DPP-DAT-021/warning/full-same-id-in-two-collections.json new file mode 100644 index 0000000..be7936d --- /dev/null +++ b/test/fixtures/files/element_ids/DPP-DAT-021/warning/full-same-id-in-two-collections.json @@ -0,0 +1,35 @@ +{ + "digitalProductPassportId": "https://dpp.example.org/dpp/1", + "uniqueProductIdentifier": "https://dpp.example.org/01/09520123456788/21/1", + "granularity": "item", + "dppSchemaVersion": "EN 18223:2026", + "dppStatus": "Active", + "lastUpdated": "2026-10-25T00:00:00Z", + "economicOperatorId": "did:web:operator.example.org", + "elements": [ + { + "elementId": "Manufacturer", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "name", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "Example Corp" + } + ] + }, + { + "elementId": "Importer", + "objectType": "DataElementCollection", + "elements": [ + { + "elementId": "name", + "objectType": "SingleValuedDataElement", + "valueDataType": "xsd:string", + "value": "Import Ltd" + } + ] + } + ] +} diff --git a/test/fixtures/files/html/DPP-SEC-021/invalid/event-handler-notifications.html b/test/fixtures/files/html/DPP-SEC-021/invalid/event-handler-notifications.html new file mode 100644 index 0000000..a2ca338 --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/invalid/event-handler-notifications.html @@ -0,0 +1,11 @@ + + + + +Passport + + + + + + diff --git a/test/fixtures/files/html/DPP-SEC-021/invalid/file-input-capture.html b/test/fixtures/files/html/DPP-SEC-021/invalid/file-input-capture.html new file mode 100644 index 0000000..e35dd79 --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/invalid/file-input-capture.html @@ -0,0 +1,11 @@ + + + + +Passport + + + +
+ + diff --git a/test/fixtures/files/html/DPP-SEC-021/invalid/iframe-allow-camera.html b/test/fixtures/files/html/DPP-SEC-021/invalid/iframe-allow-camera.html new file mode 100644 index 0000000..8d66dee --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/invalid/iframe-allow-camera.html @@ -0,0 +1,11 @@ + + + + +Passport + + + + + + diff --git a/test/fixtures/files/html/DPP-SEC-021/invalid/inline-script-location.html b/test/fixtures/files/html/DPP-SEC-021/invalid/inline-script-location.html new file mode 100644 index 0000000..70dacca --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/invalid/inline-script-location.html @@ -0,0 +1,11 @@ + + + + +Passport + + + +

Passport

+ + diff --git a/test/fixtures/files/html/DPP-SEC-021/invalid/javascript-url-vibration.html b/test/fixtures/files/html/DPP-SEC-021/invalid/javascript-url-vibration.html new file mode 100644 index 0000000..41d5167 --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/invalid/javascript-url-vibration.html @@ -0,0 +1,11 @@ + + + + +Passport + + + +buzz + + diff --git a/test/fixtures/files/html/DPP-SEC-021/invalid/modulepreload-sensors.html b/test/fixtures/files/html/DPP-SEC-021/invalid/modulepreload-sensors.html new file mode 100644 index 0000000..7a01d55 --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/invalid/modulepreload-sensors.html @@ -0,0 +1,11 @@ + + + + +Passport + + + +

Passport

+ + diff --git a/test/fixtures/files/html/DPP-SEC-021/invalid/own-script-camera.html b/test/fixtures/files/html/DPP-SEC-021/invalid/own-script-camera.html new file mode 100644 index 0000000..3e98106 --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/invalid/own-script-camera.html @@ -0,0 +1,11 @@ + + + + +Passport + + + +

Passport

+ + diff --git a/test/fixtures/files/html/DPP-SEC-021/invalid/permission-element.html b/test/fixtures/files/html/DPP-SEC-021/invalid/permission-element.html new file mode 100644 index 0000000..8b55b58 --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/invalid/permission-element.html @@ -0,0 +1,11 @@ + + + + +Passport + + + + + + diff --git a/test/fixtures/files/html/DPP-SEC-021/scripts/render.js b/test/fixtures/files/html/DPP-SEC-021/scripts/render.js new file mode 100644 index 0000000..34f9fb8 --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/scripts/render.js @@ -0,0 +1 @@ +document.querySelector("h1").textContent = "Passport"; diff --git a/test/fixtures/files/html/DPP-SEC-021/scripts/scanner.js b/test/fixtures/files/html/DPP-SEC-021/scripts/scanner.js new file mode 100644 index 0000000..0bb368a --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/scripts/scanner.js @@ -0,0 +1,4 @@ +export async function scan() { + const stream = await navigator.mediaDevices.getUserMedia({ video: true }); + return stream; +} diff --git a/test/fixtures/files/html/DPP-SEC-021/scripts/sensors.js b/test/fixtures/files/html/DPP-SEC-021/scripts/sensors.js new file mode 100644 index 0000000..010167b --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/scripts/sensors.js @@ -0,0 +1 @@ +window.addEventListener("devicemotion", (e) => console.log(e.acceleration)); diff --git a/test/fixtures/files/html/DPP-SEC-021/valid/json-ld-data-block.html b/test/fixtures/files/html/DPP-SEC-021/valid/json-ld-data-block.html new file mode 100644 index 0000000..0e26748 --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/valid/json-ld-data-block.html @@ -0,0 +1,11 @@ + + + + +Passport + + + +

Passport

+ + diff --git a/test/fixtures/files/html/DPP-SEC-021/valid/no-scripts.html b/test/fixtures/files/html/DPP-SEC-021/valid/no-scripts.html new file mode 100644 index 0000000..78eab39 --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/valid/no-scripts.html @@ -0,0 +1,11 @@ + + + + +Passport + + + +

Passport

Model M-1

+ + diff --git a/test/fixtures/files/html/DPP-SEC-021/valid/own-script-without-device-use.html b/test/fixtures/files/html/DPP-SEC-021/valid/own-script-without-device-use.html new file mode 100644 index 0000000..84e172f --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/valid/own-script-without-device-use.html @@ -0,0 +1,11 @@ + + + + +Passport + + + +

Passport

+ + diff --git a/test/fixtures/files/html/DPP-SEC-021/valid/script-from-other-host-not-fetched.html b/test/fixtures/files/html/DPP-SEC-021/valid/script-from-other-host-not-fetched.html new file mode 100644 index 0000000..170a706 --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/valid/script-from-other-host-not-fetched.html @@ -0,0 +1,11 @@ + + + + +Passport + + + +

Passport

+ + diff --git a/test/fixtures/files/html/DPP-SEC-021/valid/tel-and-sms-links.html b/test/fixtures/files/html/DPP-SEC-021/valid/tel-and-sms-links.html new file mode 100644 index 0000000..a7a2937 --- /dev/null +++ b/test/fixtures/files/html/DPP-SEC-021/valid/tel-and-sms-links.html @@ -0,0 +1,11 @@ + + + + +Passport + + + +

Hotline: call, text

+ + diff --git a/test/integration/validations_test.rb b/test/integration/validations_test.rb index ca4c910..84776f9 100644 --- a/test/integration/validations_test.rb +++ b/test/integration/validations_test.rb @@ -14,6 +14,12 @@ def lint(passport) JSON.parse(response.body) end + def lint_text(text) + post "/api/v1/validate", params: text, headers: { "Content-Type" => "application/json" } + assert_response :success + JSON.parse(response.body) + end + def criterion(result, id) = result["criteria"].find { |c| c["id"] == id } test "reference passport passes all structure checks" do @@ -88,13 +94,40 @@ def criterion(result, id) = result["criteria"].find { |c| c["id"] == id } assert_equal "passed", criterion(lint(reference.merge("uniqueProductIdentifier" => "did:web:dpp.example.org:product:4711")), "DPP-ID-054")["result"] end + test "elementIds of the reference passport are unique at each level and in the whole passport" do + result = lint(reference) + assert_equal "passed", criterion(result, "DPP-DAT-017")["result"] + assert_equal "passed", criterion(result, "DPP-DAT-021")["result"] + end + + test "an elementId repeated within a collection fails, one used in two collections gives a warning" do + passport = reference + collection = passport["elements"].first + collection["elements"] << collection["elements"].first.merge("value" => "other") + result = lint(passport) + assert_equal "failed", criterion(result, "DPP-DAT-017")["result"] + assert_match(/ModelIdentifier 2 times in ProductIdentification/, criterion(result, "DPP-DAT-017")["messages"].first["message"]) + + passport = reference + passport["elements"].last["elements"] << passport["elements"].first["elements"].first + result = lint(passport) + assert_equal "passed", criterion(result, "DPP-DAT-017")["result"] + assert_equal "warning", criterion(result, "DPP-DAT-021")["result"] + end + + test "a compressed passport posted with a repeated member name fails DPP-DAT-017" do + result = lint_text(file_fixture("element_ids/DPP-DAT-017/invalid/compressed-repeated-member.json").read) + assert_equal "failed", criterion(result, "DPP-DAT-017")["result"] + assert_equal %w[skipped not_evaluated], criterion(result, "DPP-DAT-021").values_at("result", "reason_code") + end + test "optional facility identifier is skipped when absent" do assert_equal "skipped", criterion(lint(reference), "DPP-ID-010")["result"] end test "checks that request the product identifier need one" do result = lint(reference) - %w[DPP-ID-001 DPP-ID-013 DPP-DAT-003 DPP-DAT-016 DPP-SEC-014 DPP-SEC-020].each do |id| + %w[DPP-ID-001 DPP-ID-013 DPP-DAT-003 DPP-DAT-016 DPP-SEC-014 DPP-SEC-020 DPP-SEC-021].each do |id| assert_equal "skipped", criterion(result, id)["result"], id end assert_equal "rated by dpp-validator from its daily runs", criterion(result, "DPP-ID-002")["reason"] diff --git a/test/services/element_ids_check_test.rb b/test/services/element_ids_check_test.rb new file mode 100644 index 0000000..d6656cf --- /dev/null +++ b/test/services/element_ids_check_test.rb @@ -0,0 +1,98 @@ +require "test_helper" + +class ElementIdsCheckTest < ActiveSupport::TestCase + LEVEL = { "type" => "element_ids", "unique" => "level" }.freeze + PASSPORT = { "type" => "element_ids", "unique" => "passport" }.freeze + VECTORS = Rails.root.join("test/fixtures/files/element_ids") + + def outcome(check, text, source: true) + ElementIdsCheck.new(check, JSON.parse(text), source: source ? text : nil).outcome + end + + def vectors(id, kind) = Dir[VECTORS.join(id, kind, "*.json")].sort + + # Test vectors: DPP-DAT-017 (unique: level) and DPP-DAT-021 (unique: passport). + test "DPP-DAT-017 test vectors: valid ones pass, invalid ones fail" do + assert_operator vectors("DPP-DAT-017", "valid").size, :>=, 4 + vectors("DPP-DAT-017", "valid").each do |file| + o = outcome(LEVEL, File.read(file)) + assert_nil o.skipped, file + assert_empty o.messages, file + end + assert_operator vectors("DPP-DAT-017", "invalid").size, :>=, 4 + vectors("DPP-DAT-017", "invalid").each do |file| + o = outcome(LEVEL, File.read(file)) + assert_equal ["violation"], o.messages.map { |m| m[:severity] }, file + end + end + + test "DPP-DAT-021 test vectors: valid ones pass, the others give a warning" do + vectors("DPP-DAT-021", "valid").each do |file| + assert_empty outcome(PASSPORT, File.read(file)).messages, file + end + assert_operator vectors("DPP-DAT-021", "warning").size, :>=, 2 + vectors("DPP-DAT-021", "warning").each do |file| + assert_equal ["warning"], outcome(PASSPORT, File.read(file)).messages.map { |m| m[:severity] }, file + end + end + + test "the reference passport passes both" do + text = file_fixture("reference-passport.json").read + assert_empty outcome(LEVEL, text).messages + assert_empty outcome(PASSPORT, text).messages + end + + test "messages name the repeated elementId, how often and where" do + text = VECTORS.join("DPP-DAT-017/invalid/full-collection-language-variants.json").read + assert_equal "elementId repeated within one level: SafetyInstructions 2 times in ProductInformation", + outcome(LEVEL, text).messages.first[:message] + text = VECTORS.join("DPP-DAT-017/invalid/full-top-level-repeated.json").read + assert_match(/ModelIdentifier 2 times at the top level of the passport/, outcome(LEVEL, text).messages.first[:message]) + text = VECTORS.join("DPP-DAT-017/invalid/full-multi-valued-item-repeated.json").read + assert_match(%r{efficiencyRating2 2 times in EnergyPerformance/efficiencyRatings}, outcome(LEVEL, text).messages.first[:message]) + text = VECTORS.join("DPP-DAT-021/warning/full-same-id-in-two-collections.json").read + assert_equal "elementId used at more than one place of the passport: name at 2 places (Manufacturer/name, Importer/name)", + outcome(PASSPORT, text).messages.first[:message] + end + + test "a repeated elementId is reported once by the level check, not again as a second place" do + text = VECTORS.join("DPP-DAT-017/invalid/full-collection-language-variants.json").read + assert_empty outcome(PASSPORT, text).messages + end + + test "compressed serialization: repeated member names in the passport as delivered fail" do + text = VECTORS.join("DPP-DAT-017/invalid/compressed-repeated-member.json").read + assert_equal "member name repeated within one JSON object: maxPressure 2 times in the object at $['performanceMetrics']", + outcome(LEVEL, text).messages.first[:message] + end + + test "compressed serialization without the passport as delivered is skipped" do + text = VECTORS.join("DPP-DAT-017/valid/compressed-unique.json").read + o = outcome(LEVEL, text, source: false) + assert_equal "not_evaluated", o.code + assert_match(/not available as delivered/, o.skipped) + end + + test "the passport-wide check skips the compressed serialization" do + o = outcome(PASSPORT, VECTORS.join("DPP-DAT-017/valid/compressed-unique.json").read) + assert_equal "not_evaluated", o.code + assert_match(/compressed serialization/, o.skipped) + end + + test "the full serialization is checked on the parsed passport, without the bytes" do + text = VECTORS.join("DPP-DAT-017/invalid/full-top-level-repeated.json").read + assert_equal ["violation"], outcome(LEVEL, text, source: false).messages.map { |m| m[:severity] } + end + + test "unreadable bytes skip the compressed check" do + o = ElementIdsCheck.new(LEVEL, { "dppStatus" => "Active" }, source: '{"dppStatus": "Active",}').outcome + assert_equal "not_evaluated", o.code + assert_match(/cannot be read/, o.skipped) + end + + test "more than ten findings are cut short" do + elements = (1..12).flat_map { |i| [{ "elementId" => "e#{i}" }, { "elementId" => "e#{i}" }] } + o = ElementIdsCheck.new(LEVEL, { "elements" => elements }).outcome + assert_match(/e10 2 times at the top level of the passport and 2 more\z/, o.messages.first[:message]) + end +end diff --git a/test/services/html_check_device_test.rb b/test/services/html_check_device_test.rb new file mode 100644 index 0000000..b76ed90 --- /dev/null +++ b/test/services/html_check_device_test.rb @@ -0,0 +1,122 @@ +require "test_helper" + +# html with device_features: forbidden (DPP-SEC-021): static search for +# browser interfaces that use functions of the reader's device. +class HtmlCheckDeviceTest < ActiveSupport::TestCase + UPI = "https://dpp.example.org/01/09520123456788/21/0001".freeze + CHECK = { "type" => "html", "device_features" => "forbidden" }.freeze + VECTORS = Rails.root.join("test/fixtures/files/html/DPP-SEC-021") + + # The page for the product identifier; scripts under /scripts/ from the + # test vectors; every other URL is recorded and answered with 404. + class FakeResolver + attr_reader :requested + + def initialize(page, scripts: {}) = (@page, @scripts, @requested = page, scripts, []) + + def get(url, accept: nil) + @requested << [url, accept] + return response(url, 200, "text/html; charset=utf-8", @page) if url == UPI + return response(url, 200, "text/javascript", @scripts[url]) if @scripts.key?(url) + + response(url, 404, "text/plain", "not found") + end + + def response(url, status, type, body) = HttpResolver::Response.new(url: url, status: status, content_type: type, body: body, headers: {}) + end + + def scripts + Dir[VECTORS.join("scripts", "*.js")].to_h { |f| ["https://dpp.example.org/scripts/#{File.basename(f)}", File.read(f)] } + end + + def outcome(html, check: CHECK) + resolver = FakeResolver.new(html, scripts: scripts) + [HtmlCheck.new(check, UPI, resolver).outcome, resolver] + end + + def page(head = "", body = "") = "#{head}#{body}" + + def finding(html) = outcome(html).first.messages.first&.dig(:message) + + test "DPP-SEC-021 test vectors: valid ones pass, invalid ones fail" do + valid = Dir[VECTORS.join("valid", "*.html")].sort + invalid = Dir[VECTORS.join("invalid", "*.html")].sort + assert_operator valid.size, :>=, 5 + assert_operator invalid.size, :>=, 8 + valid.each { |f| assert_empty outcome(File.read(f)).first.messages, f } + invalid.each { |f| assert_equal ["violation"], outcome(File.read(f)).first.messages.map { |m| m[:severity] }, f } + end + + test "findings name the function, the interface and the place" do + assert_equal "the HTML view uses functions of the reader's device: location (.geolocation in inline script 1)", + finding(VECTORS.join("invalid/inline-script-location.html").read) + assert_match(/notifications \(Notification\.requestPermission in attribute onclick of button\)/, + finding(VECTORS.join("invalid/event-handler-notifications.html").read)) + assert_match(%r{camera, microphone or screen \(\.mediaDevices in https://dpp\.example\.org/scripts/scanner\.js\)}, + finding(VECTORS.join("invalid/own-script-camera.html").read)) + assert_match(%r{motion or orientation sensors \(devicemotion in https://dpp\.example\.org/scripts/sensors\.js\)}, + finding(VECTORS.join("invalid/modulepreload-sensors.html").read)) + assert_match(/iframe allow grants camera\z/, finding(VECTORS.join("invalid/iframe-allow-camera.html").read)) + assert_match(/file input with capture/, finding(VECTORS.join("invalid/file-input-capture.html").read)) + assert_match(/device behaviour \(navigator\.vibrate in attribute href of a\)/, + finding(VECTORS.join("invalid/javascript-url-vibration.html").read)) + assert_match(/permission element/, finding(VECTORS.join("invalid/permission-element.html").read)) + end + + test "only scripts from the page's own host are fetched, as */*" do + _, resolver = outcome(page('' \ + '')) + assert_equal [[UPI, "text/html"], ["https://dpp.example.org/scripts/render.js", "*/*"]], resolver.requested + end + + test "at most ten scripts are fetched" do + tags = (1..12).map { |i| %() }.join + _, resolver = outcome(page(tags)) + assert_equal 11, resolver.requested.size + end + + test "a script that cannot be retrieved is not checked" do + assert_empty outcome(page('')).first.messages + end + + test "each interface group is recognised" do + { + "navigator.geolocation.watchPosition(f)" => "location", + "navigator.mediaDevices.getDisplayMedia()" => "camera, microphone or screen", + "reg.pushManager.subscribe()" => "notifications", + "new Notification('x')" => "notifications", + "navigator.contacts.select(['tel'])" => "contacts", + "navigator.bluetooth.requestDevice({})" => "Bluetooth, USB, serial, HID or NFC", + "new NDEFReader()" => "Bluetooth, USB, serial, HID or NFC", + "DeviceOrientationEvent.requestPermission()" => "motion or orientation sensors", + "new Gyroscope({ frequency: 60 })" => "motion or orientation sensors", + "window.ondeviceorientation = f" => "motion or orientation sensors", + "navigator.connection.effectiveType" => "context or network information", + "navigator.getBattery()" => "context or network information", + "new IdleDetector()" => "context or network information", + "navigator.credentials.get({ otp: { transport: ['sms'] } })" => "one-time codes from SMS", + "addEventListener('beforeinstallprompt', f)" => "app installation", + "navigator.wakeLock.request('screen')" => "device behaviour" + }.each do |code, label| + assert_equal [label], HtmlCheck.device_uses(code).map(&:first), code + end + end + + test "ordinary code and words without interface calls are not counted" do + ["const location = window.location.href;", "fetch('/dpp.json').then(r => r.json())", + "document.title = 'geolocation of the factory';", "navigator.language", "Notification"].each do |code| + assert_empty HtmlCheck.device_uses(code), code + end + end + + test "a page without device use passes, together with external_resources" do + o, = outcome(page(''), check: CHECK.merge("external_resources" => "forbidden")) + assert_empty o.messages + end + + test "both options give a message each" do + o, = outcome(page(''), + check: CHECK.merge("external_resources" => "forbidden")) + assert_equal 2, o.messages.size + end +end diff --git a/test/services/json_duplicates_test.rb b/test/services/json_duplicates_test.rb new file mode 100644 index 0000000..bd4dfa5 --- /dev/null +++ b/test/services/json_duplicates_test.rb @@ -0,0 +1,39 @@ +require "test_helper" + +class JsonDuplicatesTest < ActiveSupport::TestCase + test "no repeated names" do + assert_empty JsonDuplicates.find('{"a": 1, "b": {"a": 2}, "c": [{"a": 1}, {"a": 2}]}') + end + + test "a repeated name with its normalized path and count" do + assert_equal [{ path: "$['b'][1]", name: "x", count: 3 }], + JsonDuplicates.find('{"a":1,"b":[{"x":1},{"x":1,"x":2,"x":3}]}') + end + + test "names are compared after their escapes are resolved" do + assert_equal [{ path: "$", name: "a", count: 2 }], JsonDuplicates.find('{"a": 1, "a": 2}') + end + + test "paths escape quotes, backslashes and control characters" do + found = JsonDuplicates.find(%({"it's\\\\\\n": {"k": 1, "k": 2}})) + assert_equal "$['it\\'s\\\\\\n']", found.first[:path] + end + + test "strings, numbers and literals are skipped, also with escaped quotes" do + assert_empty JsonDuplicates.find(%([1, -2.5e3, true, false, null, "a\\"b", {"q": "}"}])) + end + + test "a byte order mark is accepted" do + assert_empty JsonDuplicates.find("{\"a\": 1}") + end + + test "text that is not one JSON value is refused" do + ['{"a": 1,}', '{"a" 1}', "[1 2]", '{"a": 1} x', "nul", "\xFF".b].each do |text| + assert_raises(JsonDuplicates::Invalid, text) { JsonDuplicates.find(text) } + end + end + + test "very deep nesting is refused" do + assert_raises(JsonDuplicates::Invalid) { JsonDuplicates.find(("[" * 600) + ("]" * 600)) } + end +end diff --git a/test/services/passport_linter_test.rb b/test/services/passport_linter_test.rb index a562dab..735239d 100644 --- a/test/services/passport_linter_test.rb +++ b/test/services/passport_linter_test.rb @@ -156,4 +156,25 @@ def lint(responses, check = CHECK) assert_equal "skipped", result[:result] assert_match(/uses a feature outside the portable subset of CRITERIA-FORMAT.md that dpplint does not evaluate \(lookahead\)/, result[:reason]) end + + test "element_ids criteria check the passport as delivered: the response bytes, or the posted body" do + text = File.read(Rails.root.join("test/fixtures/files/element_ids/DPP-DAT-017/invalid/compressed-repeated-member.json")) + check = { "type" => "element_ids", "unique" => "level" } + linter = -> { PassportLinter.new(catalogue: FakeCatalogue.new([criterion(check).merge("id" => "DPP-DAT-017")])) } + posted = linter.call.run(passport: JSON.parse(text), source: text)[:criteria].first + assert_equal "failed", posted[:result] + assert_match(/maxPressure 2 times/, posted[:messages].first[:message]) + assert_equal "failed", linter.call.run(passport: JSON.parse(text), raw: text.b)[:criteria].first[:result] + without = linter.call.run(passport: JSON.parse(text))[:criteria].first + assert_equal %w[skipped not_evaluated], without.values_at(:result, :reason_code) + end + + test "a warning of an element_ids criterion leaves it counted as passed" do + text = File.read(Rails.root.join("test/fixtures/files/element_ids/DPP-DAT-021/warning/full-same-id-in-two-collections.json")) + check = { "type" => "element_ids", "unique" => "passport" } + report = PassportLinter.new(catalogue: FakeCatalogue.new([criterion(check).merge("id" => "DPP-DAT-021")])) + .run(passport: JSON.parse(text)) + assert_equal "warning", report[:criteria].first[:result] + assert_equal "1 of 1 automated checks passed", report[:summary][:text] + end end