From 93167e354e60bcf9b347f2b141326c41a714341f Mon Sep 17 00:00:00 2001 From: pc418 <271863285+pc418@users.noreply.github.com> Date: Sun, 20 Sep 2026 22:41:23 -0700 Subject: [PATCH] Add: track the clipping path in the chunk parser GraphicsState had no clip and parseChunk had no case for W or W*, so a text chunk never knew whether anything could be seen of it. A figure placed many times under small windows -- a tiled or cropped figure, an ordinary export shape -- therefore had every copy of its text extracted, not just the one on the page. Track the clip as the bounding box of the clipping path in page space, save and restore it with the rest of the graphics state, compose it with a form's own /BBox on Do, and mark a text chunk whose box is wholly outside it. Marking only: whether to drop such a chunk is the consumer's decision, and this is deliberately not the hidden-text flag, which means low contrast and has its own consumers. The discard rule is conservative. Only a chunk separated from the clip by more than 1e-3 pt is marked; one that touches the clip or crosses it keeps its text, and a degenerate box is never marked. Non-rectangular paths are approximated by their bounding box, which only widens the clip, so disjointness still proves invisibility while overlap proves nothing. The coordinate contract is written out above the clip code. --- pom.xml | 1 + wcag-validation/pom.xml | 6 + .../gf/model/factory/chunks/ChunkParser.java | 163 ++++++++++- .../model/factory/chunks/GraphicsState.java | 52 ++++ .../model/factory/chunks/ClipTestChunks.java | 99 +++++++ .../factory/chunks/ClipTestPdfBuilder.java | 190 +++++++++++++ .../chunks/ClippedTextChunkParserTest.java | 261 ++++++++++++++++++ 7 files changed, 771 insertions(+), 1 deletion(-) create mode 100644 wcag-validation/src/test/java/org/verapdf/gf/model/factory/chunks/ClipTestChunks.java create mode 100644 wcag-validation/src/test/java/org/verapdf/gf/model/factory/chunks/ClipTestPdfBuilder.java create mode 100644 wcag-validation/src/test/java/org/verapdf/gf/model/factory/chunks/ClippedTextChunkParserTest.java diff --git a/pom.xml b/pom.xml index 562520b64..6f5559f72 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,7 @@ [1.31.0,1.32.0-RC) [1.31.0,1.32.0-RC) [1.31.0,1.32.0-RC) + 5.6.2 diff --git a/wcag-validation/pom.xml b/wcag-validation/pom.xml index c21d03bf0..30bf0739a 100644 --- a/wcag-validation/pom.xml +++ b/wcag-validation/pom.xml @@ -54,6 +54,12 @@ wcag-algorithms ${verapdf.wcag.algs.version} + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + diff --git a/wcag-validation/src/main/java/org/verapdf/gf/model/factory/chunks/ChunkParser.java b/wcag-validation/src/main/java/org/verapdf/gf/model/factory/chunks/ChunkParser.java index a5d8dc001..c1148d640 100644 --- a/wcag-validation/src/main/java/org/verapdf/gf/model/factory/chunks/ChunkParser.java +++ b/wcag-validation/src/main/java/org/verapdf/gf/model/factory/chunks/ChunkParser.java @@ -74,6 +74,13 @@ public class ChunkParser { */ public static final double TEXT_SPACE_UNIT = 1.0 / 1000.0; + /** + * How much two boxes may be apart before they count as disjoint, in points. + * A chunk is called clipped only when it is separated from the clip by more + * than this, so a chunk that merely touches the clip boundary keeps its text. + */ + private static final double CLIP_TOLERANCE = 1.0e-3; + private final Deque graphicsStateStack = new ArrayDeque<>(); private final Stack markedContentStack = new Stack<>(); private final Stack visibleContentStack = new Stack(); @@ -87,6 +94,13 @@ public class ChunkParser { private final PDResourcesHandler resourcesHandler; private final GraphicsState graphicsState; private final Path path = new Path(); + /** + * Bounding box, in page space, of the path currently being constructed -- + * the candidate clip. Null while no path point has been seen. + */ + private BoundingBox clipPathBox = null; + /** Set by W / W*, cleared by the operator that ends the path. */ + private boolean pendingClip = false; private final List artifacts = new LinkedList<>(); private List nonDrawingArtifacts = new ArrayList<>(); private final LineArtContainer lineArtContainer; @@ -402,6 +416,11 @@ public void parseChunk(Operator rawOperator, List arguments, int operat new Vertex(arguments.get(2).getReal(), arguments.get(3).getReal()), new Vertex(arguments.get(4).getReal(), arguments.get(5).getReal()), graphicsState.getLineWidth()); + // A Bezier curve stays inside the hull of its control points, + // so the control polygon is a safe over-approximation. + addClipPathPoint(arguments.get(0).getReal(), arguments.get(1).getReal()); + addClipPathPoint(arguments.get(2).getReal(), arguments.get(3).getReal()); + addClipPathPoint(arguments.get(4).getReal(), arguments.get(5).getReal()); path.setCurrentPoint(curve.getX3(), curve.getY3()); nonDrawingArtifacts.add(curve); } @@ -414,6 +433,7 @@ public void parseChunk(Operator rawOperator, List arguments, int operat case Operators.F_STAR_FILL: processh(); processf(operatorIndex); + applyPendingClip(); break; case Operators.GS: PDExtGState extGState = this.resourcesHandler.getExtGState(getLastCOSName(arguments)); @@ -428,6 +448,7 @@ public void parseChunk(Operator rawOperator, List arguments, int operat nonDrawingArtifacts.add(new LineChunk(pageNumber, path.getCurrentX(), path.getCurrentY(), x, y, graphicsState.getLineWidth())); } + addClipPathPoint(x, y); path.setCurrentPoint(x, y); } break; @@ -436,6 +457,7 @@ public void parseChunk(Operator rawOperator, List arguments, int operat arguments.get(1).getType().isNumber()) { double x = arguments.get(0).getReal(); double y = arguments.get(1).getReal(); + addClipPathPoint(x, y); path.setStartPoint(x, y); path.setCurrentPoint(x, y); } @@ -459,7 +481,15 @@ public void parseChunk(Operator rawOperator, List arguments, int operat arguments.get(3).getType().isNumber()) { double x = arguments.get(0).getReal(); double y = arguments.get(1).getReal(); - nonDrawingArtifacts.add(new Rectangle(pageNumber, x, y, arguments.get(2).getReal(), arguments.get(3).getReal())); + double width = arguments.get(2).getReal(); + double height = arguments.get(3).getReal(); + nonDrawingArtifacts.add(new Rectangle(pageNumber, x, y, width, height)); + // All four corners: under a rotating or shearing CTM the + // rectangle is not axis aligned in page space. + addClipPathPoint(x, y); + addClipPathPoint(x + width, y); + addClipPathPoint(x + width, y + height); + addClipPathPoint(x, y + height); path.setCurrentPoint(x, y); path.setStartPoint(x, y); } @@ -472,6 +502,8 @@ public void parseChunk(Operator rawOperator, List arguments, int operat new Vertex(arguments.get(0).getReal(), arguments.get(1).getReal()), new Vertex(arguments.get(2).getReal(), arguments.get(3).getReal()), graphicsState.getLineWidth(), true); + addClipPathPoint(arguments.get(0).getReal(), arguments.get(1).getReal()); + addClipPathPoint(arguments.get(2).getReal(), arguments.get(3).getReal()); path.setCurrentPoint(curve.getX3(), curve.getY3()); nonDrawingArtifacts.add(curve); } @@ -484,6 +516,8 @@ public void parseChunk(Operator rawOperator, List arguments, int operat new Vertex(arguments.get(0).getReal(), arguments.get(1).getReal()), new Vertex(arguments.get(2).getReal(), arguments.get(3).getReal()), graphicsState.getLineWidth(), false); + addClipPathPoint(arguments.get(0).getReal(), arguments.get(1).getReal()); + addClipPathPoint(arguments.get(2).getReal(), arguments.get(3).getReal()); path.setCurrentPoint(curve.getX3(), curve.getY3()); nonDrawingArtifacts.add(curve); } @@ -492,20 +526,32 @@ public void parseChunk(Operator rawOperator, List arguments, int operat case Operators.B_STAR_CLOSEPATH_EOFILL_STROKE: processh(); processB(operatorIndex); + applyPendingClip(); break; case Operators.B_FILL_STROKE: case Operators.B_STAR_EOFILL_STROKE: processB(operatorIndex); + applyPendingClip(); + break; + case Operators.W_CLIP: + case Operators.W_STAR_EOCLIP: + // W and W* do not clip by themselves: they arm the clip, and the + // operator that ends the path applies it. See the coordinate + // contract on applyPendingClip. + pendingClip = true; break; case Operators.N: nonDrawingArtifacts = new ArrayList<>(); + applyPendingClip(); break; case Operators.S_CLOSE_STROKE: processh(); processS(operatorIndex); + applyPendingClip(); break; case Operators.S_STROKE: processS(operatorIndex); + applyPendingClip(); break; case Operators.CM_CONCAT: if (arguments.size() == 6) { @@ -547,6 +593,10 @@ public void parseChunk(Operator rawOperator, List arguments, int operat } GraphicsState xFormGraphicsState = graphicsState.clone(); xFormGraphicsState.getCTM().concatenate(new Matrix(((PDXForm) xObject).getMatrix())); + // A form's /BBox clips its own content (ISO 32000-1, + // 8.10.2), and the clone already carries the clip + // inherited from here, so the two compose. + intersectFormBBoxClip(xFormGraphicsState, (PDXForm) xObject); GFSAXForm xForm = new GFSAXForm((PDXForm) xObject, resourcesHandler, xFormGraphicsState, pageNumber, key, markedContent, xObjectName.getName().getValue()); artifacts.addAll(xForm.getArtifacts()); @@ -602,6 +652,116 @@ private void processDoubleQuote(double op1, double op2) { this.graphicsState.getTextState().setCharacterSpacing(op2); } + /* + * ------------------------------------------------------------------ + * Clipping: the coordinate contract + * ------------------------------------------------------------------ + * + * Path points are transformed by the CTM current at each path + * construction operator (m, l, re, c, v, y), not by whatever CTM the + * painting operator happens to see, and are accumulated into + * clipPathBox in page space -- the same space createTextChunk's + * bounding box is built in, because the text rendering matrix that + * produces it ends with the same CTM. + * + * W and W* do not clip. They set pendingClip, and the pending clip is + * applied after the path painting operator that ends the path (n, f, F, + * f*, B, B*, b, b*, S, s), using the bounding box of the whole + * constructed path; the path is then reset. + * + * The effective clip is the intersection of the inherited clip and that + * box. Null means unclipped; an EMPTY intersection stays an empty box + * and must never become "unclipped" again (see GraphicsState). + * + * q and Q save and restore the clip with the rest of the graphics + * state, because GraphicsState carries it through both clone() and + * copyProperties. + * + * On Do of a Form XObject the nested parser inherits the clip through + * the cloned graphics state and additionally intersects the form's + * /BBox transformed by /Matrix x CTM, so nesting composes. + * + * Non-rectangular clipping paths are approximated by their bounding + * box. That approximation only ever makes the clip LARGER, so + * disjointness still proves invisibility -- but overlap proves nothing, + * and no visibility is claimed. Text inside the box but outside the + * real path is therefore retained. + */ + + /** + * Extends the path under construction with one point, transformed by the + * CTM in force right now. + */ + private void addClipPathPoint(double x, double y) { + double pageX = graphicsState.getCTM().transformX(x, y); + double pageY = graphicsState.getCTM().transformY(x, y); + if (clipPathBox == null) { + clipPathBox = new BoundingBox(pageNumber, pageX, pageY, pageX, pageY); + return; + } + clipPathBox.setLeftX(Math.min(clipPathBox.getLeftX(), pageX)); + clipPathBox.setRightX(Math.max(clipPathBox.getRightX(), pageX)); + clipPathBox.setBottomY(Math.min(clipPathBox.getBottomY(), pageY)); + clipPathBox.setTopY(Math.max(clipPathBox.getTopY(), pageY)); + } + + /** + * Applies the clip armed by W / W*, if any, and resets the path. Called by + * every path painting operator, so that a path built without W also ends + * here. + * + *

A W with no path point at all is left as a no-op rather than read as + * an empty clip: the conservative reading of a degenerate stream is that + * nothing was clipped. + */ + private void applyPendingClip() { + if (pendingClip && clipPathBox != null) { + graphicsState.intersectClip(clipPathBox); + } + pendingClip = false; + clipPathBox = null; + } + + /** + * Intersects a form's own /BBox, transformed by the CTM the form will be + * parsed with, into that state's clip. + */ + private void intersectFormBBoxClip(GraphicsState state, PDXForm form) { + double[] bbox = form.getBBox(); + if (bbox == null || bbox.length != 4) { + return; + } + state.intersectClip(state.getCTM().transformBoundingBox( + new BoundingBox(pageNumber, bbox[0], bbox[1], bbox[2], bbox[3]))); + } + + /** + * Marks a chunk whose bounding box is wholly outside the effective clip, so + * a consumer can drop text that cannot appear on the page. + * + *

Conservative on purpose: only a box separated from the clip by more + * than CLIP_TOLERANCE counts. A chunk that touches the clip boundary or + * crosses it keeps its text, because part of it is readable and a chunk is + * not split at the boundary. A degenerate (zero area) text box has no + * geometry to place outside the clip and is never marked. + */ + private void markIfClipped(TextChunk textChunk) { + BoundingBox clip = graphicsState.getClipBox(); + if (clip == null || clip.getPageNumber() == null) { + return; + } + BoundingBox box = textChunk.getBoundingBox(); + if (box == null || box.getPageNumber() == null) { + return; + } + if (box.getWidth() <= CLIP_TOLERANCE || box.getHeight() <= CLIP_TOLERANCE) { + return; + } + if (clip.notOverlaps(box, CLIP_TOLERANCE)) { + textChunk.setClippedText(true); + } + } + private void processh() { if (!NodeUtils.areCloseNumbers(path.getStartX(), path.getCurrentX()) || !NodeUtils.areCloseNumbers(path.getStartY(), path.getCurrentY())) { @@ -978,6 +1138,7 @@ private TextChunk createTextChunk(List arguments, String operatorType, if (StaticContainers.isDataLoader()) { textChunk.getStreamInfos().addAll(textPieces.getStreamInfos(operatorIndex, xObjectName)); } + markIfClipped(textChunk); if (StaticContainers.isDataLoader() && !fontNameToFontFamilyMap.containsKey(fontNameWithoutSubset)) { String fontFamily = descriptor.getFontFamily(); if (fontFamily == null || fontFamily.isEmpty()) { diff --git a/wcag-validation/src/main/java/org/verapdf/gf/model/factory/chunks/GraphicsState.java b/wcag-validation/src/main/java/org/verapdf/gf/model/factory/chunks/GraphicsState.java index 93b4b9791..42ef471b6 100644 --- a/wcag-validation/src/main/java/org/verapdf/gf/model/factory/chunks/GraphicsState.java +++ b/wcag-validation/src/main/java/org/verapdf/gf/model/factory/chunks/GraphicsState.java @@ -27,6 +27,7 @@ import org.verapdf.pd.colors.PDColorSpace; import org.verapdf.pd.font.PDFont; import org.verapdf.wcag.algorithms.entities.content.LineChunk; +import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox; /** * @author Maxim Plushchov @@ -40,6 +41,16 @@ public class GraphicsState implements Cloneable { private boolean processColorOperators = true; private double lineWidth = 1.0; private int lineCap = LineChunk.BUTT_CAP_STYLE; + /** + * The effective clipping path in force, approximated by its bounding box in + * page space, or null when nothing has clipped this state yet. + * + *

null means unclipped. An empty box means the clip admits + * nothing, and it is kept as an empty box: an empty intersection must never + * collapse back to null, because "nothing is visible" and "everything is + * visible" are opposite answers. + */ + private BoundingBox clipBox; private GraphicsState() { @@ -105,6 +116,45 @@ public void setLineCap(int lineCap) { this.lineCap = lineCap; } + /** + * @return the bounding box of the effective clipping path in page space, or + * null when this state is unclipped + */ + public BoundingBox getClipBox() { + return clipBox; + } + + /** + * Intersects the effective clipping path with another box, in page space. + * The first box to arrive becomes the clip; every later one narrows it. + * + * @param box the new clip box in page space; null and boxes without a page + * number are ignored, since neither can be compared with a chunk + */ + public void intersectClip(BoundingBox box) { + if (box == null || box.getPageNumber() == null) { + return; + } + this.clipBox = this.clipBox == null ? new BoundingBox(box) : intersection(this.clipBox, box); + } + + /** + * BoundingBox.cross returns null for disjoint boxes and the public + * constructors order their corners, so neither can express an empty clip. + * Detect the empty case here and answer the canonical empty box, which + * reports isEmpty() and does not overlap anything. + */ + private static BoundingBox intersection(BoundingBox first, BoundingBox second) { + double leftX = Math.max(first.getLeftX(), second.getLeftX()); + double rightX = Math.min(first.getRightX(), second.getRightX()); + double bottomY = Math.max(first.getBottomY(), second.getBottomY()); + double topY = Math.min(first.getTopY(), second.getTopY()); + if (leftX > rightX || bottomY > topY) { + return new BoundingBox(first.getPageNumber()); + } + return new BoundingBox(first.getPageNumber(), leftX, bottomY, rightX, topY); + } + public void copyProperties(GraphicsState graphicState) { this.CTM = graphicState.getCTM(); this.textState = graphicState.getTextState(); @@ -113,6 +163,7 @@ public void copyProperties(GraphicsState graphicState) { this.processColorOperators = graphicState.isProcessColorOperators(); this.lineWidth = graphicState.getLineWidth(); this.lineCap = graphicState.getLineCap(); + this.clipBox = graphicState.getClipBox(); } public void copyPropertiesFromExtGState(PDExtGState extGState) { @@ -138,6 +189,7 @@ public GraphicsState clone() { clone.processColorOperators = this.processColorOperators; clone.lineWidth = this.lineWidth; clone.lineCap = this.lineCap; + clone.clipBox = this.clipBox == null ? null : new BoundingBox(this.clipBox); return clone; } } diff --git a/wcag-validation/src/test/java/org/verapdf/gf/model/factory/chunks/ClipTestChunks.java b/wcag-validation/src/test/java/org/verapdf/gf/model/factory/chunks/ClipTestChunks.java new file mode 100644 index 000000000..e850f2d0c --- /dev/null +++ b/wcag-validation/src/test/java/org/verapdf/gf/model/factory/chunks/ClipTestChunks.java @@ -0,0 +1,99 @@ +/* + * This file is part of veraPDF WCAG Validation, a module of the veraPDF project. + * Copyright (c) 2015-2026, veraPDF Consortium + * All rights reserved. + * + * veraPDF WCAG Validation is free software: you can redistribute it and/or modify + * it under the terms of either: + * + * The GNU General public license GPLv3+. + * You should have received a copy of the GNU General Public License + * along with veraPDF WCAG Validation as the LICENSE.GPL file in the root of the source + * tree. If not, see http://www.gnu.org/licenses/ or + * https://www.gnu.org/licenses/gpl-3.0.en.html. + * + * The Mozilla Public License MPLv2+. + * You should have received a copy of the Mozilla Public License along with + * veraPDF WCAG Validation as the LICENSE.MPL file in the root of the source tree. + * If a copy of the MPL was not distributed with this file, you can obtain one at + * http://mozilla.org/MPL/2.0/. + */ +package org.verapdf.gf.model.factory.chunks; + +import org.verapdf.containers.StaticCoreContainers; +import org.verapdf.gf.model.impl.containers.StaticStorages; +import org.verapdf.gf.model.impl.sa.GFSAPDFDocument; +import org.verapdf.parser.PDFFlavour; +import org.verapdf.pd.PDDocument; +import org.verapdf.tools.StaticResources; +import org.verapdf.wcag.algorithms.entities.content.IChunk; +import org.verapdf.wcag.algorithms.entities.content.TextChunk; +import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Runs the chunk parser over a fixture and hands back the page's text chunks, + * so a test can assert on which strings survived clipping. + */ +final class ClipTestChunks { + + private ClipTestChunks() { + } + + /** Parses page 0 of the file and returns its text chunks, in stream order. */ + static List parse(Path file) throws IOException { + StaticResources.clear(); + StaticContainers.updateContainers(null); + StaticCoreContainers.clearAllContainers(); + + PDDocument pdDocument = new PDDocument(file.toAbsolutePath().toString()); + try { + StaticResources.setDocument(pdDocument); + GFSAPDFDocument document = new GFSAPDFDocument(pdDocument); + StaticResources.setFlavour(Collections.singletonList(PDFFlavour.WCAG_2_2_HUMAN)); + StaticContainers.setDocument(document); + StaticContainers.setFileName(file.toAbsolutePath().toString()); + StaticContainers.setIsDataLoader(true); + StaticContainers.setIsIgnoreCharactersWithoutUnicode(false); + StaticResources.setIsFontProgramsParsing(true); + StaticStorages.setIsIgnoreMCIDs(true); + StaticStorages.setIsAddSpacesBetweenTextPieces(true); + document.parseChunks(); + + List chunks = new ArrayList<>(); + for (IChunk chunk : document.getArtifacts(0)) { + if (chunk instanceof TextChunk) { + chunks.add((TextChunk) chunk); + } + } + return chunks; + } finally { + pdDocument.close(); + } + } + + /** The values of the chunks a clipped-text consumer would keep. */ + static List retained(List chunks) { + return values(chunks, false); + } + + /** The values of the chunks the parser marked as clipped away. */ + static List clipped(List chunks) { + return values(chunks, true); + } + + private static List values(List chunks, boolean wanted) { + List values = new ArrayList<>(); + for (TextChunk chunk : chunks) { + if (chunk.isClippedText() == wanted) { + values.add(chunk.getValue().trim()); + } + } + return values; + } +} diff --git a/wcag-validation/src/test/java/org/verapdf/gf/model/factory/chunks/ClipTestPdfBuilder.java b/wcag-validation/src/test/java/org/verapdf/gf/model/factory/chunks/ClipTestPdfBuilder.java new file mode 100644 index 000000000..cc3c68209 --- /dev/null +++ b/wcag-validation/src/test/java/org/verapdf/gf/model/factory/chunks/ClipTestPdfBuilder.java @@ -0,0 +1,190 @@ +/* + * This file is part of veraPDF WCAG Validation, a module of the veraPDF project. + * Copyright (c) 2015-2026, veraPDF Consortium + * All rights reserved. + * + * veraPDF WCAG Validation is free software: you can redistribute it and/or modify + * it under the terms of either: + * + * The GNU General public license GPLv3+. + * You should have received a copy of the GNU General Public License + * along with veraPDF WCAG Validation as the LICENSE.GPL file in the root of the source + * tree. If not, see http://www.gnu.org/licenses/ or + * https://www.gnu.org/licenses/gpl-3.0.en.html. + * + * The Mozilla Public License MPLv2+. + * You should have received a copy of the Mozilla Public License along with + * veraPDF WCAG Validation as the LICENSE.MPL file in the root of the source tree. + * If a copy of the MPL was not distributed with this file, you can obtain one at + * http://mozilla.org/MPL/2.0/. + */ +package org.verapdf.gf.model.factory.chunks; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Writes a one page PDF byte by byte, so a clipping fixture is the content + * stream the test author wrote and nothing else. + * + *

A PDF writing library would put its own graphics state operators around + * the stream, which is exactly what these tests are measuring. The file is + * small enough to build by hand: a catalog, a page tree, one Helvetica font, + * the page's content stream and one stream per Form XObject. + */ +final class ClipTestPdfBuilder { + + static final double PAGE_WIDTH = 612; + static final double PAGE_HEIGHT = 792; + + private final Map forms = new LinkedHashMap<>(); + private String content = ""; + + /** A Form XObject: its own /BBox, its /Matrix and its content stream. */ + static final class Form { + private final double[] bbox; + private final double[] matrix; + private final String content; + + Form(double[] bbox, double[] matrix, String content) { + this.bbox = bbox; + this.matrix = matrix; + this.content = content; + } + } + + ClipTestPdfBuilder content(String pageContent) { + this.content = pageContent; + return this; + } + + ClipTestPdfBuilder form(String name, double[] bbox, String formContent) { + return form(name, bbox, new double[]{1, 0, 0, 1, 0, 0}, formContent); + } + + ClipTestPdfBuilder form(String name, double[] bbox, double[] matrix, String formContent) { + forms.put(name, new Form(bbox, matrix, formContent)); + return this; + } + + /** One {@code Tj} on its own line, at the given point, in 10pt Helvetica. */ + static String text(double x, double y, String value) { + return text(x, y, 10, value); + } + + static String text(double x, double y, double size, String value) { + return String.format("BT /F1 %s Tf %s %s Td (%s) Tj ET%n", + number(size), number(x), number(y), value.replace("\\", "\\\\") + .replace("(", "\\(").replace(")", "\\)")); + } + + static String number(double value) { + if (value == Math.rint(value)) { + return Long.toString((long) value); + } + return String.format("%.4f", value); + } + + Path write(Path file) throws IOException { + List objects = new ArrayList<>(); + // 1 catalog, 2 page tree, 3 font, 4 page, 5 page contents, then one + // stream object per form. Numbers are fixed so the references below + // can be written straight out. + int firstForm = 6; + StringBuilder xObjects = new StringBuilder(); + int number = firstForm; + for (String name : forms.keySet()) { + xObjects.append('/').append(name).append(' ').append(number++).append(" 0 R "); + } + String resources = "/Font << /F1 3 0 R >>" + + (xObjects.length() == 0 ? "" : " /XObject << " + xObjects + ">>"); + + objects.add(ascii("<< /Type /Catalog /Pages 2 0 R >>")); + objects.add(ascii("<< /Type /Pages /Kids [4 0 R] /Count 1 >>")); + objects.add(ascii("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica " + + "/Encoding /WinAnsiEncoding >>")); + objects.add(ascii(String.format("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 %s %s] " + + "/Resources << %s >> /Contents 5 0 R >>", + number(PAGE_WIDTH), number(PAGE_HEIGHT), resources))); + objects.add(stream("", content)); + for (Form form : forms.values()) { + objects.add(stream(String.format( + "/Type /XObject /Subtype /Form /BBox [%s %s %s %s] /Matrix [%s %s %s %s %s %s] " + + "/Resources << /Font << /F1 3 0 R >> >> ", + number(form.bbox[0]), number(form.bbox[1]), number(form.bbox[2]), number(form.bbox[3]), + number(form.matrix[0]), number(form.matrix[1]), number(form.matrix[2]), + number(form.matrix[3]), number(form.matrix[4]), number(form.matrix[5])), + form.content)); + } + + ByteBuffer out = new ByteBuffer(); + out.append(ascii("%PDF-1.7\n")); + out.append(new byte[]{'%', (byte) 0xe2, (byte) 0xe3, (byte) 0xcf, (byte) 0xd3, '\n'}); + List offsets = new ArrayList<>(); + for (int index = 0; index < objects.size(); index++) { + offsets.add(out.size()); + out.append(ascii((index + 1) + " 0 obj\n")); + out.append(objects.get(index)); + out.append(ascii("\nendobj\n")); + } + int startxref = out.size(); + StringBuilder xref = new StringBuilder(); + xref.append("xref\n0 ").append(objects.size() + 1).append('\n'); + xref.append("0000000000 65535 f \n"); + for (int offset : offsets) { + xref.append(String.format("%010d 00000 n %n", offset)); + } + xref.append("trailer\n<< /Size ").append(objects.size() + 1).append(" /Root 1 0 R >>\n"); + xref.append("startxref\n").append(startxref).append("\n%%EOF\n"); + out.append(ascii(xref.toString())); + + Files.createDirectories(file.getParent()); + Files.write(file, out.toByteArray()); + return file; + } + + private static byte[] stream(String extraDictionaryEntries, String body) { + byte[] bytes = ascii(body); + ByteBuffer buffer = new ByteBuffer(); + buffer.append(ascii("<< " + extraDictionaryEntries + "/Length " + bytes.length + " >>\nstream\n")); + buffer.append(bytes); + buffer.append(ascii("\nendstream")); + return buffer.toByteArray(); + } + + private static byte[] ascii(String value) { + return value.getBytes(StandardCharsets.ISO_8859_1); + } + + /** A growable byte sink; the file is a few kilobytes at most. */ + private static final class ByteBuffer { + private byte[] bytes = new byte[4096]; + private int length = 0; + + void append(byte[] more) { + while (length + more.length > bytes.length) { + byte[] grown = new byte[bytes.length * 2]; + System.arraycopy(bytes, 0, grown, 0, length); + bytes = grown; + } + System.arraycopy(more, 0, bytes, length, more.length); + length += more.length; + } + + int size() { + return length; + } + + byte[] toByteArray() { + byte[] result = new byte[length]; + System.arraycopy(bytes, 0, result, 0, length); + return result; + } + } +} diff --git a/wcag-validation/src/test/java/org/verapdf/gf/model/factory/chunks/ClippedTextChunkParserTest.java b/wcag-validation/src/test/java/org/verapdf/gf/model/factory/chunks/ClippedTextChunkParserTest.java new file mode 100644 index 000000000..384c465bd --- /dev/null +++ b/wcag-validation/src/test/java/org/verapdf/gf/model/factory/chunks/ClippedTextChunkParserTest.java @@ -0,0 +1,261 @@ +/* + * This file is part of veraPDF WCAG Validation, a module of the veraPDF project. + * Copyright (c) 2015-2026, veraPDF Consortium + * All rights reserved. + * + * veraPDF WCAG Validation is free software: you can redistribute it and/or modify + * it under the terms of either: + * + * The GNU General public license GPLv3+. + * You should have received a copy of the GNU General Public License + * along with veraPDF WCAG Validation as the LICENSE.GPL file in the root of the source + * tree. If not, see http://www.gnu.org/licenses/ or + * https://www.gnu.org/licenses/gpl-3.0.en.html. + * + * The Mozilla Public License MPLv2+. + * You should have received a copy of the Mozilla Public License along with + * veraPDF WCAG Validation as the LICENSE.MPL file in the root of the source tree. + * If a copy of the MPL was not distributed with this file, you can obtain one at + * http://mozilla.org/MPL/2.0/. + */ +package org.verapdf.gf.model.factory.chunks; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.verapdf.wcag.algorithms.entities.content.TextChunk; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The chunk parser's clipping geometry, asserted on the exact strings that + * survive and the exact strings that do not. + * + *

Line counts are not the acceptance criterion here: a filter can improve a + * count while deleting the wrong text, so every case names both sides. + * + *

Geometry note for whoever edits these fixtures: a Helvetica chunk's + * bounding box runs from the pen's x, and from {@code baseline - 0.225em} to + * {@code baseline + 0.931em}. At 10pt that is 2.25pt below and 9.31pt above + * the {@code Td} position. + */ +class ClippedTextChunkParserTest { + + @TempDir + Path tempDir; + + /** The five lines of the tiled figure, as the reproducer draws them. */ + private static String figureLines() { + return ClipTestPdfBuilder.text(0, 100, "Figure line 1") + + ClipTestPdfBuilder.text(0, 80, "Figure line 2") + + ClipTestPdfBuilder.text(0, 60, "Figure line 3") + + ClipTestPdfBuilder.text(0, 40, "Figure line 4") + + ClipTestPdfBuilder.text(0, 20, "Figure line 5"); + } + + /** + * The shape that started this: a Form XObject carrying every line of a + * figure, placed under a window that shows only the first one. Four fifths + * of the text is on the page and none of it can be seen. + */ + @Test + void formUnderAWindowKeepsOnlyTheLineTheWindowShows() throws Exception { + Path file = new ClipTestPdfBuilder() + .form("Fx", new double[]{0, 0, 200, 120}, figureLines()) + .content(ClipTestPdfBuilder.text(72, 700, 12, "Page body.") + + "q 1 0 0 1 100 400 cm 0 95 200 30 re W n /Fx Do Q\n") + .write(tempDir.resolve("reproducer.pdf")); + + List chunks = ClipTestChunks.parse(file); + + assertEquals(Arrays.asList("Page body.", "Figure line 1"), ClipTestChunks.retained(chunks)); + assertEquals(Arrays.asList("Figure line 2", "Figure line 3", "Figure line 4", "Figure line 5"), + ClipTestChunks.clipped(chunks)); + } + + /** + * The clipped-text signal must stay distinct from the hidden-text one, so + * that the hidden-text consumers do not react to clipped chunks and the + * clipped-text opt-out can be genuine. + */ + @Test + void clippedTextIsNotMarkedAsHiddenText() throws Exception { + Path file = new ClipTestPdfBuilder() + .form("Fx", new double[]{0, 0, 200, 120}, figureLines()) + .content("q 1 0 0 1 100 400 cm 0 95 200 30 re W n /Fx Do Q\n") + .write(tempDir.resolve("distinct.pdf")); + + List chunks = ClipTestChunks.parse(file); + + assertEquals(4, ClipTestChunks.clipped(chunks).size()); + for (TextChunk chunk : chunks) { + assertFalse(chunk.isHiddenText(), + "clipping must not set the hidden-text flag: " + chunk.getValue()); + } + } + + /** + * The discard rule is conservative: only text conclusively outside the clip + * goes. A chunk the clip cuts in half is still partly readable, and a chunk + * is not split at the boundary, so it keeps its text. + */ + @Test + void partiallyClippedChunkKeepsItsText() throws Exception { + Path file = new ClipTestPdfBuilder() + .content("q 72 690 40 30 re W n\n" + + ClipTestPdfBuilder.text(72, 700, "Partly visible line") + + "Q\n") + .write(tempDir.resolve("partial.pdf")); + + List chunks = ClipTestChunks.parse(file); + + assertEquals(Collections.singletonList("Partly visible line"), ClipTestChunks.retained(chunks)); + assertEquals(Collections.emptyList(), ClipTestChunks.clipped(chunks)); + } + + /** Touching the clip boundary is not being outside it. */ + @Test + void chunkTouchingTheClipBoundaryKeepsItsText() throws Exception { + Path file = new ClipTestPdfBuilder() + // The clip's right edge is exactly the pen's x, which is exactly + // the chunk's left edge. + .content("q 32 690 40 20 re W n\n" + + ClipTestPdfBuilder.text(72, 700, "Edge touching text") + + "Q\n") + .write(tempDir.resolve("touching.pdf")); + + List chunks = ClipTestChunks.parse(file); + + assertEquals(Collections.singletonList("Edge touching text"), ClipTestChunks.retained(chunks)); + } + + /** One point of clear separation is enough, so the rule is not inert. */ + @Test + void chunkOnePointBeyondTheClipIsMarked() throws Exception { + Path file = new ClipTestPdfBuilder() + .content("q 31 690 40 20 re W n\n" + + ClipTestPdfBuilder.text(72, 700, "One point clear") + + "Q\n") + .write(tempDir.resolve("separated.pdf")); + + List chunks = ClipTestChunks.parse(file); + + assertEquals(Collections.emptyList(), ClipTestChunks.retained(chunks)); + assertEquals(Collections.singletonList("One point clear"), ClipTestChunks.clipped(chunks)); + } + + /** Q restores the clip that was in force before q, so a later caption survives. */ + @Test + void restoringGraphicsStateRestoresTheEarlierClip() throws Exception { + Path file = new ClipTestPdfBuilder() + .content("q 0 0 10 10 re W n\n" + + ClipTestPdfBuilder.text(72, 700, "Inside the window") + + "Q\n" + + ClipTestPdfBuilder.text(72, 660, "Caption after Q")) + .write(tempDir.resolve("restore.pdf")); + + List chunks = ClipTestChunks.parse(file); + + assertEquals(Collections.singletonList("Caption after Q"), ClipTestChunks.retained(chunks)); + assertEquals(Collections.singletonList("Inside the window"), ClipTestChunks.clipped(chunks)); + } + + /** + * Two placements of one form are two graphics states: each gets its own + * transform and its own window, and neither leaks into the other. + */ + @Test + void twoPlacementsOfTheSameFormEachGetTheirOwnResult() throws Exception { + Path file = new ClipTestPdfBuilder() + .form("Fx", new double[]{0, 0, 200, 120}, figureLines()) + .content("q 1 0 0 1 100 400 cm 0 95 200 30 re W n /Fx Do Q\n" + + "q 1 0 0 1 100 200 cm 0 55 200 20 re W n /Fx Do Q\n") + .write(tempDir.resolve("placements.pdf")); + + List chunks = ClipTestChunks.parse(file); + + // The first window shows line 1, the second shows line 3. + assertEquals(Arrays.asList("Figure line 1", "Figure line 3"), ClipTestChunks.retained(chunks)); + assertEquals(Arrays.asList( + "Figure line 2", "Figure line 3", "Figure line 4", "Figure line 5", + "Figure line 1", "Figure line 2", "Figure line 4", "Figure line 5"), + ClipTestChunks.clipped(chunks)); + } + + /** + * A form's own /BBox clips its content, and composes with the clip it + * inherits. When the two do not meet, the effective clip is EMPTY and every + * chunk inside is clipped away -- an empty intersection must never be read + * back as "unclipped". + */ + @Test + void formBBoxAndInheritedClipCompose() throws Exception { + String formContent = ClipTestPdfBuilder.text(5, 10, "Low line") + + ClipTestPdfBuilder.text(5, 250, "High line"); + Path file = new ClipTestPdfBuilder() + .form("Fb", new double[]{0, 0, 100, 40}, formContent) + .content(ClipTestPdfBuilder.text(72, 700, 12, "Page body.") + // No inherited clip: the /BBox alone decides. "Low line" + // lands inside it, "High line" far above it. + + "q 1 0 0 1 100 100 cm /Fb Do Q\n" + // Inherited clip y 500..600 against a /BBox at y 300..340: + // the intersection is empty, so both lines go, although + // "Low line" is inside the /BBox and "High line" is + // inside the inherited clip. + + "q 0 500 612 100 re W n 1 0 0 1 100 300 cm /Fb Do Q\n") + .write(tempDir.resolve("bbox.pdf")); + + List chunks = ClipTestChunks.parse(file); + + assertEquals(Arrays.asList("Page body.", "Low line"), ClipTestChunks.retained(chunks)); + assertEquals(Arrays.asList("High line", "Low line", "High line"), ClipTestChunks.clipped(chunks)); + } + + /** + * A non-rectangular clipping path is approximated by its bounding box. The + * approximation only makes the clip larger, so text it lets through may + * still be invisible -- disjointness proves invisibility, overlap proves + * nothing. This pins the documented limitation rather than a wish. + */ + @Test + void nonRectangularClipIsApproximatedByItsBoundingBox() throws Exception { + Path file = new ClipTestPdfBuilder() + // A triangle with vertices (100,100), (300,100), (200,300); + // its bounding box is x 100..300, y 100..300. + .content("q 100 100 m 300 100 l 200 300 l h W n\n" + + ClipTestPdfBuilder.text(170, 150, "Inside the triangle") + + ClipTestPdfBuilder.text(105, 280, "Corner outside the triangle") + + ClipTestPdfBuilder.text(400, 500, "Outside the bounding box") + + "Q\n") + .write(tempDir.resolve("triangle.pdf")); + + List chunks = ClipTestChunks.parse(file); + + // The corner text is invisible on the page but inside the box, so it is + // retained: that is the approximation, stated. + assertEquals(Arrays.asList("Inside the triangle", "Corner outside the triangle"), + ClipTestChunks.retained(chunks)); + assertEquals(Collections.singletonList("Outside the bounding box"), ClipTestChunks.clipped(chunks)); + } + + /** Without W the path is only painted; it must not clip anything. */ + @Test + void aPaintedPathWithoutWDoesNotClip() throws Exception { + Path file = new ClipTestPdfBuilder() + .content("q 0 0 10 10 re f\n" + + ClipTestPdfBuilder.text(72, 700, "Not clipped by a filled box") + + "Q\n") + .write(tempDir.resolve("nowclip.pdf")); + + List chunks = ClipTestChunks.parse(file); + + assertTrue(ClipTestChunks.clipped(chunks).isEmpty(), "a plain fill must not clip"); + assertEquals(Collections.singletonList("Not clipped by a filled box"), ClipTestChunks.retained(chunks)); + } +}