diff --git a/cms-api/src/main/java/com/condation/cms/api/eventbus/events/CollectionChangedEvent.java b/cms-api/src/main/java/com/condation/cms/api/eventbus/events/CollectionChangedEvent.java new file mode 100644 index 000000000..d01ef63e2 --- /dev/null +++ b/cms-api/src/main/java/com/condation/cms/api/eventbus/events/CollectionChangedEvent.java @@ -0,0 +1,28 @@ +package com.condation.cms.api.eventbus.events; + +/*- + * #%L + * CMS Api + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * #L% + */ + +import com.condation.cms.api.eventbus.Event; +import java.nio.file.Path; + +/** Published after collection metadata is refreshed; a directory requests a subtree refresh. */ +public record CollectionChangedEvent(Path path) implements Event {} diff --git a/cms-api/src/main/java/com/condation/cms/api/eventbus/events/ContentTypesChangedEvent.java b/cms-api/src/main/java/com/condation/cms/api/eventbus/events/ContentTypesChangedEvent.java new file mode 100644 index 000000000..bbf7c5521 --- /dev/null +++ b/cms-api/src/main/java/com/condation/cms/api/eventbus/events/ContentTypesChangedEvent.java @@ -0,0 +1,27 @@ +package com.condation.cms.api.eventbus.events; + +/*- + * #%L + * CMS Api + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * #L% + */ + +import com.condation.cms.api.eventbus.Event; + +/** Requests rebuilding references after editor schemas have changed. */ +public record ContentTypesChangedEvent() implements Event {} diff --git a/cms-api/src/main/java/com/condation/cms/api/ui/elements/ContentTypeProvider.java b/cms-api/src/main/java/com/condation/cms/api/ui/elements/ContentTypeProvider.java new file mode 100644 index 000000000..c922ea98e --- /dev/null +++ b/cms-api/src/main/java/com/condation/cms/api/ui/elements/ContentTypeProvider.java @@ -0,0 +1,35 @@ +package com.condation.cms.api.ui.elements; + +/*- + * #%L + * CMS Api + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * #L% + */ + +import com.condation.cms.api.hooks.HookSystem; + +/** Shared hook contract for the manager and background content processing. */ +public final class ContentTypeProvider { + public static final String REGISTER_HOOK = "manager/contentTypes/register"; + + private ContentTypeProvider() {} + + public static ContentTypes load(HookSystem hooks) { + return hooks.doFilter(REGISTER_HOOK, new ContentTypes()); + } +} diff --git a/cms-api/src/main/java/com/condation/cms/api/usage/Usage.java b/cms-api/src/main/java/com/condation/cms/api/usage/Usage.java new file mode 100644 index 000000000..a53389b17 --- /dev/null +++ b/cms-api/src/main/java/com/condation/cms/api/usage/Usage.java @@ -0,0 +1,30 @@ +package com.condation.cms.api.usage; + +/*- + * #%L + * CMS Api + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * #L% + */ + +/** A direct editorial reference and its exact source location. */ +public record Usage(UsageResource source, UsageResource target, String location, + Origin origin, String originalReference, String sourceTitle, String sourceStatus, + TargetStatus targetStatus) { + public enum Origin { CONTENT_TYPE, MARKDOWN, HTML } + public enum TargetStatus { EXISTS, MISSING, UNRESOLVED } +} diff --git a/cms-api/src/main/java/com/condation/cms/api/usage/UsageIndex.java b/cms-api/src/main/java/com/condation/cms/api/usage/UsageIndex.java new file mode 100644 index 000000000..a66a7ef80 --- /dev/null +++ b/cms-api/src/main/java/com/condation/cms/api/usage/UsageIndex.java @@ -0,0 +1,32 @@ +package com.condation.cms.api.usage; + +/*- + * #%L + * CMS Api + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * #L% + */ + +import java.util.List; + +/** Site-scoped, rebuildable index of editorial references. Themes and dynamic queries are excluded. */ +public interface UsageIndex { + List incoming(UsageResource target); + List outgoing(UsageResource source); + List problems(); + void rebuild(); +} diff --git a/cms-api/src/main/java/com/condation/cms/api/usage/UsageProblem.java b/cms-api/src/main/java/com/condation/cms/api/usage/UsageProblem.java new file mode 100644 index 000000000..f633407c0 --- /dev/null +++ b/cms-api/src/main/java/com/condation/cms/api/usage/UsageProblem.java @@ -0,0 +1,25 @@ +package com.condation.cms.api.usage; + +/*- + * #%L + * CMS Api + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * #L% + */ + +/** An incomplete extraction must never be reported as proof that a resource is unused. */ +public record UsageProblem(String site, String path, String message) {} diff --git a/cms-api/src/main/java/com/condation/cms/api/usage/UsageResource.java b/cms-api/src/main/java/com/condation/cms/api/usage/UsageResource.java new file mode 100644 index 000000000..d441b598a --- /dev/null +++ b/cms-api/src/main/java/com/condation/cms/api/usage/UsageResource.java @@ -0,0 +1,45 @@ +package com.condation.cms.api.usage; + +/*- + * #%L + * CMS Api + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * #L% + */ + +import java.util.Objects; + +/** Identity within a site's content, assets or collections root; never a public context path. */ +public record UsageResource(String site, Kind kind, String path) { + public enum Kind { CONTENT, MEDIA, COLLECTION_ITEM, UNRESOLVED_URL } + + public UsageResource { + Objects.requireNonNull(site, "site"); + Objects.requireNonNull(kind, "kind"); + Objects.requireNonNull(path, "path"); + path = path.replace('\\', '/').replaceAll("^/+", ""); + if (site.isBlank() || path.contains("\u0000")) { + throw new IllegalArgumentException("invalid usage resource"); + } + if (kind != Kind.UNRESOLVED_URL) { + path = java.nio.file.Path.of(path).normalize().toString().replace('\\', '/'); + if (path.isBlank() || path.equals("..") || path.startsWith("../")) { + throw new IllegalArgumentException("invalid site-local resource path"); + } + } + } +} diff --git a/cms-content/pom.xml b/cms-content/pom.xml index fabde89e3..e8c1c4a25 100644 --- a/cms-content/pom.xml +++ b/cms-content/pom.xml @@ -25,6 +25,18 @@ + + com.google.code.gson + gson + + + org.apache.lucene + lucene-core + + + org.apache.lucene + lucene-analysis-common + com.condation.cms cms-api diff --git a/cms-content/src/main/java/com/condation/cms/content/markdown/rules/inline/ImageLinkInlineRule.java b/cms-content/src/main/java/com/condation/cms/content/markdown/rules/inline/ImageLinkInlineRule.java index 329dcc654..7e0114e71 100644 --- a/cms-content/src/main/java/com/condation/cms/content/markdown/rules/inline/ImageLinkInlineRule.java +++ b/cms-content/src/main/java/com/condation/cms/content/markdown/rules/inline/ImageLinkInlineRule.java @@ -36,6 +36,13 @@ */ public class ImageLinkInlineRule implements InlineElementRule { + private final boolean modifyUrls; + + public ImageLinkInlineRule() { this(true); } + + /** Allows static consumers to inspect the stored URL without request transformations. */ + public ImageLinkInlineRule(boolean modifyUrls) { this.modifyUrls = modifyUrls; } + static final Slugify SLUG = Slugify.builder().build(); static final String IMAGE_PATTERN = "!\\[(?[^\\[\\]]*)\\]\\((?[^\\s\\)]+)(?: \"(?[^\"]*)\")?\\)"; @@ -55,7 +62,7 @@ public InlineBlock next(InlineElementTokenizer tokenizer, String md) { - if (RequestContextScope.REQUEST_CONTEXT.isBound() + if (modifyUrls && RequestContextScope.REQUEST_CONTEXT.isBound() && isInternalUrl(href)) { var requestContext = RequestContextScope.REQUEST_CONTEXT.get(); diff --git a/cms-content/src/main/java/com/condation/cms/content/markdown/rules/inline/LinkInlineRule.java b/cms-content/src/main/java/com/condation/cms/content/markdown/rules/inline/LinkInlineRule.java index 36f564986..9fdce5bc6 100644 --- a/cms-content/src/main/java/com/condation/cms/content/markdown/rules/inline/LinkInlineRule.java +++ b/cms-content/src/main/java/com/condation/cms/content/markdown/rules/inline/LinkInlineRule.java @@ -37,6 +37,13 @@ */ public class LinkInlineRule implements InlineElementRule { + private final boolean modifyUrls; + + public LinkInlineRule() { this(true); } + + /** Allows static consumers to inspect the stored URL without request transformations. */ + public LinkInlineRule(boolean modifyUrls) { this.modifyUrls = modifyUrls; } + static final Slugify SLUG = Slugify.builder().build(); static final Pattern PATTERN = Pattern.compile("\\[(?<text>[^\\]]*)\\]\\((?<url>[^\\s)]+)(?: \"(?<title>[^\"]*)\")?\\)"); @@ -53,7 +60,7 @@ public InlineBlock next(InlineElementTokenizer tokenizer, String md) { var id = SLUG.slugify(text); - if (RequestContextScope.REQUEST_CONTEXT.isBound() + if (modifyUrls && RequestContextScope.REQUEST_CONTEXT.isBound() && isInternalUrl(href)) { var requestContext = RequestContextScope.REQUEST_CONTEXT.get(); diff --git a/cms-content/src/main/java/com/condation/cms/content/usage/EditorialUsageIndex.java b/cms-content/src/main/java/com/condation/cms/content/usage/EditorialUsageIndex.java new file mode 100644 index 000000000..390cb8c58 --- /dev/null +++ b/cms-content/src/main/java/com/condation/cms/content/usage/EditorialUsageIndex.java @@ -0,0 +1,426 @@ +package com.condation.cms.content.usage; + +/*- + * #%L + * CMS Content + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +import com.condation.cms.api.usage.*; +import com.condation.cms.api.ui.elements.ContentTypes; +import com.condation.cms.api.utils.PathUtil; +import com.condation.cms.content.CollectionRouteTemplate; +import com.condation.cms.core.content.io.ContentFileParser; +import com.google.gson.*; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.*; + +/** + * Persistent usage index for exactly one site. + */ +public final class EditorialUsageIndex implements UsageIndex, AutoCloseable { + + private final UsageSite site; + private LuceneUsageStore store; + private volatile boolean reconciliationPending = true; + private volatile UsageProblem operationalProblem; + + public EditorialUsageIndex(UsageSite site) { + this.site = Objects.requireNonNull(site); + try { + store = new LuceneUsageStore(site.root()); + store.visitSources(source -> { + var resource = source.document().resource(); + if (!resource.site().equals(site.id())) { + store.delete(resource); + } + }); + var siteProblems = store.siteProblems().stream().filter(problem -> problem.site().equals(site.id())).toList(); + store.commit(siteProblems); + + } catch (IOException ex) { + if (this.store != null) { + try { + store.close(); + } catch (IOException ex1) { + throw new UncheckedIOException("Cannot closing usage index for " + site.id(), ex); + } + } + throw new UncheckedIOException("Cannot open usage index for " + site.id(), ex); + } + } + + /** + * Startup/configuration reconciliation: unchanged files and schemas reuse + * stored extractions. + */ + public synchronized void synchronize() { + reconcile(false); + } + + @Override + public synchronized void rebuild() { + reconcile(true); + } + + /** + * Called after the primary metadata index has processed a file or directory + * change. + */ + public synchronized void refresh(Path changed) { + Path file = changed.toAbsolutePath().normalize(); + var root = sourceRoot(site, file); + if (root == null) { + return; + } + try { + if (!file.toString().endsWith(".md")) { + updateSite(false); + } else { + updateFile(root, file); + } + operationalProblem = null; + } catch (IOException ex) { + operationalProblem = problem(site, "", "Usage index update failed", ex); + } + } + + private void reconcile(boolean force) { + try { + updateSite(force); + reconciliationPending = false; + operationalProblem = null; + } catch (IOException ex) { + operationalProblem = problem(site, "", "Usage index reconciliation failed", ex); + } + } + + private void updateSite(boolean force) throws IOException { + var siteProblems = new ArrayList<UsageProblem>(); + var seen = new HashSet<UsageResource>(); + boolean scanCompleted = false; + try { + ContentTypes types = site.contentTypes().get(); + String schema = schema(types); + for (String folder : List.of("content", "collections")) { + Path root = site.root().resolve(folder); + if (!Files.isDirectory(root)) { + continue; + } + try (var files = Files.walk(root)) { + for (var file : files.filter(Files::isRegularFile).filter(PathUtil::isContentFile).sorted().toList()) { + var resource = resource(site, root, file); + if (!localSource(site, resource)) { + continue; + } + seen.add(resource); + var old = store.source(resource); + var extractionProblems = new ArrayList<UsageProblem>(); + try { + String stamp = fileStamp(file); + if (!force && old.isPresent() && stamp.equals(old.get().fileStamp()) + && schema.equals(old.get().schema())) { + updateRoute(old.get()); + } else { + store.update(readFile(types, schema, root, file, old, extractionProblems)); + } + } catch (Exception ex) { + extractionProblems.add(problem(site, resource.path(), "Extraction failed", ex)); + retainLastGood(old, extractionProblems, siteProblems); + } + } + } + } + scanCompleted = true; + } catch (Exception ex) { + siteProblems.add(problem(site, "", "Extraction failed", ex)); + } + if (scanCompleted) { + store.visitSources(source -> { + var resource = source.document().resource(); + if (resource.site().equals(site.id()) && !seen.contains(resource)) { + store.delete(resource); + } + }); + } + publishCommittedSources(siteProblems); + } + + private void updateFile(Path root, Path file) throws IOException { + var resource = resource(site, root, file); + var siteProblems = new ArrayList<>(store.siteProblems()); + siteProblems.removeIf(problem -> problem.site().equals(site.id()) && problem.path().equals(resource.path())); + var old = store.source(resource); + if (!Files.exists(file) || !localSource(site, resource)) { + store.delete(resource); + } else { + var extractionProblems = new ArrayList<UsageProblem>(); + try { + var types = site.contentTypes().get(); + store.update(readFile(types, schema(types), root, file, old, extractionProblems)); + } catch (Exception ex) { + extractionProblems.add(problem(site, resource.path(), "Extraction failed", ex)); + retainLastGood(old, extractionProblems, siteProblems); + } + } + publishCommittedSources(siteProblems); + } + + private PersistedUsageSource readFile(ContentTypes types, String schema, Path root, Path file, + Optional<PersistedUsageSource> old, List<UsageProblem> extractionProblems) throws IOException { + if (!file.toRealPath().startsWith(root.toRealPath())) { + throw new IOException("Source is outside its site root"); + } + var resource = resource(site, root, file); + String before = fileStamp(file); + var parser = new ContentFileParser(file.toString()); + var metadata = parser.getHeader(); + String publicPath = publicPath(site, resource, metadata); + var references = new UsageExtractor(types, site, extractionProblems).extract(resource, metadata, parser.getContent()); + if (!before.equals(fileStamp(file))) { + throw new IOException("Source changed during extraction; retry required"); + } + var document = new UsageDocument(resource, publicPath, metadata, references); + var previousUsages = old.map(PersistedUsageSource::usages).orElseGet(List::of); + var problems = List.copyOf(extractionProblems); + return new PersistedUsageSource(document, before, schema, previousUsages, problems, problems); + } + + private void updateRoute(PersistedUsageSource source) throws IOException { + var current = currentDocument(source.document()); + if (!current.equals(source.document())) { + store.update(new PersistedUsageSource(current, source.fileStamp(), source.schema(), source.usages(), + source.extractionProblems(), source.problems())); + } + } + + private void retainLastGood(Optional<PersistedUsageSource> old, List<UsageProblem> extractionProblems, + List<UsageProblem> siteProblems) throws IOException { + if (old.isEmpty()) { + siteProblems.addAll(extractionProblems); + return; + } + var source = old.get(); + var document = currentDocument(source.document()); + var problems = List.copyOf(extractionProblems); + store.update(new PersistedUsageSource(document, source.fileStamp(), source.schema(), source.usages(), + problems, problems)); + } + + private static boolean localSource(UsageSite site, UsageResource resource) { + if (resource.kind() != UsageResource.Kind.COLLECTION_ITEM) { + return true; + } + String[] parts = resource.path().split("/"); + return parts.length == 2 && site.collectionSite(parts[0]).equals(site.id()); + } + + private static UsageResource resource(UsageSite site, Path root, Path file) { + return new UsageResource(site.id(), root.endsWith("collections") + ? UsageResource.Kind.COLLECTION_ITEM : UsageResource.Kind.CONTENT, + root.relativize(file).toString().replace('\\', '/')); + } + + private static String fileStamp(Path file) throws IOException { + var attributes = Files.readAttributes(file, BasicFileAttributes.class); + return attributes.lastModifiedTime() + ":" + attributes.size() + ":" + attributes.fileKey(); + } + + private static String schema(ContentTypes types) { + try { + // Map iteration order differs between JVMs. Canonicalize object keys before hashing. + String serialized = canonical(new Gson().toJsonTree(types)).toString(); + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(("usage-extractor-1\n" + serialized).getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException ex) { + throw new IllegalStateException(ex); + } + } + + private static JsonElement canonical(JsonElement element) { + if (element.isJsonObject()) { + var result = new JsonObject(); + element.getAsJsonObject().keySet().stream().sorted().forEach(key + -> result.add(key, canonical(element.getAsJsonObject().get(key)))); + return result; + } + if (element.isJsonArray()) { + var result = new JsonArray(); + element.getAsJsonArray().forEach(value -> result.add(canonical(value))); + return result; + } + return element; + } + + private String publicPath(UsageSite site, UsageResource resource, Map<String, Object> metadata) { + if (resource.kind() == UsageResource.Kind.CONTENT) { + String path = resource.path(); + String name = Path.of(path).getFileName().toString(); + if (com.condation.cms.api.utils.SectionUtil.isSectionEntry(name)) { + path = path.substring(0, path.length() - name.length()) + name.substring(0, name.indexOf('.')) + ".md"; + } + return site.db().getContent().byPath(path).map(node -> node.url()).orElse(PathUtil.toURL(path)); + } + if (site.collections() == null) { + return null; + } + String[] parts = resource.path().split("/", 2); + return site.collections().collection(parts[0]).flatMap(definition -> definition.detailPage()).map(detail -> { + try { + return new CollectionRouteTemplate(detail).render(parts[1].substring(0, parts[1].length() - 3), metadata); + } catch (IllegalArgumentException ex) { + return null; + } + }).orElse(null); + } + + private void publishCommittedSources(List<UsageProblem> siteProblems) throws IOException { + // Publish source facts first so route lookup sees the new route catalog. Edge fields still + // contain their previous batch until the second commit replaces them atomically per source. + store.commit(siteProblems); + var resolver = resolver(); + store.visitSources(source -> { + var document = source.document(); + var usages = new ArrayList<Usage>(); + var problems = new ArrayList<>(source.extractionProblems()); + var current = currentDocument(document); + for (var ref : document.references()) { + try { + var target = resolver.resolve(current, ref); + if (target.isEmpty()) { + continue; + } + UsageResource resource = target.get(); + Usage.TargetStatus status = resolver.status(resource); + if (resource.kind() == UsageResource.Kind.UNRESOLVED_URL || status == Usage.TargetStatus.MISSING) { + var previous = source.usages().stream() + .filter(usage -> usage.location().equals(ref.location()) && usage.originalReference().equals(ref.value())) + .filter(usage -> usage.target().kind() != UsageResource.Kind.UNRESOLVED_URL).findFirst(); + if (previous.isPresent()) { + resource = previous.get().target(); + status = Usage.TargetStatus.MISSING; + } + } + usages.add(new Usage(document.resource(), resource, ref.location(), ref.origin(), ref.value(), + document.title(), document.status(), status)); + } catch (Exception ex) { + problems.add(new UsageProblem(site.id(), document.resource().path(), + "Cannot resolve " + ref.location() + ": " + ref.value() + " (" + ex.getMessage() + ")")); + } + } + var next = new PersistedUsageSource(current, source.fileStamp(), source.schema(), List.copyOf(usages), + source.extractionProblems(), List.copyOf(problems)); + if (!next.equals(source)) { + store.update(next); + } + }); + store.commit(siteProblems); + } + + private UsageDocument currentDocument(UsageDocument document) { + return new UsageDocument(document.resource(), publicPath(site, document.resource(), document.metadata()), + document.metadata(), document.references()); + } + + private UsageReferenceResolver resolver() { + return new UsageReferenceResolver(site, new UsageReferenceResolver.PublicTargetLookup() { + @Override + public Optional<UsageResource> aliasTarget(String path) throws IOException { + return store.aliasTarget(path); + } + + @Override + public Optional<UsageResource> collectionTarget(String path) throws IOException { + return store.collectionTarget(path); + } + }); + } + + private static Path sourceRoot(UsageSite site, Path file) { + for (String folder : List.of("content", "collections")) { + Path root = site.root().resolve(folder); + if (file.startsWith(root)) { + return root; + } + } + return null; + } + + private static UsageProblem problem(UsageSite site, String path, String message, Exception ex) { + return new UsageProblem(site.id(), path, message + ": " + ex.getMessage()); + } + + @Override + public List<Usage> incoming(UsageResource target) { + try { + return withLiveMediaStatus(store.incoming(target)); + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + + @Override + public List<Usage> outgoing(UsageResource source) { + if (!source.site().equals(site.id())) { + return List.of(); + } + try { + return withLiveMediaStatus(store.outgoing(source)); + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + + private List<Usage> withLiveMediaStatus(List<Usage> usages) { + var resolver = resolver(); + return usages.stream().map(usage -> usage.target().kind() != UsageResource.Kind.MEDIA ? usage + : new Usage(usage.source(), usage.target(), usage.location(), usage.origin(), usage.originalReference(), + usage.sourceTitle(), usage.sourceStatus(), resolver.status(usage.target()))).toList(); + } + + @Override + public synchronized List<UsageProblem> problems() { + try { + var problems = new ArrayList<>(store.problems()); + if (reconciliationPending) { + problems.add(new UsageProblem(site.id(), "", "Startup reconciliation pending")); + } + if (operationalProblem != null) { + problems.add(operationalProblem); + } + return problems.stream().distinct().toList(); + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + + @Override + public synchronized void close() { + try { + store.close(); + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } +} diff --git a/cms-content/src/main/java/com/condation/cms/content/usage/LuceneUsageStore.java b/cms-content/src/main/java/com/condation/cms/content/usage/LuceneUsageStore.java new file mode 100644 index 000000000..43a4f2bb2 --- /dev/null +++ b/cms-content/src/main/java/com/condation/cms/content/usage/LuceneUsageStore.java @@ -0,0 +1,251 @@ +package com.condation.cms.content.usage; + +/*- + * #%L + * CMS Content + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.condation.cms.api.usage.*; +import com.condation.cms.api.utils.PathUtil; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import java.io.IOException; +import java.nio.file.Path; +import java.util.*; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.*; +import org.apache.lucene.store.FSDirectory; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; + +/** Persistent inverted index for one source site. All reference directions share a commit. */ +final class LuceneUsageStore implements AutoCloseable { + private static final String VERSION = "2"; + private static final String SOURCE = "source"; + private static final String TARGET = "target"; + private static final String ALIAS = "alias"; + private static final String COLLECTION_ROUTE = "collection-route"; + private static final Gson JSON = new Gson(); + private final FSDirectory directory; + private final IndexWriter writer; + private final SearcherManager searchers; + private boolean dirty; + + LuceneUsageStore(Path siteRoot) throws IOException { + directory = FSDirectory.open(siteRoot.resolve("data/usage/index")); + IndexWriter openedWriter = null; + SearcherManager openedSearchers = null; + try { + boolean recreate = false; + if (DirectoryReader.indexExists(directory)) { + try (var existing = DirectoryReader.open(directory)) { + recreate = !VERSION.equals(existing.getIndexCommit().getUserData().get("usage-format")); + } + } + openedWriter = new IndexWriter(directory, new IndexWriterConfig(new KeywordAnalyzer()) + .setOpenMode(recreate ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.CREATE_OR_APPEND)); + writer = openedWriter; + openedSearchers = new SearcherManager(writer, true, true, new SearcherFactory()); + searchers = openedSearchers; + var settings = new HashMap<String, String>(); + writer.getLiveCommitData().forEach(entry -> settings.put(entry.getKey(), entry.getValue())); + dirty = !VERSION.equals(settings.get("usage-format")); + } catch (Exception ex) { + if (openedSearchers != null) openedSearchers.close(); + if (openedWriter != null) openedWriter.close(); + directory.close(); + throw ex; + } + } + + synchronized List<PersistedUsageSource> sources() throws IOException { + var result = new ArrayList<PersistedUsageSource>(); + visitSources(result::add); + return List.copyOf(result); + } + + synchronized Optional<PersistedUsageSource> source(UsageResource resource) throws IOException { + var documents = search(new TermQuery(new Term(SOURCE, key(resource))), 1); + return documents.isEmpty() ? Optional.empty() : Optional.of(source(documents.getFirst())); + } + + synchronized void visitSources(SourceVisitor visitor) throws IOException { + var searcher = searchers.acquire(); + try { + var stored = searcher.storedFields(); + ScoreDoc after = null; + while (true) { + var page = searcher.searchAfter(after, MatchAllDocsQuery.INSTANCE, 256); + for (var hit : page.scoreDocs) visitor.accept(source(stored.document(hit.doc))); + if (page.scoreDocs.length < 256) return; + after = page.scoreDocs[page.scoreDocs.length - 1]; + } + } finally { + searchers.release(searcher); + } + } + + synchronized void update(PersistedUsageSource source) throws IOException { + var value = source.document(); + var document = new Document(); + document.add(new StringField(SOURCE, key(value.resource()), Field.Store.NO)); + source.usages().stream().map(Usage::target).distinct().forEach(target -> + document.add(new StringField(TARGET, key(target), Field.Store.NO))); + if (value.resource().kind() == UsageResource.Kind.CONTENT + && value.metadata().get("aliases") instanceof Collection<?> aliases) { + aliases.stream().filter(String.class::isInstance).map(String.class::cast) + .map(PathUtil::normalizeURL).distinct().forEach(alias -> + document.add(new StringField(ALIAS, alias, Field.Store.NO))); + } + if (value.resource().kind() == UsageResource.Kind.COLLECTION_ITEM && value.publicPath() != null) { + document.add(new StringField(COLLECTION_ROUTE, PathUtil.normalizeURL(value.publicPath()), Field.Store.NO)); + } + document.add(new StoredField("resource", JSON.toJson(value.resource()))); + if (value.publicPath() != null) document.add(new StoredField("public-path", value.publicPath())); + // YAML preserves date/number metadata types required by collection route templates. + document.add(new StoredField("metadata", new Yaml().dump(value.metadata()))); + document.add(new StoredField("references", JSON.toJson(value.references()))); + document.add(new StoredField("usages", JSON.toJson(source.usages()))); + document.add(new StoredField("problems", JSON.toJson(source.problems()))); + document.add(new StoredField("extraction-problems", JSON.toJson(source.extractionProblems()))); + document.add(new StoredField("file-stamp", source.fileStamp())); + document.add(new StoredField("schema", source.schema())); + writer.updateDocument(new Term(SOURCE, key(value.resource())), document); + dirty = true; + } + + synchronized void delete(UsageResource source) throws IOException { + writer.deleteDocuments(new Term(SOURCE, key(source))); + dirty = true; + } + + List<Usage> incoming(UsageResource target) throws IOException { + return search(new TermQuery(new Term(TARGET, key(target)))).stream().flatMap(doc -> usages(doc).stream()) + .filter(usage -> usage.target().equals(target)).toList(); + } + + List<Usage> outgoing(UsageResource source) throws IOException { + return search(new TermQuery(new Term(SOURCE, key(source)))).stream().flatMap(doc -> usages(doc).stream()).toList(); + } + + synchronized Optional<UsageResource> aliasTarget(String path) throws IOException { + return uniqueTarget(ALIAS, path); + } + + synchronized Optional<UsageResource> collectionTarget(String path) throws IOException { + return uniqueTarget(COLLECTION_ROUTE, path); + } + + synchronized List<UsageProblem> siteProblems() { + var settings = new HashMap<String, String>(); + writer.getLiveCommitData().forEach(entry -> settings.put(entry.getKey(), entry.getValue())); + String problems = settings.get("site-problems"); + return problems == null ? List.of() + : JSON.fromJson(problems, new TypeToken<List<UsageProblem>>() {}.getType()); + } + + synchronized List<UsageProblem> problems() throws IOException { + var result = new ArrayList<>(siteProblems()); + visitSources(source -> result.addAll(source.problems())); + return result.stream().distinct().toList(); + } + + synchronized void commit(List<UsageProblem> problems) throws IOException { + var nextProblems = List.copyOf(problems); + if (!siteProblems().equals(nextProblems)) dirty = true; + if (!dirty) return; + writer.setLiveCommitData(Map.of("usage-format", VERSION, "site-problems", JSON.toJson(nextProblems)).entrySet()); + writer.commit(); + searchers.maybeRefreshBlocking(); + dirty = false; + } + + private List<Document> search(Query query) throws IOException { + return search(query, Integer.MAX_VALUE); + } + + private List<Document> search(Query query, int limit) throws IOException { + var searcher = searchers.acquire(); + try { + var stored = searcher.storedFields(); + var result = new ArrayList<Document>(); + ScoreDoc after = null; + while (result.size() < limit) { + int pageSize = Math.min(256, limit - result.size()); + var page = searcher.searchAfter(after, query, pageSize); + for (var hit : page.scoreDocs) result.add(stored.document(hit.doc)); + if (page.scoreDocs.length < pageSize) break; + after = page.scoreDocs[page.scoreDocs.length - 1]; + } + return result; + } finally { + searchers.release(searcher); + } + } + + private Optional<UsageResource> uniqueTarget(String field, String path) throws IOException { + var documents = search(new TermQuery(new Term(field, PathUtil.normalizeURL(path))), 2); + return documents.size() == 1 + ? Optional.of(JSON.fromJson(documents.getFirst().get("resource"), UsageResource.class)) + : Optional.empty(); + } + + private static PersistedUsageSource source(Document document) { + UsageResource resource = JSON.fromJson(document.get("resource"), UsageResource.class); + Map<String, Object> metadata = new Yaml(new SafeConstructor(new LoaderOptions())).load(document.get("metadata")); + List<UsageReference> references = JSON.fromJson(document.get("references"), new TypeToken<List<UsageReference>>() {}.getType()); + List<UsageProblem> problems = JSON.fromJson(document.get("problems"), new TypeToken<List<UsageProblem>>() {}.getType()); + List<UsageProblem> extractionProblems = JSON.fromJson(document.get("extraction-problems"), new TypeToken<List<UsageProblem>>() {}.getType()); + return new PersistedUsageSource(new UsageDocument(resource, document.get("public-path"), metadata, references), + document.get("file-stamp"), document.get("schema"), usages(document), extractionProblems, problems); + } + + private static List<Usage> usages(Document document) { + return JSON.fromJson(document.get("usages"), new TypeToken<List<Usage>>() {}.getType()); + } + + private static String key(UsageResource resource) { return JSON.toJson(resource); } + + @FunctionalInterface + interface SourceVisitor { + void accept(PersistedUsageSource source) throws IOException; + } + + @Override + public synchronized void close() throws IOException { + try { + commit(siteProblems()); + } finally { + try { searchers.close(); } + finally { + try { writer.close(); } + finally { directory.close(); } + } + } + } +} diff --git a/cms-content/src/main/java/com/condation/cms/content/usage/PersistedUsageSource.java b/cms-content/src/main/java/com/condation/cms/content/usage/PersistedUsageSource.java new file mode 100644 index 000000000..9be86d8b9 --- /dev/null +++ b/cms-content/src/main/java/com/condation/cms/content/usage/PersistedUsageSource.java @@ -0,0 +1,30 @@ +package com.condation.cms.content.usage; + +/*- + * #%L + * CMS Content + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.condation.cms.api.usage.Usage; +import com.condation.cms.api.usage.UsageProblem; +import java.util.List; + +/** One atomic Lucene document contains the extraction, file stamp and resolved outgoing edges. */ +record PersistedUsageSource(UsageDocument document, String fileStamp, String schema, + List<Usage> usages, List<UsageProblem> extractionProblems, List<UsageProblem> problems) {} diff --git a/cms-content/src/main/java/com/condation/cms/content/usage/UsageDocument.java b/cms-content/src/main/java/com/condation/cms/content/usage/UsageDocument.java new file mode 100644 index 000000000..b0c8ddcfc --- /dev/null +++ b/cms-content/src/main/java/com/condation/cms/content/usage/UsageDocument.java @@ -0,0 +1,32 @@ +package com.condation.cms.content.usage; + +/*- + * #%L + * CMS Content + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.condation.cms.api.usage.UsageResource; +import java.util.List; +import java.util.Map; + +record UsageDocument(UsageResource resource, String publicPath, Map<String, Object> metadata, + List<UsageReference> references) { + String title() { return String.valueOf(metadata.getOrDefault("title", resource.path())); } + String status() { return String.valueOf(metadata.getOrDefault("status", "unknown")); } +} diff --git a/cms-content/src/main/java/com/condation/cms/content/usage/UsageExtractor.java b/cms-content/src/main/java/com/condation/cms/content/usage/UsageExtractor.java new file mode 100644 index 000000000..141cd6436 --- /dev/null +++ b/cms-content/src/main/java/com/condation/cms/content/usage/UsageExtractor.java @@ -0,0 +1,227 @@ +package com.condation.cms.content.usage; + +/*- + * #%L + * CMS Content + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.condation.cms.api.ui.elements.*; +import com.condation.cms.api.ui.elements.fields.*; +import com.condation.cms.api.usage.*; +import com.condation.cms.api.utils.MapUtil; +import com.condation.cms.api.utils.SectionUtil; +import com.condation.cms.content.markdown.InlineElementTokenizer; +import com.condation.cms.content.markdown.Options; +import com.condation.cms.content.markdown.rules.inline.*; +import java.io.IOException; +import java.util.*; +import java.util.regex.Pattern; +import org.jsoup.Jsoup; +import org.jsoup.nodes.TextNode; + +/** Static extraction only: no template evaluation, shortcode execution or media rendering. */ +final class UsageExtractor { + private final ContentTypes types; + private final UsageSite site; + private final List<UsageProblem> problems; + private final InlineElementTokenizer tokenizer; + private static final Pattern INLINE_CODE = Pattern.compile("(`+)([\\s\\S]*?)(?<!`)\\1(?!`)"); + + UsageExtractor(ContentTypes types, UsageSite site, List<UsageProblem> problems) { + this.types = types; + this.site = site; + this.problems = problems; + var options = new Options(); + options.addInlineRule(new ImageLinkInlineRule(false)); + options.addInlineRule(new ImageInlineRule()); + options.addInlineRule(new LinkInlineRule(false)); + tokenizer = new InlineElementTokenizer(options); + } + + List<UsageReference> extract(UsageResource source, Map<String, Object> metadata, String body) throws IOException { + var result = new ArrayList<UsageReference>(); + var forms = forms(source, metadata); + Set<String> listForms = new HashSet<>(); + forms.values().forEach(form -> fields(form).stream().filter(field -> field instanceof ListField) + .forEach(field -> listForms.add(field.getName()))); + for (var entry : forms.entrySet()) { + if (!listForms.contains(entry.getKey())) { + extractForm(source, entry.getValue(), forms, metadata, "metadata", result, 0); + } + } + extractText(body, "body", result); + return List.copyOf(new LinkedHashSet<>(result)); + } + + private Map<String, FormDefinition> forms(UsageResource source, Map<String, Object> metadata) { + if (source.kind() == UsageResource.Kind.COLLECTION_ITEM) { + String collection = source.path().split("/", 2)[0]; + var type = types.getCollection(collection); + if (type.isPresent()) return type.get().forms(); + } else { + Object template = metadata.get("template"); + String name = source.path().substring(source.path().lastIndexOf('/') + 1); + List<Map<String, FormDefinition>> matches; + if (SectionUtil.isSectionEntry(name)) { + matches = types.getSectionEntryTemplates(SectionUtil.getSectionName(name)).stream() + .filter(type -> type.template().equals(template)).map(SectionEntryTemplate::forms).toList(); + } else { + matches = types.getPageTemplates().stream().filter(type -> type.template().equals(template)) + .map(PageTemplate::forms).toList(); + } + if (matches.size() == 1) return matches.getFirst(); + } + problems.add(new UsageProblem(site.id(), source.path(), "Missing or ambiguous content type; only body references indexed")); + return Map.of(); + } + + private void extractForm(UsageResource source, FormDefinition form, Map<String, FormDefinition> forms, + Map<String, Object> values, String location, List<UsageReference> result, int depth) throws IOException { + if (depth > 64) throw new IOException("Maximum form nesting depth exceeded"); + for (var field : fields(form)) { + Object value = MapUtil.getValue(values, field.getName()); + if (value == null) continue; + String fieldPath = location + "." + field.getName(); + if (field instanceof ListField && value instanceof List<?> items) { + var itemForm = forms.get(field.getName()); + if (itemForm == null || fields(itemForm).isEmpty()) { + itemForm = types.getListItemTypes().stream().filter(type -> type.name().equals(field.getName())) + .map(ListItemType::form).findFirst().orElse(null); + } + if (itemForm == null) { + problems.add(new UsageProblem(site.id(), source.path(), "Missing list form: " + fieldPath)); + continue; + } + for (int i = 0; i < items.size(); i++) { + if (items.get(i) instanceof Map<?, ?> item) { + @SuppressWarnings("unchecked") var map = (Map<String, Object>) item; + extractForm(source, itemForm, forms, map, fieldPath + "[" + i + "]", result, depth + 1); + } + } + } else if (value instanceof String text && !text.isBlank()) { + if (field instanceof MediaField) { + result.add(new UsageReference(text, fieldPath, Usage.Origin.CONTENT_TYPE, + UsageResource.Kind.MEDIA, site.id(), null, false)); + } else if (field instanceof ReferenceField reference) { + result.add(new UsageReference(text, fieldPath, Usage.Origin.CONTENT_TYPE, + UsageResource.Kind.CONTENT, reference.getOptions().siteid(), null, false)); + } else if (field instanceof CollectionField collection) { + if (collection.getOptions().collection() == null || collection.getOptions().collection().isBlank()) { + problems.add(new UsageProblem(site.id(), source.path(), "Missing collection for " + fieldPath)); + } else { + result.add(new UsageReference(text, fieldPath, Usage.Origin.CONTENT_TYPE, + UsageResource.Kind.COLLECTION_ITEM, site.id(), collection.getOptions().collection(), false)); + } + } else if (field instanceof MarkdownField || field instanceof EasyMdeField) { + extractText(text, fieldPath, result); + } + } + } + } + + private static List<FormField> fields(FormDefinition form) { + var result = new ArrayList<>(form.fields()); + form.tabs().forEach(tab -> result.addAll(tab.fields())); + return result; + } + + private void extractText(String text, String location, List<UsageReference> result) throws IOException { + var html = Jsoup.parseBodyFragment(maskCode(text)); + html.select("script,style,pre,code").remove(); + int elementIndex = 0; + for (var element : html.getAllElements()) { + String at = location + ".html[" + elementIndex++ + "]"; + for (String attribute : List.of("href", "src", "poster")) { + if (element.hasAttr(attribute)) addPublic(result, element.attr(attribute), at + "." + attribute, Usage.Origin.HTML); + } + if (element.hasAttr("srcset")) { + int candidate = 0; + // URL tokens may contain commas (notably data URLs); descriptors end at a comma. + String input = element.attr("srcset"); + int offset = 0; + while (offset < input.length()) { + while (offset < input.length() && (Character.isWhitespace(input.charAt(offset)) || input.charAt(offset) == ',')) offset++; + int start = offset; + while (offset < input.length() && !Character.isWhitespace(input.charAt(offset))) offset++; + String url = input.substring(start, offset); + boolean ended = url.endsWith(","); + url = url.replaceAll(",+$", ""); + addPublic(result, url, at + ".srcset[" + candidate++ + "]", Usage.Origin.HTML); + if (!ended) { + while (offset < input.length() && input.charAt(offset) != ',') offset++; + } + } + } + } + int textIndex = 0; + for (var node : html.nodeStream().filter(node -> node instanceof TextNode).toList()) { + String content = ((TextNode) node).getWholeText(); + for (var located : tokenizer.tokenize(content)) { + String at = location + ".text[" + textIndex + "]@" + located.absoluteStart(); + switch (located.block()) { + case LinkInlineRule.LinkBlock link -> addMarkdownLink(result, link.href(), at); + case ImageInlineRule.ImageInlineBlock image -> addPublic(result, image.src(), at, Usage.Origin.MARKDOWN); + case ImageLinkInlineRule.ImageLinkBlock link -> { + addMarkdownLink(result, link.href(), at + ".link"); + addPublic(result, link.imageSrc(), at + ".image", Usage.Origin.MARKDOWN); + } + default -> { } + } + } + textIndex++; + } + } + + private void addMarkdownLink(List<UsageReference> result, String value, String location) { + // LinkInlineRule uses HTTPUtil.prependContext. Preserve the original separately in the index. + result.add(new UsageReference(value, location, Usage.Origin.MARKDOWN, + UsageResource.Kind.CONTENT, null, null, true)); + } + + private static void addPublic(List<UsageReference> result, String value, String location, Usage.Origin origin) { + if (!value.isBlank()) result.add(new UsageReference(value, location, origin, null, null, null, true)); + } + + private static String maskCode(String text) { + var result = new StringBuilder(); + char fence = 0; + int fenceLength = 0; + for (String line : text.replace("\r\n", "\n").split("\n", -1)) { + String trimmed = line.stripLeading(); + int run = 0; + if (!trimmed.isEmpty() && (trimmed.charAt(0) == '`' || trimmed.charAt(0) == '~')) { + while (run < trimmed.length() && trimmed.charAt(run) == trimmed.charAt(0)) run++; + } + boolean delimiter = run >= 3; + if (fence != 0) { + if (delimiter && trimmed.charAt(0) == fence && run >= fenceLength && trimmed.substring(run).isBlank()) fence = 0; + result.append('\n'); + } else if (delimiter) { + fence = trimmed.charAt(0); + fenceLength = run; + result.append('\n'); + } else if (line.startsWith(" ") || line.startsWith("\t")) { + result.append('\n'); + } else { + result.append(line).append('\n'); + } + } + return INLINE_CODE.matcher(result).replaceAll(""); + } +} diff --git a/cms-content/src/main/java/com/condation/cms/content/usage/UsageReference.java b/cms-content/src/main/java/com/condation/cms/content/usage/UsageReference.java new file mode 100644 index 000000000..5bf431f80 --- /dev/null +++ b/cms-content/src/main/java/com/condation/cms/content/usage/UsageReference.java @@ -0,0 +1,29 @@ +package com.condation.cms.content.usage; + +/*- + * #%L + * CMS Content + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.condation.cms.api.usage.Usage; +import com.condation.cms.api.usage.UsageResource; + +/** Unresolved extraction, retained so routing changes do not require reparsing every body. */ +record UsageReference(String value, String location, Usage.Origin origin, + UsageResource.Kind kind, String targetSite, String collection, boolean publicUrl) {} diff --git a/cms-content/src/main/java/com/condation/cms/content/usage/UsageReferenceResolver.java b/cms-content/src/main/java/com/condation/cms/content/usage/UsageReferenceResolver.java new file mode 100644 index 000000000..a997e618d --- /dev/null +++ b/cms-content/src/main/java/com/condation/cms/content/usage/UsageReferenceResolver.java @@ -0,0 +1,161 @@ +package com.condation.cms.content.usage; + +/*- + * #%L + * CMS Content + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.condation.cms.api.usage.Usage; +import com.condation.cms.api.usage.UsageResource; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; + +/** Resolves public URLs using host/context routing, and typed fields using their site-local contract. */ +final class UsageReferenceResolver { + private final UsageSite site; + private final PublicTargetLookup targets; + + UsageReferenceResolver(UsageSite site, PublicTargetLookup targets) { + this.site = site; + this.targets = targets; + } + + Optional<UsageResource> resolve(UsageDocument source, UsageReference ref) throws IOException { + String value = ref.value().trim(); + if (value.isEmpty() || value.startsWith("#")) return Optional.empty(); + URI uri = URI.create(value.replace(" ", "%20")); + if (uri.getScheme() != null && !isHttp(uri)) return Optional.empty(); + if (uri.getRawAuthority() != null && !isHttp(uri) && !ref.publicUrl()) return Optional.empty(); + + if (ref.origin() == Usage.Origin.MARKDOWN && ref.kind() == UsageResource.Kind.CONTENT + && !uri.isAbsolute() && uri.getRawAuthority() == null) { + uri = URI.create(com.condation.cms.api.utils.HTTPUtil.prependContext(value, site.properties()) + .replace(" ", "%20")); + } + + if (ref.publicUrl() || uri.isAbsolute() || uri.getRawAuthority() != null) { + if (!uri.isAbsolute() && uri.getRawAuthority() == null && source.publicPath() == null + && !value.startsWith("/")) { + return Optional.of(new UsageResource(site.id(), UsageResource.Kind.UNRESOLVED_URL, value)); + } + var base = URI.create(site.properties().baseUrl()); + String publicPath = context(site) + (source.publicPath() == null ? "/" : source.publicPath()); + var sourceUrl = base.resolve(publicPath); + var url = sourceUrl.resolve(uri).normalize(); + if (!matchesHost(site, url) || !matchesContext(url.getPath(), context(site))) { + if (uri.isAbsolute() || uri.getRawAuthority() != null) return Optional.empty(); + return Optional.of(new UsageResource(site.id(), UsageResource.Kind.UNRESOLVED_URL, url.getPath())); + } + String path = url.getPath().substring(context(site).length()); + return Optional.of(publicTarget(path.isEmpty() ? "/" : path)); + } + + String siteId = ref.targetSite() == null ? site.id() : ref.targetSite(); + String path = normalize(uri.getPath()); + if (ref.kind() == UsageResource.Kind.COLLECTION_ITEM) { + siteId = site.collectionSite(ref.collection()); + // CollectionField stores the filename stem, which may itself end in ".md". + com.condation.cms.api.db.collection.CollectionItemId.requireValid(path); + String item = path + ".md"; + return Optional.of(new UsageResource(siteId, ref.kind(), ref.collection() + "/" + item)); + } + if (ref.kind() == UsageResource.Kind.CONTENT && siteId.equals(site.id())) { + var node = site.db().getContent().byPath(path); + if (node.isEmpty()) node = site.db().getContent().byUrl("/" + path); + if (node.isPresent()) path = node.get().path(); + } + return Optional.of(new UsageResource(siteId, ref.kind(), path)); + } + + private UsageResource publicTarget(String path) throws IOException { + return findPublicTarget(path); + } + + private UsageResource findPublicTarget(String path) throws IOException { + String normalized = normalize(path); + for (var prefix : new String[]{"media/", "assets/"}) { + if (normalized.startsWith(prefix)) { + return new UsageResource(site.id(), UsageResource.Kind.MEDIA, normalized.substring(prefix.length())); + } + } + var node = site.db().getContent().byUrl(path); + if (node.isPresent()) return new UsageResource(site.id(), UsageResource.Kind.CONTENT, node.get().path()); + String url = com.condation.cms.api.utils.PathUtil.normalizeURL(path); + var alias = targets.aliasTarget(url); + if (alias.isPresent()) return alias.get(); + var collection = targets.collectionTarget(url); + if (collection.isPresent()) return collection.get(); + return new UsageResource(site.id(), UsageResource.Kind.UNRESOLVED_URL, path); + } + + Usage.TargetStatus status(UsageResource resource) { + if (!site.id().equals(resource.site()) || resource.kind() == UsageResource.Kind.UNRESOLVED_URL) { + return Usage.TargetStatus.UNRESOLVED; + } + String folder = switch (resource.kind()) { + case CONTENT -> "content"; + case MEDIA -> "assets"; + case COLLECTION_ITEM -> "collections"; + case UNRESOLVED_URL -> throw new IllegalStateException(); + }; + return Files.isRegularFile(site.root().resolve(folder).resolve(resource.path())) + ? Usage.TargetStatus.EXISTS : Usage.TargetStatus.MISSING; + } + + static String context(UsageSite site) { + String value = site.properties().contextPath(); + return value == null || value.equals("/") ? "" : "/" + value.replaceAll("^/+|/+$", ""); + } + + private static boolean matchesContext(String path, String context) { + return context.isEmpty() || path.equals(context) || path.startsWith(context + "/"); + } + + private static boolean matchesHost(UsageSite site, URI url) { + var base = URI.create(site.properties().baseUrl()); + if (url.getHost() == null) return false; + boolean hostMatches = url.getHost().equalsIgnoreCase(base.getHost()) + || (site.properties().hostnames() != null && site.properties().hostnames().stream() + .anyMatch(host -> host.equalsIgnoreCase(url.getHost()))); + return hostMatches && port(base) == port(url); + } + + private static int port(URI uri) { + return uri.getPort() >= 0 ? uri.getPort() : "https".equalsIgnoreCase(uri.getScheme()) ? 443 : 80; + } + + private static boolean isHttp(URI uri) { + return "http".equalsIgnoreCase(uri.getScheme()) || "https".equalsIgnoreCase(uri.getScheme()); + } + + private static String normalize(String path) { + String result = Path.of(path == null || path.isBlank() ? "." : path.replaceAll("^/+", "")) + .normalize().toString().replace('\\', '/'); + if (result.equals("..") || result.startsWith("../")) throw new IllegalArgumentException("path escapes site root"); + return result.equals(".") ? "" : result; + } + + interface PublicTargetLookup { + Optional<UsageResource> aliasTarget(String path) throws IOException; + Optional<UsageResource> collectionTarget(String path) throws IOException; + } +} diff --git a/cms-content/src/main/java/com/condation/cms/content/usage/UsageSite.java b/cms-content/src/main/java/com/condation/cms/content/usage/UsageSite.java new file mode 100644 index 000000000..3c2b43c3f --- /dev/null +++ b/cms-content/src/main/java/com/condation/cms/content/usage/UsageSite.java @@ -0,0 +1,53 @@ +package com.condation.cms.content.usage; + +/*- + * #%L + * CMS Content + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.condation.cms.api.SiteProperties; +import com.condation.cms.api.configuration.Configuration; +import com.condation.cms.api.configuration.configs.CollectionConfiguration; +import com.condation.cms.api.configuration.configs.SiteConfiguration; +import com.condation.cms.api.db.DB; +import com.condation.cms.api.ui.elements.ContentTypes; +import java.nio.file.Path; +import java.util.function.Supplier; + +/** Live site configuration and a request-independent loader for editor schemas. */ +public record UsageSite(String id, Path root, DB db, Configuration configuration, + Supplier<ContentTypes> contentTypes) { + public UsageSite { + root = root.toAbsolutePath().normalize(); + } + + public SiteProperties properties() { + return configuration.get(SiteConfiguration.class).siteProperties(); + } + + public CollectionConfiguration collections() { + return configuration.get(CollectionConfiguration.class); + } + + public String collectionSite(String collection) { + var config = collections(); + return config == null ? id : config.collection(collection) + .flatMap(definition -> definition.sourceSite()).orElse(id); + } +} diff --git a/cms-content/src/test/java/com/condation/cms/content/usage/EditorialUsageIndexTest.java b/cms-content/src/test/java/com/condation/cms/content/usage/EditorialUsageIndexTest.java new file mode 100644 index 000000000..7df1483d8 --- /dev/null +++ b/cms-content/src/test/java/com/condation/cms/content/usage/EditorialUsageIndexTest.java @@ -0,0 +1,439 @@ +package com.condation.cms.content.usage; + +/*- + * #%L + * CMS Content + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.condation.cms.api.SiteProperties; +import com.condation.cms.api.configuration.Configuration; +import com.condation.cms.api.configuration.configs.*; +import com.condation.cms.api.eventbus.events.*; +import com.condation.cms.api.ui.elements.*; +import com.condation.cms.api.ui.elements.fields.*; +import com.condation.cms.api.usage.*; +import com.condation.cms.core.content.io.ContentFileParser; +import com.condation.cms.core.eventbus.DefaultEventBus; +import com.condation.cms.core.serivce.ServiceRegistry; +import com.condation.cms.core.serivce.impl.SiteDBService; +import com.condation.cms.filesystem.FileDB; +import java.nio.file.*; +import java.util.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.io.TempDir; +import static org.assertj.core.api.Assertions.*; +import static org.mockito.Mockito.*; +import static org.awaitility.Awaitility.await; + +class EditorialUsageIndexTest { + @TempDir Path temp; + EditorialUsageIndex index; + final List<EditorialUsageIndex> indexes = new ArrayList<>(); + final List<FileDB> databases = new ArrayList<>(); + final Map<String, DefaultEventBus> buses = new HashMap<>(); + + @AfterEach void close() throws Exception { + for (var usageIndex : indexes) usageIndex.close(); + for (var db : databases) db.close(); + ServiceRegistry.getInstance().clear(); + } + + @Test void extractsTypedNestedAndBodyReferencesWithoutRenderingTemplates() throws Exception { + var types = pageTypes(new MediaField("hero", "Hero"), new ReferenceField("related", "Related"), + new CollectionField("author", "Author", "authors"), new ListField("teasers", "Teasers"), + new MarkdownField("description", "Description")); + types.registerListItemType(new ListItemType("teasers", new FormDefinition(List.of(new MediaField("image", "Image"))))); + types.registerCollection(new CollectionType("authors", Map.of("main", new FormDefinition(List.of(new MediaField("portrait", "Portrait")))))); + write("en/content/index.md", "template: page.html\nstatus: draft\nhero: images/hero.svg\nrelated: about.md\nauthor: jane\nteasers:\n - image: images/teaser.svg\ndescription: '[About](/about)'", + "![Hero](/media/images/hero.svg?format=small)\n<a href='/about#team'>Team</a>\n<img srcset='/assets/images/one.svg 1x, /assets/images/two.svg 2x'>\n" + + "```html\n<img src='/media/ignored.svg'>\n```\n`[ignored](/ignored)`\n[external](https://elsewhere.test/a)\n[mail](mailto:a@example.org)"); + write("en/content/about.md", "template: page.html", "About"); + write("en/collections/authors/jane.md", "portrait: images/jane.svg", "Jane"); + write("en/templates/page.html", "", "<img src='/media/template-only.svg'>"); + Files.createDirectories(temp.resolve("en/assets/images")); + Files.writeString(temp.resolve("en/assets/images/hero.svg"), "svg"); + site("en", "/", types, Map.of()); + index.rebuild(); + + var usages = index.outgoing(key("en", UsageResource.Kind.CONTENT, "index.md")); + assertThat(usages).extracting(Usage::target).contains( + key("en", UsageResource.Kind.MEDIA, "images/hero.svg"), + key("en", UsageResource.Kind.MEDIA, "images/teaser.svg"), + key("en", UsageResource.Kind.CONTENT, "about.md"), + key("en", UsageResource.Kind.COLLECTION_ITEM, "authors/jane.md")); + assertThat(usages).allMatch(usage -> usage.sourceStatus().equals("draft")); + assertThat(usages).noneMatch(usage -> usage.originalReference().contains("ignored") || usage.originalReference().contains("elsewhere") || usage.originalReference().startsWith("mailto:")); + assertThat(usages).anyMatch(usage -> usage.location().equals("metadata.teasers[0].image")); + assertThat(index.incoming(key("en", UsageResource.Kind.MEDIA, "images/hero.svg"))).hasSize(2) + .allMatch(usage -> usage.targetStatus() == Usage.TargetStatus.EXISTS); + assertThat(index.incoming(key("en", UsageResource.Kind.MEDIA, "images/jane.svg"))).singleElement() + .satisfies(usage -> assertThat(usage.source()).isEqualTo(key("en", UsageResource.Kind.COLLECTION_ITEM, "authors/jane.md"))); + assertThat(index.incoming(key("en", UsageResource.Kind.MEDIA, "template-only.svg"))).isEmpty(); + assertThat(index.problems()).isEmpty(); + } + + @Test void resolvesOnlyTheCurrentSitesPublicContextAndKeepsExplicitTargetSites() throws Exception { + var types = pageTypes(new ReferenceField("related", "Related"), new ReferenceField("english", "English", "en")); + write("en/content/about.md", "template: page.html", ""); + write("en/content/details.md", "template: page.html", ""); + write("de/content/about.md", "template: page.html", ""); + write("de/content/de/nested.md", "template: page.html", ""); + write("de/content/index.md", "template: page.html\nrelated: de/nested.md\nenglish: about.md", + "[German](/about)\n<a href='/about'>English</a><a href='/de/about'>German</a>" + + "<a href='https://example.test/de/about?preview=manager#x'>German absolute</a>" + + "<a href='/details'>English details</a><a href='//elsewhere.test/de/about'>External</a>"); + site("en", "/", types, Map.of()); + site("de", "/de", types, Map.of()); + index.rebuild(); + + var usages = index.outgoing(key("de", UsageResource.Kind.CONTENT, "index.md")); + assertThat(usages).filteredOn(usage -> usage.origin() == Usage.Origin.MARKDOWN).singleElement() + .satisfies(usage -> assertThat(usage.target()).isEqualTo(key("de", UsageResource.Kind.CONTENT, "about.md"))); + assertThat(usages).filteredOn(usage -> usage.location().equals("metadata.related")).singleElement() + .satisfies(usage -> assertThat(usage.target().path()).isEqualTo("de/nested.md")); + assertThat(usages).filteredOn(usage -> usage.originalReference().equals("/about") && usage.origin() == Usage.Origin.HTML).singleElement() + .satisfies(usage -> assertThat(usage.target().kind()).isEqualTo(UsageResource.Kind.UNRESOLVED_URL)); + assertThat(usages).anyMatch(usage -> usage.location().equals("metadata.english") + && usage.target().equals(key("en", UsageResource.Kind.CONTENT, "about.md"))); + assertThat(usages).noneMatch(usage -> usage.target().equals(key("en", UsageResource.Kind.CONTENT, "details.md"))); + assertThat(usages).noneMatch(usage -> usage.originalReference().startsWith("//elsewhere")); + assertThat(index.problems()).isEmpty(); + } + + @Test void replacesOutgoingEdgesAndKeepsBrokenIncomingEdgesAcrossDeletion() throws Exception { + var types = pageTypes(new MediaField("hero", "Hero")); + var source = write("en/content/index.md", "template: page.html\nhero: first.svg", "[About](/about)"); + var about = write("en/content/about.md", "template: page.html", ""); + var site = site("en", "/", types, Map.of()); + index.rebuild(); + write("en/content/index.md", "template: page.html\nhero: second.svg", "[About](/about)"); + index.refresh(source); + assertThat(index.incoming(key("en", UsageResource.Kind.MEDIA, "first.svg"))).isEmpty(); + assertThat(index.incoming(key("en", UsageResource.Kind.MEDIA, "second.svg"))).hasSize(1); + + Files.delete(about); + ((FileDB) site.db()).reindex(); + index.refresh(about); + assertThat(index.incoming(key("en", UsageResource.Kind.CONTENT, "about.md"))).singleElement() + .satisfies(usage -> assertThat(usage.targetStatus()).isEqualTo(Usage.TargetStatus.MISSING)); + Files.delete(source); + index.refresh(source); + assertThat(index.incoming(key("en", UsageResource.Kind.CONTENT, "about.md"))).isEmpty(); + assertThat(index.incoming(key("en", UsageResource.Kind.MEDIA, "second.svg"))).isEmpty(); + } + + @Test void resolvesCollectionRoutesAndSharedCollectionsToTheirOwningSite() throws Exception { + var types = pageTypes(new CollectionField("author", "Author", "authors")); + types.registerCollection(new CollectionType("authors", Map.of("main", FormDefinition.empty()))); + write("en/collections/authors/jane.md", "title: Jane", ""); + write("de/content/index.md", "template: page.html\nauthor: jane", "<a href='/de/authors/jane'>Jane</a>"); + var en = site("en", "/", types, Map.of("authors", new CollectionDefinition("authors", new CollectionDetailConfiguration("/authors/{id}", "author.html")))); + var enIndex = index; + ServiceRegistry.getInstance().register("en", SiteDBService.class, new SiteDBService(en.db())); + site("de", "/de", types, Map.of("authors", new CollectionDefinition("authors", "en", new CollectionDetailConfiguration("/authors/{id}", "author.html")))); + index.rebuild(); + assertThat(index.problems()).isEmpty(); + assertThat(enIndex.incoming(key("en", UsageResource.Kind.COLLECTION_ITEM, "authors/jane.md"))) + .as("a site index never aggregates sources from another site").isEmpty(); + assertThat(index.incoming(key("en", UsageResource.Kind.COLLECTION_ITEM, "authors/jane.md"))) + .as("outgoing: %s", index.outgoing(key("de", UsageResource.Kind.CONTENT, "index.md"))).hasSize(1) + .allMatch(usage -> usage.source().site().equals("de") + && usage.targetStatus() == Usage.TargetStatus.UNRESOLVED); + assertThat(index.problems()).isEmpty(); + } + + @Test void reportsMissingSchemasAndRetainsLastGoodReferencesOnParseFailure() throws Exception { + var types = pageTypes(new MediaField("hero", "Hero")); + var source = write("en/content/index.md", "template: page.html\nhero: first.svg", ""); + write("en/content/unknown.md", "template: unknown.html", "![Still indexed](/media/body.svg)"); + site("en", "/", types, Map.of()); + index.rebuild(); + assertThat(index.problems()).anyMatch(problem -> problem.path().equals("unknown.md")); + assertThat(index.incoming(key("en", UsageResource.Kind.MEDIA, "body.svg"))).hasSize(1); + Files.writeString(source, "---\nhero: [\n---\n"); + index.refresh(source); + assertThat(index.incoming(key("en", UsageResource.Kind.MEDIA, "first.svg"))).hasSize(1); + assertThat(index.problems()).anyMatch(problem -> problem.path().equals("index.md")); + } + + @Test void updatesFromCollectionAndContentEventsAndRebuildsChangedSchemas() throws Exception { + var types = pageTypes(new MediaField("hero", "Hero")); + types.registerCollection(new CollectionType("authors", Map.of("main", new FormDefinition(List.of(new MediaField("portrait", "Portrait")))))); + write("en/collections/authors/jane.md", "portrait: first.svg", ""); + write("en/content/index.md", "template: page.html\nhero: before.svg", ""); + var site = site("en", "/", types, Map.of()); + var bus = buses.get("en"); + bus.register(CollectionChangedEvent.class, event -> index.refresh(event.path())); + bus.register(ContentChangedEvent.class, event -> index.refresh(event.contentPath())); + index.rebuild(); + write("en/collections/authors/jane.md", "portrait: second.svg", ""); + site.db().getCollections().refresh("authors", "jane"); + await().untilAsserted(() -> { + assertThat(index.incoming(key("en", UsageResource.Kind.MEDIA, "second.svg"))).hasSize(1); + assertThat(index.incoming(key("en", UsageResource.Kind.MEDIA, "first.svg"))).isEmpty(); + }); + write("en/content/index.md", "template: page.html\nhero: after.svg", ""); + bus.syncPublish(new ReIndexContentMetaDataEvent("index.md")); + site.db().getFileSystem().flushContentChanges(); + await().untilAsserted(() -> assertThat(index.incoming(key("en", UsageResource.Kind.MEDIA, "after.svg"))).hasSize(1)); + types.registerCollection(new CollectionType("authors", Map.of("main", FormDefinition.empty()))); + index.rebuild(); + assertThat(index.incoming(key("en", UsageResource.Kind.MEDIA, "second.svg"))).isEmpty(); + } + + @Test void includesSectionsVariantsTabsNestedListsAndRelativeLinksFromCustomUrls() throws Exception { + var types = new ContentTypes(); + var main = new FormDefinition(List.of(new ListField("cards", "Cards")), + List.of(new FormTab("Images", List.of(new MediaField("seo.image", "SEO"))))); + types.registerPageTemplate(new PageTemplate("page", "page.html", Map.of( + "main", main, + "cards", new FormDefinition(List.of(new MediaField("image", "Image"), new ListField("links", "Links"))), + "links", new FormDefinition(List.of(new ReferenceField("target", "Target")))))); + types.registerSectionEntryTemplate(new SectionEntryTemplate("hero", "hero", "hero.html", + Map.of("main", new FormDefinition(List.of(new MediaField("image", "Image")))))); + write("de/content/index.md", "template: page.html\nurl: /news/article\nseo:\n image: seo.svg\ncards:\n - image: card.svg\n links:\n - target: about.md\nimage: not-a-root-field.svg", ""); + write("de/content/index.hero.one.md", "template: hero.html\nimage: section.svg", "<img src='../media/relative.svg'>"); + write("de/content/about.md", "template: page.html\naliases: [/old-about]", ""); + write("de/content/variant.md", "template: page.html\nstatus: draft\nseo:\n image: variant.svg", "<a href='/de/old-about'>Alias</a>"); + site("de", "/de", types, Map.of()); + index.rebuild(); + assertThat(index.incoming(key("de", UsageResource.Kind.MEDIA, "not-a-root-field.svg"))).isEmpty(); + assertThat(index.incoming(key("de", UsageResource.Kind.MEDIA, "seo.svg"))).hasSize(1); + assertThat(index.incoming(key("de", UsageResource.Kind.MEDIA, "card.svg"))).hasSize(1); + assertThat(index.incoming(key("de", UsageResource.Kind.MEDIA, "variant.svg"))).hasSize(1); + assertThat(index.incoming(key("de", UsageResource.Kind.MEDIA, "relative.svg"))).singleElement() + .satisfies(usage -> assertThat(usage.source().path()).isEqualTo("index.hero.one.md")); + assertThat(index.incoming(key("de", UsageResource.Kind.CONTENT, "about.md"))).hasSize(2) + .anyMatch(usage -> usage.location().equals("metadata.cards[0].links[0].target")); + assertThat(index.problems()).isEmpty(); + } + + @Test void updatesMediaExistenceWithoutReparsingSourcesAndPreservesUnknownUrls() throws Exception { + write("en/content/index.md", "template: page.html", "<img src='/media/folder/a%20b.svg?format=small#x'>" + + "<a href='/missing'>Missing</a><a href='#here'>Same page</a>" + + "<img srcset='data:image/svg+xml;base64,AAAA 1x, /media/real.svg 2x'>"); + site("en", "/", pageTypes(), Map.of()); + index.rebuild(); + var media = key("en", UsageResource.Kind.MEDIA, "folder/a b.svg"); + assertThat(index.incoming(media)).singleElement().satisfies(usage -> assertThat(usage.targetStatus()).isEqualTo(Usage.TargetStatus.MISSING)); + Files.createDirectories(temp.resolve("en/assets/folder")); + Files.writeString(temp.resolve("en/assets/folder/a b.svg"), "svg"); + assertThat(index.incoming(media)).singleElement().satisfies(usage -> assertThat(usage.targetStatus()).isEqualTo(Usage.TargetStatus.EXISTS)); + assertThat(index.outgoing(key("en", UsageResource.Kind.CONTENT, "index.md"))) + .anyMatch(usage -> usage.target().kind() == UsageResource.Kind.UNRESOLVED_URL && usage.originalReference().equals("/missing")) + .noneMatch(usage -> usage.originalReference().startsWith("data:") || usage.originalReference().startsWith("#")); + assertThat(index.incoming(key("en", UsageResource.Kind.MEDIA, "real.svg"))).hasSize(1); + } + + @Test void resolvesRelativeCollectionLinksOnlyWhenTheirPublicBaseIsKnown() throws Exception { + var types = pageTypes(); + types.registerCollection(new CollectionType("authors", Map.of("main", FormDefinition.empty()))); + write("en/collections/authors/jane.md", "title: Jane", "<img src='portrait.svg'>"); + site("en", "/", types, Map.of()); + index.rebuild(); + assertThat(index.outgoing(key("en", UsageResource.Kind.COLLECTION_ITEM, "authors/jane.md"))).singleElement() + .satisfies(usage -> assertThat(usage.targetStatus()).isEqualTo(Usage.TargetStatus.UNRESOLVED)); + } + + @Test void contextReloadReevaluatesPublicUrlsWithoutStrippingTypedFieldPaths() throws Exception { + var types = pageTypes(new ReferenceField("related", "Related")); + write("de/content/index.md", "template: page.html\nrelated: about.md", "<a href='/de/about'>Old URL</a>"); + write("de/content/about.md", "template: page.html", ""); + var site = site("de", "/de", types, Map.of()); + index.rebuild(); + when(site.properties().contextPath()).thenReturn("/german"); + index.rebuild(); + assertThat(index.incoming(key("de", UsageResource.Kind.CONTENT, "about.md"))).hasSize(2) + .anyMatch(usage -> usage.origin() == Usage.Origin.CONTENT_TYPE && usage.targetStatus() == Usage.TargetStatus.EXISTS) + .anyMatch(usage -> usage.origin() == Usage.Origin.HTML && usage.targetStatus() == Usage.TargetStatus.MISSING); + } + + @Test void reopensLuceneWithoutReextractingUnchangedSourcesOrCreatingANewCommit() throws Exception { + var types = pageTypes(new MediaField("hero", "Hero")); + write("en/content/index.md", "template: page.html\nhero: persistent.svg", ""); + var site = site("en", "/", types, Map.of()); + index.rebuild(); + index.close(); + long generation = generation(site.root()); + + var unchangedTypes = new FailOnSchemaLookup(); + unchangedTypes.registerPageTemplate(new PageTemplate("page", "page.html", Map.of("main", + new FormDefinition(List.of(new MediaField("hero", "Hero")))))); + var reopenedSite = new UsageSite(site.id(), site.root(), site.db(), site.configuration(), () -> unchangedTypes); + try (var reopened = new EditorialUsageIndex(reopenedSite)) { + // Available directly from Lucene, before filesystem reconciliation. + assertThat(reopened.incoming(key("en", UsageResource.Kind.MEDIA, "persistent.svg"))).hasSize(1); + reopened.synchronize(); + assertThat(reopened.problems()).isEmpty(); + assertThat(generation(site.root())).isEqualTo(generation); + } + } + + @Test void reconcilesOfflineChangesAndPreservesDeletedCustomUrlTargetsAcrossRestarts() throws Exception { + var types = pageTypes(new MediaField("hero", "Hero")); + write("en/content/index.md", "template: page.html\nhero: first.svg", "<a href='/special'>Target</a>"); + var target = write("en/content/target.md", "template: page.html\nurl: /special", ""); + var removed = write("en/content/removed.md", "template: page.html\nhero: removed.svg", ""); + var site = site("en", "/", types, Map.of()); + index.rebuild(); + index.close(); + Files.delete(target); + Files.delete(removed); + write("en/content/index.md", "template: page.html\nhero: second.svg", "<a href='/special'>Target</a>"); + write("en/content/added.md", "template: page.html\nhero: added.svg", ""); + ((FileDB) site.db()).reindex(); + try (var reopened = new EditorialUsageIndex(site)) { + reopened.synchronize(); + assertThat(reopened.incoming(key("en", UsageResource.Kind.MEDIA, "first.svg"))).isEmpty(); + assertThat(reopened.incoming(key("en", UsageResource.Kind.MEDIA, "removed.svg"))).isEmpty(); + assertThat(reopened.incoming(key("en", UsageResource.Kind.MEDIA, "second.svg"))).hasSize(1); + assertThat(reopened.incoming(key("en", UsageResource.Kind.MEDIA, "added.svg"))).hasSize(1); + assertThat(reopened.incoming(key("en", UsageResource.Kind.CONTENT, "target.md"))).singleElement() + .satisfies(usage -> assertThat(usage.targetStatus()).isEqualTo(Usage.TargetStatus.MISSING)); + } + try (var again = new EditorialUsageIndex(site)) { + again.synchronize(); + assertThat(again.incoming(key("en", UsageResource.Kind.CONTENT, "target.md"))).singleElement() + .satisfies(usage -> assertThat(usage.targetStatus()).isEqualTo(Usage.TargetStatus.MISSING)); + assertThat(again.problems()).isEmpty(); + } + } + + @Test void changedSchemaInvalidatesStoredExtractionEvenWhenSourceFileIsUnchanged() throws Exception { + var types = pageTypes(new MediaField("hero", "Hero")); + write("en/content/index.md", "template: page.html\nhero: first.svg\nother: second.svg", ""); + var site = site("en", "/", types, Map.of()); + index.rebuild(); + index.close(); + var newTypes = pageTypes(new MediaField("other", "Other")); + try (var reopened = new EditorialUsageIndex( + new UsageSite(site.id(), site.root(), site.db(), site.configuration(), () -> newTypes))) { + reopened.synchronize(); + assertThat(reopened.incoming(key("en", UsageResource.Kind.MEDIA, "first.svg"))).isEmpty(); + assertThat(reopened.incoming(key("en", UsageResource.Kind.MEDIA, "second.svg"))).hasSize(1); + } + } + + @Test void preservesDateMetadataForCollectionRoutesAfterReopening() throws Exception { + var types = pageTypes(); + types.registerCollection(new CollectionType("events", Map.of("main", FormDefinition.empty()))); + write("en/content/index.md", "template: page.html", "<a href='/events/2026/launch'>Launch</a>"); + write("en/collections/events/launch.md", "date: 2026-09-08\nstatus: draft", ""); + var site = site("en", "/", types, Map.of("events", new CollectionDefinition("events", + new CollectionDetailConfiguration("/events/{date:yyyy}/{id}", "event.html")))); + index.rebuild(); + index.close(); + try (var reopened = new EditorialUsageIndex(site)) { + reopened.synchronize(); + assertThat(reopened.incoming(key("en", UsageResource.Kind.COLLECTION_ITEM, "events/launch.md"))).hasSize(1); + assertThat(reopened.problems()).isEmpty(); + } + } + + @Test void parseErrorsAndLastGoodReferencesSurviveRestartAndRecoverOnNextValidSave() throws Exception { + var types = pageTypes(new MediaField("hero", "Hero")); + var source = write("en/content/index.md", "template: page.html\nhero: original.svg", ""); + var site = site("en", "/", types, Map.of()); + index.rebuild(); + Files.writeString(source, "---\nhero: [\n---\n"); + index.refresh(source); + index.close(); + try (var reopened = new EditorialUsageIndex(site)) { + assertThat(reopened.problems()).anyMatch(problem -> problem.path().equals("index.md")); + assertThat(reopened.incoming(key("en", UsageResource.Kind.MEDIA, "original.svg"))).hasSize(1); + reopened.synchronize(); + assertThat(reopened.problems()).anyMatch(problem -> problem.path().equals("index.md")); + write("en/content/index.md", "template: page.html\nhero: recovered.svg", ""); + reopened.refresh(source); + assertThat(reopened.problems()).isEmpty(); + assertThat(reopened.incoming(key("en", UsageResource.Kind.MEDIA, "original.svg"))).isEmpty(); + assertThat(reopened.incoming(key("en", UsageResource.Kind.MEDIA, "recovered.svg"))).hasSize(1); + } + } + + @Test void failedReconciliationKeepsTheCommittedLuceneGraph() throws Exception { + var types = pageTypes(new MediaField("hero", "Hero")); + write("en/content/index.md", "template: page.html\nhero: persistent.svg", ""); + var site = site("en", "/", types, Map.of()); + index.rebuild(); + index.close(); + + var unavailableTypes = new UsageSite(site.id(), site.root(), site.db(), site.configuration(), () -> { + throw new IllegalStateException("Content Types unavailable"); + }); + try (var reopened = new EditorialUsageIndex(unavailableTypes)) { + reopened.synchronize(); + assertThat(reopened.incoming(key("en", UsageResource.Kind.MEDIA, "persistent.svg"))).hasSize(1); + assertThat(reopened.problems()).anyMatch(problem -> problem.message().contains("Content Types unavailable")); + } + } + + private long generation(Path root) throws Exception { + try (var directory = org.apache.lucene.store.FSDirectory.open(root.resolve("data/usage/index")); + var reader = org.apache.lucene.index.DirectoryReader.open(directory)) { + return reader.getIndexCommit().getGeneration(); + } + } + + static class FailOnSchemaLookup extends ContentTypes { + @Override public Set<PageTemplate> getPageTemplates() { + throw new AssertionError("Unchanged source was extracted again"); + } + } + + private ContentTypes pageTypes(FormField... fields) { + var types = new ContentTypes(); + types.registerPageTemplate(new PageTemplate("page", "page.html", Map.of("main", new FormDefinition(List.of(fields))))); + return types; + } + + private UsageSite site(String id, String context, ContentTypes types, Map<String, CollectionDefinition> collections) throws Exception { + Path root = temp.resolve(id); + for (String folder : List.of("content", "assets", "templates", "extensions", "collections")) Files.createDirectories(root.resolve(folder)); + var properties = mock(SiteProperties.class); + when(properties.id()).thenReturn(id); + when(properties.contextPath()).thenReturn(context); + when(properties.baseUrl()).thenReturn("https://example.test"); + when(properties.hostnames()).thenReturn(List.of("example.test")); + var configuration = new Configuration(); + configuration.add(SiteConfiguration.class, new SiteConfiguration(properties)); + configuration.add(CollectionConfiguration.class, new CollectionConfiguration(collections)); + var bus = new DefaultEventBus(); + buses.put(id, bus); + var db = new FileDB(root, bus, path -> { + try { return new ContentFileParser(path.toString()).getHeader(); } + catch (Exception ex) { throw new IllegalStateException(ex); } + }, configuration); + databases.add(db); + db.init(); + var site = new UsageSite(id, root, db, configuration, () -> types); + index = new EditorialUsageIndex(site); + indexes.add(index); + return site; + } + + private Path write(String file, String header, String body) throws Exception { + Path path = temp.resolve(file); + Files.createDirectories(path.getParent()); + Files.writeString(path, "---\n" + header + "\n---\n" + body); + return path; + } + + private UsageResource key(String site, UsageResource.Kind kind, String path) { return new UsageResource(site, kind, path); } +} diff --git a/cms-content/src/test/java/com/condation/cms/content/usage/LuceneUsageStoreTest.java b/cms-content/src/test/java/com/condation/cms/content/usage/LuceneUsageStoreTest.java new file mode 100644 index 000000000..7c23b4596 --- /dev/null +++ b/cms-content/src/test/java/com/condation/cms/content/usage/LuceneUsageStoreTest.java @@ -0,0 +1,95 @@ +package com.condation.cms.content.usage; + +/*- + * #%L + * CMS Content + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.condation.cms.api.usage.*; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import static org.assertj.core.api.Assertions.*; + +class LuceneUsageStoreTest { + @TempDir Path root; + + @Test void queriesAllHitsAcrossPagesAndAtomicallyReplacesBothDirections() throws Exception { + var target = new UsageResource("de", UsageResource.Kind.MEDIA, "image.svg"); + var replacement = new UsageResource("de", UsageResource.Kind.MEDIA, "replacement.svg"); + try (var store = new LuceneUsageStore(root)) { + for (int i = 0; i < 300; i++) store.update(source("page" + i + ".md", target)); + store.commit(List.of()); + assertThat(store.incoming(target)).hasSize(300); + store.update(source("page0.md", replacement)); + // Readers see the last committed batch until both directions have been committed. + assertThat(store.incoming(target)).hasSize(300); + assertThat(store.incoming(replacement)).isEmpty(); + store.commit(List.of()); + assertThat(store.incoming(target)).hasSize(299); + assertThat(store.incoming(replacement)).hasSize(1); + } + try (var store = new LuceneUsageStore(root)) { + assertThat(store.sources()).hasSize(300); + assertThat(store.incoming(target)).hasSize(299); + var source = new UsageResource("en", UsageResource.Kind.CONTENT, "page0.md"); + assertThat(store.outgoing(source)).singleElement().satisfies(usage -> assertThat(usage.target()).isEqualTo(replacement)); + store.delete(source); + store.commit(List.of()); + assertThat(store.incoming(replacement)).isEmpty(); + assertThat(store.outgoing(source)).isEmpty(); + } + } + + @Test void resolvesAliasesAndCollectionRoutesFromLucene() throws Exception { + var page = new UsageResource("en", UsageResource.Kind.CONTENT, "page.md"); + var item = new UsageResource("en", UsageResource.Kind.COLLECTION_ITEM, "events/launch.md"); + var duplicate = new UsageResource("en", UsageResource.Kind.CONTENT, "duplicate.md"); + try (var store = new LuceneUsageStore(root)) { + store.update(new PersistedUsageSource( + new UsageDocument(page, "/page", Map.of("aliases", List.of("/special")), List.of()), + "stamp", "schema", List.of(), List.of(), List.of())); + store.update(new PersistedUsageSource( + new UsageDocument(item, "/events/2026/launch", Map.of("title", "Launch"), List.of()), + "stamp", "schema", List.of(), List.of(), List.of())); + store.commit(List.of()); + + assertThat(store.source(page)).get().extracting(PersistedUsageSource::document) + .isEqualTo(new UsageDocument(page, "/page", Map.of("aliases", List.of("/special")), List.of())); + assertThat(store.aliasTarget("special")).contains(page); + assertThat(store.collectionTarget("/events/2026/launch/")).contains(item); + + store.update(new PersistedUsageSource( + new UsageDocument(duplicate, "/duplicate", Map.of("aliases", List.of("/special")), List.of()), + "stamp", "schema", List.of(), List.of(), List.of())); + store.commit(List.of()); + assertThat(store.aliasTarget("/special")).as("ambiguous aliases are unresolved").isEmpty(); + } + } + + private PersistedUsageSource source(String path, UsageResource target) { + var source = new UsageResource("en", UsageResource.Kind.CONTENT, path); + var document = new UsageDocument(source, "/", Map.of("title", "Page"), List.of()); + var usage = new Usage(source, target, "metadata.image", Usage.Origin.CONTENT_TYPE, target.path(), + "Page", "draft", Usage.TargetStatus.EXISTS); + return new PersistedUsageSource(document, "stamp", "schema", List.of(usage), List.of(), List.of()); + } +} diff --git a/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileCollections.java b/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileCollections.java index 7b7405f50..db7667d66 100644 --- a/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileCollections.java +++ b/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileCollections.java @@ -71,6 +71,12 @@ public class FileCollections implements Collections, CollectionCursorSupport, Au private CollectionMetaData metaData; private MultiRootRecursiveWatcher watcher; private ContentChangeCoordinator changeCoordinator; + private Consumer<Path> changeListener = path -> {}; + + /** Invoked after metadata changes are visible to readers. */ + public void onChange(Consumer<Path> listener) { + this.changeListener = Objects.requireNonNull(listener); + } public FileCollections( String siteId, @@ -147,6 +153,7 @@ public void refresh(String collection, String id) { } else { metaData.removeFile(collection + "/" + id + ".md"); } + changeListener.accept(file); } catch (IOException ex) { throw new IllegalStateException("could not refresh collection item", ex); } @@ -173,10 +180,12 @@ private void processChanges(boolean fullResync, Set<Path> paths) { try { if (fullResync) { rebuild(true); + changeListener.accept(collectionsBase); return; } for (var path : paths) { processPath(path); + changeListener.accept(path); } } catch (IOException ex) { log.error("error processing collection changes", ex); diff --git a/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileDB.java b/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileDB.java index 9c1bb0bd3..1335e6717 100644 --- a/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileDB.java +++ b/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileDB.java @@ -72,6 +72,8 @@ public void init () throws IOException { content = new FileContent(fileSystem); localCollections = new FileCollections(siteProperties.id(), hostBaseDirectory, contentParser); + localCollections.onChange(path -> eventBus.publish( + new com.condation.cms.api.eventbus.events.CollectionChangedEvent(path))); localCollections.init(); var collectionConfiguration = configuration.get( com.condation.cms.api.configuration.configs.CollectionConfiguration.class); diff --git a/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileSystem.java b/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileSystem.java index a28e6061c..8d1034d4d 100644 --- a/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileSystem.java +++ b/cms-filesystem/src/main/java/com/condation/cms/filesystem/FileSystem.java @@ -285,8 +285,16 @@ public void init() throws IOException { this.contentChangeCoordinator = new ContentChangeCoordinator( CONTENT_CHANGE_QUIET_PERIOD, this::processContentChanges); var templateBase = resolve("templates/"); + var extensionsBase = resolve("extensions/"); log.debug("init filewatcher"); - this.fileWatcher = new MultiRootRecursiveWatcher(siteId, List.of(contentBase, templateBase)); + this.fileWatcher = new MultiRootRecursiveWatcher(siteId, List.of(contentBase, templateBase, extensionsBase)); + fileWatcher.getPublisher(extensionsBase).subscribe(new MultiRootRecursiveWatcher.AbstractFileEventSubscriber() { + @Override + public void onNext(FileEvent item) { + eventBus.publish(new com.condation.cms.api.eventbus.events.ContentTypesChangedEvent()); + this.subscription.request(1); + } + }); fileWatcher.getPublisher(contentBase).subscribe(new MultiRootRecursiveWatcher.AbstractFileEventSubscriber() { @Override public void onNext(FileEvent item) { diff --git a/cms-server/src/main/java/com/condation/cms/cli/commands/server/AddUser.java b/cms-server/src/main/java/com/condation/cms/cli/commands/server/AddUser.java index 36c470ca3..5503ad3a0 100644 --- a/cms-server/src/main/java/com/condation/cms/cli/commands/server/AddUser.java +++ b/cms-server/src/main/java/com/condation/cms/cli/commands/server/AddUser.java @@ -21,7 +21,6 @@ * #L% */ -import com.condation.cms.api.Constants; import com.condation.cms.api.utils.ServerUtil; import com.condation.cms.auth.services.Realm; diff --git a/cms-server/src/main/java/com/condation/cms/request/RequestContextFactory.java b/cms-server/src/main/java/com/condation/cms/request/RequestContextFactory.java index 889c84162..6a4859113 100644 --- a/cms-server/src/main/java/com/condation/cms/request/RequestContextFactory.java +++ b/cms-server/src/main/java/com/condation/cms/request/RequestContextFactory.java @@ -87,11 +87,22 @@ public class RequestContextFactory { private final Injector injector; + /** Loads the same typed editor schemas as the manager, without requiring an HTTP request. */ + public com.condation.cms.api.ui.elements.ContentTypes contentTypes() { + var properties = injector.getInstance(Configuration.class).get(SiteConfiguration.class).siteProperties(); + try (var context = create(properties.contextPath(), "/", Map.of())) { + return ScopedValue.where(com.condation.cms.api.request.RequestContextScope.REQUEST_CONTEXT, context) + .call(() -> com.condation.cms.api.ui.elements.ContentTypeProvider.load( + context.get(HookSystemFeature.class).hookSystem())); + } catch (Exception ex) { + throw new IllegalStateException("Cannot load content types for " + properties.id(), ex); + } + } + public RequestContext createContext () { var requestContext = new RequestContext(); var theme = injector.getInstance(Theme.class); -// var siteProperties = injector.getInstance(SiteProperties.class); var siteMediaService = injector.getInstance(MediaService.class); requestContext.add(InjectorFeature.class, new InjectorFeature(injector)); diff --git a/cms-server/src/main/java/com/condation/cms/server/configs/SiteModule.java b/cms-server/src/main/java/com/condation/cms/server/configs/SiteModule.java index f585515c6..50f229571 100644 --- a/cms-server/src/main/java/com/condation/cms/server/configs/SiteModule.java +++ b/cms-server/src/main/java/com/condation/cms/server/configs/SiteModule.java @@ -36,6 +36,7 @@ import com.condation.cms.api.cache.ICache; import com.condation.cms.api.configuration.Configuration; import com.condation.cms.api.configuration.configs.ServerConfiguration; +import com.condation.cms.api.configuration.configs.SiteConfiguration; import com.condation.cms.api.content.ContentParser; import com.condation.cms.api.content.RenderContentFunction; import com.condation.cms.api.db.DB; @@ -71,6 +72,8 @@ import com.condation.cms.api.variants.VariantSelector; import com.condation.cms.content.VariantSelectorConfigurationRepository; import com.condation.cms.content.ViewResolver; +import com.condation.cms.content.usage.EditorialUsageIndex; +import com.condation.cms.content.usage.UsageSite; import com.condation.cms.content.shortcodes.ShortCodeParser; import com.condation.cms.content.template.functions.taxonomy.TaxonomyFunction; import com.condation.cms.core.request.visitor.VisitorContextService; @@ -123,6 +126,7 @@ protected void configure() { //bind(ContentParser.class).to(DefaultContentParser.class).in(Singleton.class); bind(TaxonomyFunction.class).in(Singleton.class); bind(TaxonomyResolver.class).in(Singleton.class); + bind(com.condation.cms.api.usage.UsageIndex.class).to(EditorialUsageIndex.class); } @Provides @@ -293,6 +297,15 @@ public FileDB fileDb(DB db) throws IOException { return (FileDB) db; } + @Provides + @Singleton + public EditorialUsageIndex usageIndex(DB db, Configuration configuration, + RequestContextFactory requestContextFactory) { + var id = configuration.get(SiteConfiguration.class).siteProperties().id(); + return new EditorialUsageIndex(new UsageSite(id, db.getFileSystem().hostBase(), db, + configuration, requestContextFactory::contentTypes)); + } + @Provides @Singleton public MessageSource messages(SiteProperties site, DB db, CacheManager cacheManager) throws IOException { diff --git a/cms-server/src/main/java/com/condation/cms/server/host/Initializer.java b/cms-server/src/main/java/com/condation/cms/server/host/Initializer.java index ba9337b99..2c6be8199 100644 --- a/cms-server/src/main/java/com/condation/cms/server/host/Initializer.java +++ b/cms-server/src/main/java/com/condation/cms/server/host/Initializer.java @@ -24,6 +24,14 @@ import com.condation.cms.api.configuration.Configuration; import com.condation.cms.api.db.DB; import com.condation.cms.api.eventbus.EventBus; +import com.condation.cms.api.eventbus.events.CollectionChangedEvent; +import com.condation.cms.api.eventbus.events.ConfigurationReloadEvent; +import com.condation.cms.api.eventbus.events.ContentChangedEvent; +import com.condation.cms.api.eventbus.events.ContentTypesChangedEvent; +import com.condation.cms.api.eventbus.events.lifecycle.HostReadyEvent; +import com.condation.cms.api.eventbus.events.lifecycle.HostReloadedEvent; +import com.condation.cms.api.eventbus.events.lifecycle.HostStoppedEvent; +import com.condation.cms.content.usage.EditorialUsageIndex; import com.condation.cms.core.serivce.ServiceRegistry; import com.condation.cms.core.serivce.impl.NodeTranslationService; import com.condation.cms.core.serivce.impl.SiteDBService; @@ -52,5 +60,20 @@ void initServices () { ServiceRegistry.getInstance().register(host.id(), SitePropertiesService.class, new SitePropertiesService(config)); ServiceRegistry.getInstance().register(host.id(), NodeTranslationService.class, new NodeTranslationService(db, host.injector.getInstance(EventBus.class))); + initUsageIndex(); + } + + private void initUsageIndex() { + var index = host.injector.getInstance(EditorialUsageIndex.class); + var events = host.injector.getInstance(EventBus.class); + events.register(ContentChangedEvent.class, + event -> index.refresh(event.contentPath())); + events.register(CollectionChangedEvent.class, + event -> index.refresh(event.path())); + events.register(HostReadyEvent.class, event -> index.synchronize()); + events.register(HostReloadedEvent.class, event -> index.synchronize()); + events.register(ConfigurationReloadEvent.class, event -> index.synchronize()); + events.register(ContentTypesChangedEvent.class, event -> index.synchronize()); + events.register(HostStoppedEvent.class, event -> index.close()); } } diff --git a/cms-server/src/main/java/com/condation/cms/server/host/VHost.java b/cms-server/src/main/java/com/condation/cms/server/host/VHost.java index ebef54628..897903dcf 100644 --- a/cms-server/src/main/java/com/condation/cms/server/host/VHost.java +++ b/cms-server/src/main/java/com/condation/cms/server/host/VHost.java @@ -173,6 +173,7 @@ public void reindex() { try { injector.getInstance(ConfigManagement.class).reload(); injector.getInstance(FileDB.class).reindex(); + injector.getInstance(com.condation.cms.api.usage.UsageIndex.class).rebuild(); log.info("reindex of host {} completed", id()); } catch (Exception e) { log.error("reindex of host {} failed", id(), e); diff --git a/documentation/usage-index.md b/documentation/usage-index.md new file mode 100644 index 000000000..279274f9c --- /dev/null +++ b/documentation/usage-index.md @@ -0,0 +1,137 @@ +# Editorial usage index + +Each site injector owns one persistent Lucene index of direct references in that site's +pages, sections, variants and local collection items. Drafts and scheduled content are included. +Themes, templates, dynamic queries and shortcode execution are outside its scope. + +## Identity and extraction + +Resources use `(site, kind, path)` as their identity. Paths are relative to the content, +assets or collections root, including the filename; collection paths include the collection: + +```text +(de, CONTENT, about/index.md) +(de, MEDIA, images/logo.svg) +(shared, COLLECTION_ITEM, authors/jane.md) +``` + +The existing `manager/contentTypes/register` hook supplies schemas through the shared +`ContentTypeProvider`. Background indexing loads these in a temporary execution context +using `RequestContextFactory.contentTypes()`; the manager need not be open. + +Media, reference and collection fields are inspected, including tabs, dot-separated field +names and nested lists. List forms use the enclosing type's named form, then the globally +registered list item type, following the manager's convention. List item forms are not +also interpreted as top-level metadata forms. Missing or ambiguous schemas produce a +coverage problem; body references can still be indexed. + +Markdown extraction uses the CMS inline tokenizer for inline links, images and linked +images. Raw HTML supports `href`, `src`, `poster` and `srcset`. Code fences, code spans, +indented code, HTML comments and script/style/pre/code contents are excluded. Locations +use a metadata field path or a text-node/HTML-element location, not a source-file line number. +Markdown constructs unsupported by the CMS tokenizer and references in arbitrary string, +code or shortcode parameters are not inferred. + +## Paths and site contexts + +Typed field values are site-local paths. Their context prefix is **not** removed. Reference +fields can specify a target site; collection fields resolve the configured collection's +owning site. Existing content URLs accepted in reference fields are resolved to file paths. + +Raw HTML URLs and Markdown image URLs are public URLs. Relative URLs are resolved against +the source page's public URL (sections use their owning page). Host, port and the current +site's context must match before that context is stripped. `/de` never matches `/details`. +An absolute HTTP(S) URL or a protocol-relative URL is internal only if it matches the site. + +Markdown **links** already receive the site context through `HTTPUtil.prependContext` +when the CMS renders them. The index reproduces that behavior before resolving the URL. +For a German site with context `/de`, `[About](/about)` therefore refers to its own +`/about`, while raw HTML `<a href="/about">` is outside this site and remains unresolved. +No templates are evaluated to discover `cms.links.createUrl` calls. + +Media URLs under `/assets/` and `/media/`, including format/preview query parameters, +resolve to the original asset. Fragment-only links and non-HTTP schemes are ignored. +Content custom URLs, aliases and configured collection detail routes resolve to file paths. +Collection routes are calculated from raw metadata and indexed as exact route fields, so +unpublished targets are included without loading all collection items into memory. +Relative URLs in collection items without a configured detail route remain unresolved; +their eventual embedding page cannot be inferred statically. + +## Updates and incomplete results + +Each site's outgoing references are stored under `data/usage/index`. Incoming and outgoing +queries run only against that site's Lucene index. A target may carry another site ID, for +example for an explicitly targeted reference or shared collection, but the index never opens +or aggregates another site's store. Such a target's existence status remains `UNRESOLVED` +inside this index. +One Lucene document per source stores its extracted references, resolved usages, last known +targets, metadata and coverage problems. Inverted target, alias and collection-route fields +support reverse and public-route lookup. +Updates to both reference directions become visible together after a durable commit. + +When the site-scoped instance is created, stored usages are immediately queryable directly +from Lucene. No complete in-memory copy is restored. When its host becomes ready, the index +compares filesystem stamps (modification time, size and file key) and a canonical fingerprint +of the Content Types. Only new/changed files or changed schemas require extraction. Deleted +sources are removed. This detects changes made while the server was stopped; deliberately +preserving all file-stamp attributes while changing bytes requires an explicit rebuild. +Stored sources are streamed from a Lucene reader snapshot and their references are resolved +again against the current indexed route catalog. Unchanged Lucene documents are not rewritten. +The index format and extraction algorithm are versioned. + +Content changes are processed after the +content metadata index; `CollectionChangedEvent` likewise runs after collection metadata +updates, including manager saves and filesystem changes. Updating a source replaces its +outgoing references atomically. Directory changes rescan the source site. Other stored +references are resolved again because target routes may have changed, without rereading +their bodies. Media existence is checked when usages are requested. + +Configuration/host reloads and `ContentTypesChangedEvent` reconcile the index. Changes in a +site's `extensions` directory publish the latter event; extensions changing schemas by +other means should publish it explicitly. `UsageIndex.rebuild()` forces a complete extraction +when needed; the existing host reindex command also forces extraction for that site. +Shutdown commits and closes the Lucene writers without deleting their data. + +Deleting a source removes its outgoing references. Incoming references to deleted targets +remain. Unresolvable public URLs use kind `UNRESOLVED_URL`. When a previously resolved URL +breaks, the index retains its last known target with status `MISSING`, including after a +restart. A URL that was already broken before the first extraction remains unresolved; +its original target cannot be inferred. Removing the index data also removes this history. +Renaming is treated as deletion plus creation. References are not rewritten automatically. + +Parse/read failures retain the last successfully extracted references and publish a coverage +problem. Before initial indexing finishes, a pending problem is reported. An empty result +must be labelled **“No usages found in editorial content”**, never proof that deletion is safe. + +## Java and manager API + +`UsageIndex` is a site-scoped singleton registered by `SiteModule` and available through the +site injector: + +```java +var target = new UsageResource("de", UsageResource.Kind.MEDIA, "images/logo.svg"); +var incoming = usageIndex.incoming(target); +var outgoing = usageIndex.outgoing( + new UsageResource("de", UsageResource.Kind.CONTENT, "index.md")); +var incompleteSources = usageIndex.problems(); +``` + +Manager RPC methods require `content.edit`: + +| Method | Parameters | Result | +| --- | --- | --- | +| `usage.incoming` | `kind`, `path` | `{items, problems, scope: "editorial"}`; includes the current source site's coverage problems | +| `usage.outgoing` | `kind`, `path` | `{items, problems, scope: "editorial"}`; current site's coverage problems | +| `usage.problems` | none | Current site's coverage problems | + +The requested resource belongs to the current manager site; shared collection items are +resolved to their configured source site. Incoming results contain only sources managed by +the current site's index. Each usage includes source/target identities, location, origin, +original reference, source title/status and target status (`EXISTS`, `MISSING`, `UNRESOLVED`). +The manager shortcut **Show usages** in the Page section (**Ctrl+9**) opens a read-only +dialog for the current preview node, including variants and collection detail pages. +It lists incoming references with source title/path, site, location, original reference and +source status. Empty results are described as no indexed usages; coverage problems are +shown explicitly. Themes, templates and dynamic queries remain outside the scope. +The dialog uses the active preview identity or resolves the preview URL through `content.node`, +so public site contexts are not mistaken for stored file paths. diff --git a/modules/ui-module/src/main/java/com/condation/cms/modules/ui/extensionpoints/PageMenuExtension.java b/modules/ui-module/src/main/java/com/condation/cms/modules/ui/extensionpoints/PageMenuExtension.java index 712b93b3f..3833a8578 100644 --- a/modules/ui-module/src/main/java/com/condation/cms/modules/ui/extensionpoints/PageMenuExtension.java +++ b/modules/ui-module/src/main/java/com/condation/cms/modules/ui/extensionpoints/PageMenuExtension.java @@ -126,11 +126,24 @@ public void configureVariantSelector() { // can be empty } + @ShortCut( + id = "page-usages", + title = "Show usages", + permissions = {Permissions.CONTENT_EDIT}, + hotkey = "ctrl-9", + section = "Page", + scriptAction = @com.condation.cms.api.ui.annotations.ScriptAction(module = "/manager/actions/page/show-usages") + ) + public void showUsages() { + // Registered through the shortcut annotation. + } + @Override public Map<String, Map<String, String>> getLocalizations() { return Map.of( "de", Map.of( "pageMenu", "Seite", + "page-usages", "Verwendungen anzeigen", "page-create", "Neue Seite erstellen", "page-edit-content", "Inhalt bearbeiten", "page-edit-meta", "Metadaten bearbeiten", @@ -142,6 +155,7 @@ public Map<String, Map<String, String>> getLocalizations() { ), "en", Map.of( "pageMenu", "Page", + "page-usages", "Show usages", "page-create", "Create new page", "page-edit-content", "Edit content", "page-edit-meta", "Edit metadata", diff --git a/modules/ui-module/src/main/java/com/condation/cms/modules/ui/extensionpoints/remotemethods/RemoteUsageEndpoints.java b/modules/ui-module/src/main/java/com/condation/cms/modules/ui/extensionpoints/remotemethods/RemoteUsageEndpoints.java new file mode 100644 index 000000000..cef52fd20 --- /dev/null +++ b/modules/ui-module/src/main/java/com/condation/cms/modules/ui/extensionpoints/remotemethods/RemoteUsageEndpoints.java @@ -0,0 +1,82 @@ +package com.condation.cms.modules.ui.extensionpoints.remotemethods; + +/*- + * #%L + * UI Module + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.condation.cms.api.auth.Permissions; +import com.condation.cms.api.configuration.configs.CollectionConfiguration; +import com.condation.cms.api.feature.features.ConfigurationFeature; +import com.condation.cms.api.feature.features.InjectorFeature; +import com.condation.cms.api.feature.features.SitePropertiesFeature; +import com.condation.cms.api.ui.annotations.RemoteMethod; +import com.condation.cms.api.ui.extensions.UIRemoteMethodExtensionPoint; +import com.condation.cms.api.ui.rpc.RPCException; +import com.condation.cms.api.usage.UsageIndex; +import com.condation.cms.api.usage.UsageResource; +import com.condation.modules.api.annotation.Extension; +import java.util.Locale; +import java.util.Map; + +/** Read-only manager access to editorial usages of resources in the current site. */ +@Extension(UIRemoteMethodExtensionPoint.class) +public class RemoteUsageEndpoints extends AbstractRemoteMethodeExtension { + @RemoteMethod(name = "usage.incoming", permissions = {Permissions.CONTENT_EDIT}) + public Object incoming(Map<String, Object> parameters) throws RPCException { + return Map.of("items", index().incoming(resource(parameters)), "problems", index().problems(), "scope", "editorial"); + } + + @RemoteMethod(name = "usage.outgoing", permissions = {Permissions.CONTENT_EDIT}) + public Object outgoing(Map<String, Object> parameters) throws RPCException { + return Map.of("items", index().outgoing(resource(parameters)), "problems", problems(parameters), "scope", "editorial"); + } + + @RemoteMethod(name = "usage.problems", permissions = {Permissions.CONTENT_EDIT}) + public Object problems(Map<String, Object> parameters) { + return index().problems().stream().filter(problem -> problem.site().equals(site())).toList(); + } + + private UsageResource resource(Map<String, Object> parameters) throws RPCException { + if (!(parameters.get("kind") instanceof String) || !(parameters.get("path") instanceof String)) { + throw new RPCException(0, "kind and site-local path are required"); + } + try { + var resource = new UsageResource(site(), UsageResource.Kind.valueOf( + ((String) parameters.get("kind")).toUpperCase(Locale.ROOT)), (String) parameters.get("path")); + if (resource.kind() == UsageResource.Kind.COLLECTION_ITEM + && getRequestContext().has(ConfigurationFeature.class)) { + var configuration = getRequestContext().get(ConfigurationFeature.class) + .configuration().get(CollectionConfiguration.class); + if (configuration != null) { + var collection = resource.path().split("/", 2)[0]; + var owner = configuration.collection(collection) + .flatMap(definition -> definition.sourceSite()).orElse(site()); + return new UsageResource(owner, resource.kind(), resource.path()); + } + } + return resource; + } catch (IllegalArgumentException | NullPointerException ex) { + throw new RPCException(0, "kind and site-local path are required"); + } + } + + private String site() { return getRequestContext().get(SitePropertiesFeature.class).siteProperties().id(); } + private UsageIndex index() { return getRequestContext().get(InjectorFeature.class).injector().getInstance(UsageIndex.class); } +} diff --git a/modules/ui-module/src/main/java/com/condation/cms/modules/ui/utils/UIHooks.java b/modules/ui-module/src/main/java/com/condation/cms/modules/ui/utils/UIHooks.java index 7a75d370a..3011bf1e0 100644 --- a/modules/ui-module/src/main/java/com/condation/cms/modules/ui/utils/UIHooks.java +++ b/modules/ui-module/src/main/java/com/condation/cms/modules/ui/utils/UIHooks.java @@ -35,7 +35,7 @@ public class UIHooks { public static final String HOOK_MENU = "module/ui/menu"; public static final String HOOK_TRANSLATIONS = "module/ui/translations"; - public static final String HOOK_REGISTER_CONTENT_TYPES = "manager/contentTypes/register"; + public static final String HOOK_REGISTER_CONTENT_TYPES = com.condation.cms.api.ui.elements.ContentTypeProvider.REGISTER_HOOK; public static final String HOOK_REGISTER_MEDIA_FORMS = "manager/media/forms"; private final HookSystem hookSystem; @@ -45,9 +45,7 @@ public UIHooks (final HookSystem hookSystem) { } public ContentTypes contentTypes () { - var contentTypes = new ContentTypes(); - - return hookSystem.doFilter(HOOK_REGISTER_CONTENT_TYPES, contentTypes); + return com.condation.cms.api.ui.elements.ContentTypeProvider.load(hookSystem); } public MediaForms mediaForms () { diff --git a/modules/ui-module/src/main/resources/manager/actions/page/show-usages.d.ts b/modules/ui-module/src/main/resources/manager/actions/page/show-usages.d.ts new file mode 100644 index 000000000..9907a5d80 --- /dev/null +++ b/modules/ui-module/src/main/resources/manager/actions/page/show-usages.d.ts @@ -0,0 +1,21 @@ +/*- + * #%L + * UI Module + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +export declare const runAction: () => void; diff --git a/modules/ui-module/src/main/resources/manager/actions/page/show-usages.js b/modules/ui-module/src/main/resources/manager/actions/page/show-usages.js new file mode 100644 index 000000000..3c4c873cb --- /dev/null +++ b/modules/ui-module/src/main/resources/manager/actions/page/show-usages.js @@ -0,0 +1,114 @@ +/*- + * #%L + * UI Module + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ +import { i18n } from '@cms/modules/localization.js'; +import { openModal } from '@cms/modules/modal.js'; +import { getPreviewUrl } from '@cms/modules/preview.utils.js'; +import { getActivePreviewContent } from '@cms/modules/preview-context.js'; +import { getContentNode } from '@cms/modules/rpc/rpc-content.js'; +import { executeRemoteCall } from '@cms/modules/rpc/rpc.js'; +const t = (key, fallback) => i18n.t(`usage.${key}`, fallback); +const appendText = (parent, tag, text, className = '') => { + const element = document.createElement(tag); + element.textContent = text; + element.className = className; + parent.appendChild(element); + return element; +}; +const render = (container, path, result) => { + container.replaceChildren(); + appendText(container, 'p', path, 'fw-semibold text-break'); + appendText(container, 'p', t('scope', 'Shows direct editorial references. Templates, themes and dynamic queries are not included.'), 'text-body-secondary'); + if (result.problems.length) { + const warning = appendText(container, 'div', t('incomplete', 'Some sources could not be fully checked. This list may be incomplete.'), 'alert alert-warning'); + warning.setAttribute('role', 'status'); + const details = document.createElement('details'); + appendText(details, 'summary', t('problems', 'Details')); + const list = document.createElement('ul'); + result.problems.forEach(problem => appendText(list, 'li', `${problem.site} / ${problem.path}: ${problem.message}`, 'text-break')); + details.appendChild(list); + warning.appendChild(details); + } + if (!result.items.length) { + appendText(container, 'p', t('empty', 'No usages found in editorial content.'), 'mb-0'); + return; + } + appendText(container, 'p', `${t('count', 'Usages')}: ${result.items.length}`); + const wrapper = document.createElement('div'); + wrapper.className = 'table-responsive'; + const table = document.createElement('table'); + table.className = 'table table-striped align-middle'; + const headings = table.createTHead().insertRow(); + [t('source', 'Source'), t('site', 'Site'), t('location', 'Location'), t('status', 'Source status')] + .forEach(label => { + const th = appendText(headings, 'th', label); + th.setAttribute('scope', 'col'); + }); + const body = table.createTBody(); + result.items.forEach(usage => { + const row = body.insertRow(); + const source = row.insertCell(); + appendText(source, 'div', usage.sourceTitle || usage.source.path, 'fw-semibold text-break'); + appendText(source, 'small', usage.source.path, 'text-body-secondary text-break'); + row.insertCell().textContent = usage.source.site; + const location = row.insertCell(); + appendText(location, 'div', usage.location, 'text-break'); + appendText(location, 'small', usage.originalReference, 'text-body-secondary text-break'); + row.insertCell().textContent = usage.sourceStatus || '—'; + }); + wrapper.appendChild(table); + container.appendChild(wrapper); +}; +export const runAction = () => { + // Capture the node before opening the dialog or waiting for a request. + const url = getPreviewUrl(); + const activeNode = getActivePreviewContent(url); + openModal({ + title: t('title', 'Show usages'), + size: 'lg', + showFooter: false, + body: '<div data-usage-content aria-live="polite"></div>', + onShow: async (modalElement) => { + const container = modalElement.querySelector('[data-usage-content]'); + appendText(container, 'p', t('loading', 'Loading usages…')); + try { + const node = activeNode || (url ? (await getContentNode({ url })).result : null); + if (!node?.uri) { + container.replaceChildren(); + appendText(container, 'p', t('noNode', 'No content node is selected.')); + return; + } + const response = await executeRemoteCall({ + method: 'usage.incoming', + parameters: { + kind: node.contentKind === 'collection' ? 'COLLECTION_ITEM' : 'CONTENT', + path: node.uri + } + }); + render(container, node.uri, response.result); + } + catch (error) { + container.replaceChildren(); + const message = appendText(container, 'div', t('error', 'Could not load usages.'), 'alert alert-danger'); + message.setAttribute('role', 'alert'); + } + } + }); +}; diff --git a/modules/ui-module/src/main/resources/manager/index.html b/modules/ui-module/src/main/resources/manager/index.html index 7ed77cbf1..c09c43acf 100644 --- a/modules/ui-module/src/main/resources/manager/index.html +++ b/modules/ui-module/src/main/resources/manager/index.html @@ -374,12 +374,14 @@ <h2 class="modal-title h5" id="cms-apps-modal-title">Apps</h2> <script type="module"> import {executeScriptAction, executeHookAction} from "@cms/js/manager-globals.js"; + import { i18n } from "@cms/modules/localization.js"; + await i18n.init(); var shortCuts = [ {% for entry in actionFactory.createShortCuts() %} { id: '{{ entry.id }}', - label: '{{ entry.title }}', + label: i18n.t('{{ entry.id }}', '{{ entry.title }}'), icon: '{{ entry.icon }}', group: '{{ entry.section }}', action: () => { @@ -393,6 +395,17 @@ <h2 class="modal-title h5" id="cms-apps-modal-title">Apps</h2> }, {% endfor %} ]; + document.addEventListener('keydown', (event) => { + if (event.ctrlKey && !event.altKey && !event.shiftKey && !event.metaKey + && event.key === '9' && !event.repeat + && !document.querySelector('.modal.show')) { + const command = shortCuts.find(entry => entry.id === 'page-usages'); + if (command) { + event.preventDefault(); + command.action(); + } + } + }); window.manager.commandPalette = new BWCommandPalette({ trigger: 'mod+k', placeholder: 'Type a command...', diff --git a/modules/ui-module/src/main/resources/manager/js/modules/localization-actions.d.ts b/modules/ui-module/src/main/resources/manager/js/modules/localization-actions.d.ts index 6db9e8af0..51ec74ad6 100644 --- a/modules/ui-module/src/main/resources/manager/js/modules/localization-actions.d.ts +++ b/modules/ui-module/src/main/resources/manager/js/modules/localization-actions.d.ts @@ -21,6 +21,19 @@ export namespace ACTION_LOCALIZATIONS { let en: {}; let de: { + "usage.title": string; + "usage.scope": string; + "usage.incomplete": string; + "usage.problems": string; + "usage.empty": string; + "usage.count": string; + "usage.source": string; + "usage.site": string; + "usage.location": string; + "usage.status": string; + "usage.loading": string; + "usage.noNode": string; + "usage.error": string; "addsection.input.name": string; "addsection.select.": string; "addsection.titles.modal": string; diff --git a/modules/ui-module/src/main/resources/manager/js/modules/localization-actions.js b/modules/ui-module/src/main/resources/manager/js/modules/localization-actions.js index 51eceec86..9258194dd 100644 --- a/modules/ui-module/src/main/resources/manager/js/modules/localization-actions.js +++ b/modules/ui-module/src/main/resources/manager/js/modules/localization-actions.js @@ -21,6 +21,19 @@ export const ACTION_LOCALIZATIONS = { en: {}, de: { + "usage.title": "Verwendungen anzeigen", + "usage.scope": "Zeigt direkte redaktionelle Verweise. Templates, Themes und dynamische Queries werden nicht erfasst.", + "usage.incomplete": "Einige Quellen konnten nicht vollständig geprüft werden. Diese Liste kann unvollständig sein.", + "usage.problems": "Details", + "usage.empty": "Keine Verwendungen in redaktionellen Inhalten gefunden.", + "usage.count": "Verwendungen", + "usage.source": "Quelle", + "usage.site": "Site", + "usage.location": "Fundstelle", + "usage.status": "Status der Quelle", + "usage.loading": "Verwendungen werden geladen…", + "usage.noNode": "Kein Inhaltsknoten ausgewählt.", + "usage.error": "Verwendungen konnten nicht geladen werden.", "addsection.input.name": "Name des Absatzes", "addsection.select.": "Name of the section", "addsection.titles.modal": "Absatz hinuzfügen", diff --git a/modules/ui-module/src/main/ts/src/actions/page/show-usages.ts b/modules/ui-module/src/main/ts/src/actions/page/show-usages.ts new file mode 100644 index 000000000..b52a1f012 --- /dev/null +++ b/modules/ui-module/src/main/ts/src/actions/page/show-usages.ts @@ -0,0 +1,135 @@ +/*- + * #%L + * UI Module + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import { i18n } from '@cms/modules/localization.js'; +import { openModal } from '@cms/modules/modal.js'; +import { getPreviewUrl } from '@cms/modules/preview.utils.js'; +import { getActivePreviewContent } from '@cms/modules/preview-context.js'; +import { getContentNode } from '@cms/modules/rpc/rpc-content.js'; +import { executeRemoteCall } from '@cms/modules/rpc/rpc.js'; + +interface UsageResource { site: string; kind: string; path: string; } +interface Usage { + source: UsageResource; + sourceTitle: string; + sourceStatus: string; + location: string; + originalReference: string; +} +interface UsageResult { + items: Usage[]; + problems: { site: string; path: string; message: string }[]; +} + +const t = (key: string, fallback: string) => i18n.t(`usage.${key}`, fallback); +const appendText = (parent: HTMLElement, tag: string, text: string, className = '') => { + const element = document.createElement(tag); + element.textContent = text; + element.className = className; + parent.appendChild(element); + return element; +}; + +const render = (container: HTMLElement, path: string, result: UsageResult) => { + container.replaceChildren(); + appendText(container, 'p', path, 'fw-semibold text-break'); + appendText(container, 'p', t('scope', + 'Shows direct editorial references. Templates, themes and dynamic queries are not included.'), + 'text-body-secondary'); + if (result.problems.length) { + const warning = appendText(container, 'div', t('incomplete', + 'Some sources could not be fully checked. This list may be incomplete.'), 'alert alert-warning'); + warning.setAttribute('role', 'status'); + const details = document.createElement('details'); + appendText(details, 'summary', t('problems', 'Details')); + const list = document.createElement('ul'); + result.problems.forEach(problem => appendText(list, 'li', + `${problem.site} / ${problem.path}: ${problem.message}`, 'text-break')); + details.appendChild(list); + warning.appendChild(details); + } + if (!result.items.length) { + appendText(container, 'p', t('empty', 'No usages found in editorial content.'), 'mb-0'); + return; + } + appendText(container, 'p', `${t('count', 'Usages')}: ${result.items.length}`); + const wrapper = document.createElement('div'); + wrapper.className = 'table-responsive'; + const table = document.createElement('table'); + table.className = 'table table-striped align-middle'; + const headings = table.createTHead().insertRow(); + [t('source', 'Source'), t('site', 'Site'), t('location', 'Location'), t('status', 'Source status')] + .forEach(label => { + const th = appendText(headings, 'th', label); + th.setAttribute('scope', 'col'); + }); + const body = table.createTBody(); + result.items.forEach(usage => { + const row = body.insertRow(); + const source = row.insertCell(); + appendText(source, 'div', usage.sourceTitle || usage.source.path, 'fw-semibold text-break'); + appendText(source, 'small', usage.source.path, 'text-body-secondary text-break'); + row.insertCell().textContent = usage.source.site; + const location = row.insertCell(); + appendText(location, 'div', usage.location, 'text-break'); + appendText(location, 'small', usage.originalReference, 'text-body-secondary text-break'); + row.insertCell().textContent = usage.sourceStatus || '—'; + }); + wrapper.appendChild(table); + container.appendChild(wrapper); +}; + +export const runAction = () => { + // Capture the node before opening the dialog or waiting for a request. + const url = getPreviewUrl(); + const activeNode = getActivePreviewContent(url); + openModal({ + title: t('title', 'Show usages'), + size: 'lg', + showFooter: false, + body: '<div data-usage-content aria-live="polite"></div>', + onShow: async (modalElement: HTMLElement) => { + const container = modalElement.querySelector('[data-usage-content]') as HTMLElement; + appendText(container, 'p', t('loading', 'Loading usages…')); + try { + const node = activeNode || (url ? (await getContentNode({ url })).result : null); + if (!node?.uri) { + container.replaceChildren(); + appendText(container, 'p', t('noNode', 'No content node is selected.')); + return; + } + const response = await executeRemoteCall({ + method: 'usage.incoming', + parameters: { + kind: node.contentKind === 'collection' ? 'COLLECTION_ITEM' : 'CONTENT', + path: node.uri + } + }); + render(container, node.uri, response.result as UsageResult); + } catch (error) { + container.replaceChildren(); + const message = appendText(container, 'div', + t('error', 'Could not load usages.'), 'alert alert-danger'); + message.setAttribute('role', 'alert'); + } + } + }); +}; diff --git a/modules/ui-module/src/main/ts/src/js/modules/localization-actions.js b/modules/ui-module/src/main/ts/src/js/modules/localization-actions.js index 5ef29b9b4..f71edd53f 100644 --- a/modules/ui-module/src/main/ts/src/js/modules/localization-actions.js +++ b/modules/ui-module/src/main/ts/src/js/modules/localization-actions.js @@ -22,6 +22,19 @@ export const ACTION_LOCALIZATIONS = { en: { }, de: { + "usage.title": "Verwendungen anzeigen", + "usage.scope": "Zeigt direkte redaktionelle Verweise. Templates, Themes und dynamische Queries werden nicht erfasst.", + "usage.incomplete": "Einige Quellen konnten nicht vollständig geprüft werden. Diese Liste kann unvollständig sein.", + "usage.problems": "Details", + "usage.empty": "Keine Verwendungen in redaktionellen Inhalten gefunden.", + "usage.count": "Verwendungen", + "usage.source": "Quelle", + "usage.site": "Site", + "usage.location": "Fundstelle", + "usage.status": "Status der Quelle", + "usage.loading": "Verwendungen werden geladen…", + "usage.noNode": "Kein Inhaltsknoten ausgewählt.", + "usage.error": "Verwendungen konnten nicht geladen werden.", "addsection.input.name": "Name des Absatzes", "addsection.select.": "Name of the section", "addsection.titles.modal": "Absatz hinuzfügen", diff --git a/modules/ui-module/src/test/java/com/condation/cms/modules/ui/extensionpoints/remotemethods/RemoteUsageEndpointsTest.java b/modules/ui-module/src/test/java/com/condation/cms/modules/ui/extensionpoints/remotemethods/RemoteUsageEndpointsTest.java new file mode 100644 index 000000000..a05a86f55 --- /dev/null +++ b/modules/ui-module/src/test/java/com/condation/cms/modules/ui/extensionpoints/remotemethods/RemoteUsageEndpointsTest.java @@ -0,0 +1,91 @@ +package com.condation.cms.modules.ui.extensionpoints.remotemethods; + +/*- + * #%L + * UI Module + * %% + * Copyright (C) 2023 - 2026 CondationCMS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * #L% + */ + +import com.condation.cms.api.SiteProperties; +import com.condation.cms.api.feature.features.InjectorFeature; +import com.condation.cms.api.feature.features.SitePropertiesFeature; +import com.condation.cms.api.request.RequestContext; +import com.condation.cms.api.request.RequestContextScope; +import com.condation.cms.api.ui.rpc.RPCException; +import com.condation.cms.api.usage.*; +import com.google.inject.Guice; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.*; +import static org.mockito.Mockito.*; + +class RemoteUsageEndpointsTest { + @Test void incomingIncludesSiteCoverageProblemsAndUsesCurrentSiteIdentity() throws Exception { + var index = mock(UsageIndex.class); + var target = new UsageResource("de", UsageResource.Kind.MEDIA, "images/logo.svg"); + var usage = new Usage(new UsageResource("de", UsageResource.Kind.CONTENT, "index.md"), target, + "metadata.logo", Usage.Origin.CONTENT_TYPE, "images/logo.svg", "Home", "draft", Usage.TargetStatus.EXISTS); + when(index.incoming(target)).thenReturn(List.of(usage)); + var problem = new UsageProblem("de", "other.md", "Missing content type"); + when(index.problems()).thenReturn(List.of(problem)); + var context = context(index); + var endpoint = new RemoteUsageEndpoints(); + var result = ScopedValue.where(RequestContextScope.REQUEST_CONTEXT, context) + .call(() -> endpoint.incoming(Map.of("kind", "media", "path", "/images/logo.svg", "site", "ignored"))); + assertThat((Map<String, ?>) result).containsKey("items"); + assertThat(((Map<?, ?>) result).get("items")).isEqualTo(List.of(usage)); + assertThat(((Map<?, ?>) result).get("problems")).isEqualTo(List.of(problem)); + assertThat(((Map<?, ?>) result).get("scope")).isEqualTo("editorial"); + verify(index).incoming(target); + } + + @Test void rejectsInvalidKindsAndPathsEscapingSiteRoot() throws Exception { + var context = context(mock(UsageIndex.class)); + var endpoint = new RemoteUsageEndpoints(); + ScopedValue.where(RequestContextScope.REQUEST_CONTEXT, context).run(() -> { + assertThatThrownBy(() -> endpoint.outgoing(Map.of("kind", "content", "path", "../../outside.md"))) + .isInstanceOf(RPCException.class); + assertThatThrownBy(() -> endpoint.outgoing(Map.of("kind", "template", "path", "page.html"))) + .isInstanceOf(RPCException.class); + }); + } + + @Test void sharedCollectionUsesConfiguredOwner() throws Exception { + var index = mock(UsageIndex.class); + var context = context(index); + var configuration = mock(com.condation.cms.api.configuration.Configuration.class); + when(configuration.get(com.condation.cms.api.configuration.configs.CollectionConfiguration.class)) + .thenReturn(new com.condation.cms.api.configuration.configs.CollectionConfiguration(Map.of( + "news", new com.condation.cms.api.configuration.configs.CollectionDefinition("news", "en", null)))); + context.add(com.condation.cms.api.feature.features.ConfigurationFeature.class, + new com.condation.cms.api.feature.features.ConfigurationFeature(configuration)); + ScopedValue.where(RequestContextScope.REQUEST_CONTEXT, context).call(() -> + new RemoteUsageEndpoints().incoming(Map.of("kind", "COLLECTION_ITEM", "path", "news/example.md"))); + verify(index).incoming(new UsageResource("en", UsageResource.Kind.COLLECTION_ITEM, "news/example.md")); + } + + private RequestContext context(UsageIndex index) { + var properties = mock(SiteProperties.class); + when(properties.id()).thenReturn("de"); + var context = new RequestContext(); + context.add(SitePropertiesFeature.class, new SitePropertiesFeature(properties)); + context.add(InjectorFeature.class, new InjectorFeature(Guice.createInjector(binder -> binder.bind(UsageIndex.class).toInstance(index)))); + return context; + } +}