Thematic mapping: metadata enrichment, keyword cleanup, and area-label improvements

.gitignore

@@ -81,5 +81,13 @@ local_dev/tools/**/*.log
 /blob-report/
 /playwright/.cache/
 
+# inspection output
+e2e/reproducibility/output/*
+e2e/reproducibility/logs/*
+local_dev/tools/label_analysis
+local_dev/tools/keyword_analysis
+local_dev/tools/reproducibility
+local_dev/tools/dedup_review/*
+
 # mac os
 .DS_Store

docker-compose.yml

@@ -129,6 +129,17 @@ services:
       REDIS_PASSWORD: "${REDIS_PASSWORD}"
       LOGLEVEL: "${LOGLEVEL}"
       LOGFILE: "${LOGFILE}"
+      RANKING_MODE: "${RANKING_MODE}"
+      RANKING_MODE_BASE: "${RANKING_MODE_BASE}"
+      RANKING_MODE_PUBMED: "${RANKING_MODE_PUBMED}"
+      RANKING_MODE_ORCID: "${RANKING_MODE_ORCID}"
+      RANKING_MODE_OPENAIRE: "${RANKING_MODE_OPENAIRE}"
+      NGRAM_SETTING: "${NGRAM_SETTING}"
+      NGRAM_SETTING_BASE: "${NGRAM_SETTING_BASE}"
+      NGRAM_SETTING_PUBMED: "${NGRAM_SETTING_PUBMED}"
+      NGRAM_SETTING_ORCID: "${NGRAM_SETTING_ORCID}"
+      NGRAM_SETTING_OPENAIRE: "${NGRAM_SETTING_OPENAIRE}"
+      INCLUDE_ABSTRACTS: "${INCLUDE_ABSTRACTS}"
       RENV_VERSION: 0.14.0-5
       CRAN_REPOS: https://cran.wu.ac.at
       LC_ALL: "en_US.UTF-8"

e2e/basicTests/OpenAIRE.spec.ts

@@ -17,7 +17,7 @@ test.describe("Basic tests for OpenAIRE integration (Knowledge Map visualisation
       );
       await page.getByTestId("context").getByText("More information").click();
       await expect(page.locator("#info-body")).toContainText(
-        "This knowledge map presents you with a topical overview of research conducted in the following project:Project DetailsTitleOPENing UP new methods, indicators and tools for peer review, impact measurement and dissemination of research resultsAcronymOpenUPFunderECFunding programCSAH2020CallH2020-GARRI-2015-1Contract (GA) number710722Start Date2016-06-01End Date2018-11-30Open Access mandatetrueOrganizationsUGOE, UvA, PUBLIC POLICY AND MANAGEMENT INSTITUTE, FRONTIERS MEDIA  SA, UoA, DZHW, AIT, Know Center, CNROpenAire LinkLinkWe use text similarity to create a knowledge map. The algorithm groups those resources together that have many words in common. Area titles are created from subject keywords of resources that have been assigned to the same area. We select those keywords and phrases that appear frequently in one area, and seldom in other areas.Knowledge maps provide an instant overview of a topic by showing the main areas at a glance, and resources related to them. This makes it possible to easily identify useful, pertinent information.Please read our FAQs to find out more about knowledge maps.Data sourceThe data is taken from OpenAIRE. OpenAIRE is a key infrastructure that enables the European transition to open science. It provides access to 3 million research projects and more than 70 million research outputs from more than 100,000 data sources.Open source softwareThe visualization is created with the award winning open source software Head Start provided by Open Knowledge Maps. Open Knowledge Maps is a non-profit organisation run by a group of dedicated team members and volunteers. In order to improve our free and open service, we need your support. Please send us your feedback to info@openknowledgemaps.org.If you want to support us financially, you can:Make a donationBecome an organisational memberBecome a funder of our roadmapSign-up for our newsletter to receive occasional updates.",
+        "This knowledge map presents you with a topical overview of research conducted in the following project:Project DetailsTitleOPENing UP new methods, indicators and tools for peer review, impact measurement and dissemination of research resultsAcronymOpenUPFunderECFunding programCSAH2020CallH2020-GARRI-2015-1Contract (GA) number710722Start Date2016-06-01End Date2018-11-30Open Access mandatetrueOrganizationsKnow Center, UvA, UOA (ΕΚΠΑ), DZHW, CNR, AIT, PPMI, UGOE, FRONTIERS MEDIA SAOpenAire LinkLinkWe use text similarity to create a knowledge map. The algorithm groups those resources together that have many words in common. Area titles are created from subject keywords of resources that have been assigned to the same area. We select those keywords and phrases that appear frequently in one area, and seldom in other areas.Knowledge maps provide an instant overview of a topic by showing the main areas at a glance, and resources related to them. This makes it possible to easily identify useful, pertinent information.Please read our FAQs to find out more about knowledge maps.Data sourceThe data is taken from OpenAIRE. OpenAIRE is a key infrastructure that enables the European transition to open science. It provides access to 3 million research projects and more than 70 million research outputs from more than 100,000 data sources.Open source softwareThe visualization is created with the award winning open source software Head Start provided by Open Knowledge Maps. Open Knowledge Maps is a non-profit organisation run by a group of dedicated team members and volunteers. In order to improve our free and open service, we need your support. Please send us your feedback to info@openknowledgemaps.org.If you want to support us financially, you can:Make a donationBecome an organisational memberBecome a funder of our roadmapSign-up for our newsletter to receive occasional updates.",
       );
     });
   });

e2e/keywordsCleaning/BASE.spec.ts

@@ -0,0 +1,82 @@
+import { test, expect } from "@playwright/test";
+import { prepareVisualisation } from "../../vis/js/utils/e2eVisualisationLoader";
+import { assertNoClassificationLeaks, extractDocuments } from "./invariant";
+
+// Classification-leak invariant for BASE searches (mirrors ORCID.spec.ts). The
+// queries are chosen to surface the schemes/forms the cleaner targets:
+//   * MeSH descriptors, qualifiers and space-joined blobs (biomedical queries:
+//     "pharmacology", "therapeutic use", "chemically induced", ...);
+//   * FOS / arXiv / ddc / LCC classifications (discipline queries: "game theory",
+//     "machine translation", ...).
+// document_types=121 (articles) + min_descsize=300 reproduces the records the
+// cleaning was validated against. See ./invariant.ts for the marker set.
+
+interface BaseSearch {
+  query: string;
+  search_params: string;
+}
+
+// most-relevant articles, min description size 300 (the records the cleaning was
+// validated against).
+//
+// NOTE: `type=get` returns the cached revision for a given (query, params, date)
+// hash. So after a cleaning change, re-running the same search returns the stale
+// pre-change map. To force a fresh recomputation against the current backend,
+// vary a date param, e.g. append "&from=1665-02-05" (any unused value). A clean
+// CI database computes fresh on first run, so the default below is fine there.
+const ARTICLES =
+  "&service=base&sorting=most-relevant&document_types%5B%5D=121&lang_id%5B%5D=all-lang&min_descsize=300";
+
+const searches: BaseSearch[] = [
+  { query: "pharmacology", search_params: ARTICLES },
+  { query: "therapeutic use", search_params: ARTICLES },
+  { query: "chemically induced", search_params: ARTICLES },
+  { query: "antagonists & inhibitors", search_params: ARTICLES },
+  { query: "enzyme inhibitors", search_params: ARTICLES },
+  { query: '"game theory"', search_params: ARTICLES },
+  { query: "machine translation", search_params: ARTICLES },
+  { query: "physics", search_params: ARTICLES },
+  { query: "medicine", search_params: ARTICLES },
+];
+
+// Readiness check only (the assertions under test are the leak invariants). The
+// heading currently renders "&" as a literal "&" (double HTML escaping in the
+// title component), so the expected display text mirrors that until the display
+// bug is fixed; quotes are stripped by the UI.
+function displayedQuery(query: string): string {
+  return query.replace(/"/g, "").replace(/&/g, "&");
+}
+
+// Pre-load the BASE searches so subsequent runs are faster.
+// npx playwright test "e2e/keywordsCleaning/BASE.spec.ts" --grep "Warm-up: pre-load BASE searches"
+test.describe("Warm-up: pre-load BASE searches", () => {
+  for (const { query, search_params } of searches) {
+    test(`${query}`, async ({ page }) => {
+      const url = `/search?type=get&vis_type=overview&q=${encodeURIComponent(query)}${search_params}`;
+      await prepareVisualisation(page, url);
+      await expect(page.locator("#search-term-unique")).toContainText(displayedQuery(query));
+    });
+  }
+});
+
+test.describe("No classification markers leak into subject or area titles", () => {
+  for (const { query, search_params } of searches) {
+    test(`subject + area titles are clean — ${query}`, async ({ page }) => {
+      const url = `/search?type=get&vis_type=overview&q=${encodeURIComponent(query)}${search_params}`;
+
+      // Register before navigation so we capture the data load that populates
+      // the visualisation. The backend pipeline can take minutes, so match the
+      // global test timeout.
+      const responsePromise = page.waitForResponse(
+        (r) => r.url().includes("getLatestRevision.php") && r.ok(),
+        { timeout: 5 * 60 * 1000 },
+      );
+
+      await prepareVisualisation(page, url);
+      await expect(page.locator("#search-term-unique")).toContainText(displayedQuery(query));
+
+      const json = await (await responsePromise).json();
+      assertNoClassificationLeaks(extractDocuments(json), query);
+    });
+  }
+});

e2e/keywordsCleaning/ORCID.spec.ts

@@ -0,0 +1,67 @@
+import { test, expect } from "@playwright/test";
+import { prepareVisualisation } from "../../vis/js/utils/e2eVisualisationLoader";
+import { assertNoClassificationLeaks, extractDocuments } from "./invariant";
+
+// const uniqueOrcids = [...new Set(testCases.map((tc) => tc.orcid).filter(Boolean))];
+
+const additionalUniqueOrcids = [
+  "0000-0001-6011-4382",
+  "0000-0003-0204-881X",
+  "0000-0001-5116-955X",
+  "0000-0003-2897-6075",
+  "0000-0002-8911-7832",
+  "0000-0002-9843-6798",
+  "0000-0003-4221-6275",
+  "0000-0002-4505-0517",
+  "0000-0002-2233-6926",
+  "0000-0002-2441-4043",
+  "0000-0001-9287-3770",
+  "0000-0001-9612-7791",
+  "0000-0002-4971-2944",
+  "0000-0001-9062-6039",
+  "0000-0002-3924-6636",
+  "0000-0001-5849-8137",
+  "0000-0003-0297-9614",
+];
+
+const allUniqueOrcids = [...new Set([ ...additionalUniqueOrcids])];
+
+
+// This can also be used to pre-load the unique ORCID profiles, so that the subsequent tests run faster
+// npx playwright test "e2e/keywordsCleaning/ORCID.spec.ts" --grep "Warm-up: pre-load unique ORCID profiles"
+test.describe("Warm-up: pre-load unique ORCID profiles", () => {
+  for (const orcid of allUniqueOrcids) {
+    test(`${orcid}`, async ({ page }) => {
+      const url = `/search?type=get&vis_type=overview&orcid=${orcid}&service=orcid&embed=true&academic_age_offset=1`;
+      await prepareVisualisation(page, url);
+      await expect(page.locator("#search-term-unique")).toContainText(`(${orcid})`);
+    });
+  }
+});
+
+// Classification-leak invariant: no classification marker or MeSH qualifier may
+// survive in the cleaned `subject` or in an area title. See ./invariant.ts for
+// the rationale and the marker set.
+test.describe("No classification markers leak into subject or area titles", () => {
+  for (const orcid of allUniqueOrcids) {
+    test(`subject + area titles are clean — ${orcid}`, async ({ page }) => {
+      const url = `/search?type=get&vis_type=overview&orcid=${orcid}&service=orcid&embed=true&academic_age_offset=1`;
+
+      // Register before navigation so we capture the data load that populates
+      // the visualisation. The backend pipeline can take minutes, so match the
+      // global test timeout.
+      const responsePromise = page.waitForResponse(
+        (r) => r.url().includes("getLatestRevision.php") && r.ok(),
+        { timeout: 5 * 60 * 1000 },
+      );
+
+      await prepareVisualisation(page, url);
+      await expect(page.locator("#search-term-unique")).toContainText(`(${orcid})`);
+
+      const json = await (await responsePromise).json();
+      // A loaded profile should carry documents; if it is genuinely empty there
+      // is nothing to clean, which trivially satisfies the invariant.
+      assertNoClassificationLeaks(extractDocuments(json), orcid);
+    });
+  }
+});

e2e/keywordsCleaning/invariant.ts

@@ -0,0 +1,169 @@
+import { expect } from "@playwright/test";
+
+// Shared classification-leak invariant for the keyword-cleaning e2e tests
+// (ORCID + BASE). The cleaning step removes classification-scheme keywords and
+// MeSH subheading qualifiers from the display `subject` field, and that cleaned
+// field also feeds the cluster/area titles (summarize.R derives labels from
+// metadata$subject). These are *invariant* checks, not exact-output ones: the
+// live BASE API returns different records run-to-run, so we cannot pin the
+// cleaned string. Instead we assert that no recognised classification marker or
+// MeSH qualifier survives anywhere in `subject` or in an area title, whatever
+// BASE happens to return. Exact input -> output is covered deterministically by
+// the R unit tests (other-scripts/test/test_subject_cleaning.R).
+//
+// Vocabulary-list schemes that carry no inline marker (e.g. the Toulouse
+// letter-domain list) cannot be detected by pattern without false positives, so
+// they are out of scope here and remain covered by the unit tests only.
+//
+// Deliberately NOT asserted: the "anzsrc-for: 3402 ..." form. In the current
+// pipeline an earlier generic gsub strips its "for: ..." tail and leaves an
+// orphan "anzsrc-" fragment, so neither the full marker nor a clean drop holds;
+// the fix requires refactoring the existing cleanup, so it is left out rather
+// than encoding a failing or misleading assertion.
+
+// Classification-scheme markers. Keep in sync with kwlib.SCHEMES / the cleaner.
+// Each pattern has been checked against real cleaned `subject` data: it is absent
+// after cleaning while the scheme is present in the source `subject_orig`.
+export const CLASSIFICATION_MARKERS: { name: string; re: RegExp }[] = [
+  { name: "mesh [MeSH]", re: /\[MeSH\]/i },
+  { name: "mesh (mesh)", re: /\(mesh\)/i },
+  { name: "mesh-chemical", re: /\[Chemical\]/i },
+  { name: "rcdc", re: /\(rcdc\)/i },
+  { name: "for / for-2020", re: /\(for(-2020)?\)/i },
+  { name: "science-metrix", re: /\(science-metrix\)/i },
+  { name: "sdg (suffix marker)", re: /\(sdg\)/i },
+  { name: "sdg (numbered prefix)", re: /^SDG ?\d+ ?[:.-]/i },
+  { name: "hrcs", re: /\(hrcs-[a-z]+\)/i },
+  { name: "acm-ccs arrow", re: /→/ },
+  { name: "keyvalue name=", re: /^name=/i },
+  {
+    name: "hal-shs domain code",
+    re: /^\[(CHIM|INFO|MATH|NLIN|PHYS|SCCO|SDE|SDU|SDV|SHS|SPI|STAT|QFIN)(\.[A-Z-]+)*\]/,
+  },
+  { name: "ddc", re: /\bddc:\s*\d/i },
+  { name: "info:eu-repo", re: /info:eu-repo/i },
+  { name: "pure ontology path", re: /\/dk\/atira/i },
+  { name: "fos", re: /^FOS:\s/i },
+  {
+    name: "arxiv category (dotted)",
+    re: /\b(cs|econ|eess|math|astro-ph|nlin|q-bio|q-fin|stat)\.[A-Z]{2}\b/,
+  },
+  {
+    name: "arxiv category (bare)",
+    re: /\b(quant-ph|gr-qc|math-ph|cond-mat|hep-(ex|lat|ph|th)|nucl-(ex|th))\b/,
+  },
+  { name: "url", re: /^https?:\/\// },
+  { name: "not elsewhere classified", re: /not elsewhere classified/i },
+  { name: "numeric path", re: /^\/[0-9/]+$/ },
+  // Funder grant / scheme reference, e.g. "SP/19/3/34678", "MR/S003991/1".
+  // Conservative regression marker: requires 3+ slash-separated alphanumeric
+  // segments with both a letter and a digit, so 1-slash MeSH/gene forms
+  // ("COVID-19/epidemiology", "HER-2/neu") and all-digit dates are not matched.
+  {
+    name: "grant id (slash code)",
+    re: /^(?=.*[A-Za-z])(?=.*\d)[A-Za-z0-9][A-Za-z0-9-]*(\/[A-Za-z0-9][A-Za-z0-9-]*){2,}$/,
+  },
+];
+
+// MeSH subheadings/qualifiers (mirrors MESH_QUALIFIERS in subject_cleaning.R).
+// A leak is a "Descriptor/qualifier" or "Descriptor - qualifier" pair, or a
+// space-joined blob of headings, surviving into the cleaned subject.
+const MESH_QUALIFIERS = [
+  "analysis", "blood", "cerebrospinal fluid", "isolation & purification", "urine",
+  "anatomy & histology", "blood supply", "cytology", "ultrastructure", "embryology",
+  "abnormalities", "innervation", "pathology", "chemistry", "agonists",
+  "analogs & derivatives", "antagonists & inhibitors", "chemical synthesis",
+  "diagnosis", "diagnostic imaging", "etiology", "chemically induced", "complications",
+  "secondary", "congenital", "genetics", "immunology", "microbiology", "virology",
+  "parasitology", "transmission", "organization & administration", "economics",
+  "legislation & jurisprudence", "standards", "supply & distribution", "trends",
+  "pharmacology", "adverse effects", "poisoning", "toxicity", "pharmacokinetics",
+  "physiology", "growth & development", "metabolism", "biosynthesis", "deficiency",
+  "enzymology", "physiopathology", "statistics & numerical data", "epidemiology",
+  "ethnology", "mortality", "therapeutic use", "therapy", "diet therapy",
+  "drug therapy", "nursing", "prevention & control", "radiotherapy", "rehabilitation",
+  "surgery", "transplantation", "classification", "drug effects", "education",
+  "ethics", "history", "injuries", "instrumentation", "methods", "pathogenicity",
+  "psychology", "radiation effects", "veterinary",
+];
+const QUAL = MESH_QUALIFIERS.slice()
+  .sort((a, b) => b.length - a.length)
+  .map((q) => q.replace(/[&]/g, "\\$&"))
+  .join("|");
+
+export const QUALIFIER_MARKERS: { name: string; re: RegExp }[] = [
+  { name: "mesh qualifier (slash)", re: new RegExp(`/\\s*\\*?\\s*(?:${QUAL})\\*?\\s*$`, "i") },
+  { name: "mesh qualifier (dash)", re: new RegExp(`\\s-\\s\\*?(?:${QUAL})\\*?\\s*$`, "i") },
+  { name: "mesh blob (space-joined headings)", re: /\S+\/\S+\s+\S+\/\S+/ },
+];
+
+// Library of Congress Classification — clear-cut forms only, as regression
+// markers. Deliberately conservative: each pattern matches a form that is
+// unambiguously a classification, so an ambiguous true-negative (a non-LCC
+// keyword that the cleaner legitimately keeps) does NOT trip it. We do not assert
+// the bare-code / code+caption forms here, as those are intentionally ambiguous.
+export const LCC_MARKERS: { name: string; re: RegExp }[] = [
+  // Numeric class-number range, e.g. "QC1-999", "QH301-705.5", "HF5001-6182".
+  // Both sides are pure digits, so MRI sequences ("T1-T2") and cell markers
+  // ("CD4-CD8") — which have a letter after the dash — are not matched.
+  { name: "lcc range", re: /^[A-Z]{1,3}\d{1,4}(\.\d+)?-\d{1,4}(\.\d+)?$/ },
+];
+
+const ALL_MARKERS = [...CLASSIFICATION_MARKERS, ...QUALIFIER_MARKERS, ...LCC_MARKERS];
+
+export interface BackendDocument {
+  subject?: string;
+  area?: string;
+}
+
+// getLatestRevision.php returns { context, data }, where data (a string that
+// needs parsing) carries documents[] (itself sometimes a nested JSON string).
+// Each document has the cleaned `subject` and the `area` (cluster) title.
+export function extractDocuments(json: any): BackendDocument[] {
+  let data = json?.data;
+  if (typeof data === "string") {
+    data = JSON.parse(data);
+  }
+  let documents = data?.documents ?? [];
+  if (typeof documents === "string") {
+    documents = JSON.parse(documents);
+  }
+  return Array.isArray(documents) ? documents : [];
+}
+
+export function splitKeywords(subject: string | undefined): string[] {
+  return String(subject ?? "")
+    .split(";")
+    .map((s) => s.trim())
+    .filter(Boolean);
+}
+
+// Assert that no classification marker or MeSH qualifier survives in any
+// `subject` keyword or in any area title. `label` identifies the search (ORCID /
+// query) in the failure message. An empty document set trivially passes.
+export function assertNoClassificationLeaks(
+  documents: BackendDocument[],
+  label: string,
+): void {
+  const keywords = documents.flatMap((d) => splitKeywords(d.subject));
+  for (const kw of keywords) {
+    for (const { name, re } of ALL_MARKERS) {
+      expect(
+        re.test(kw),
+        `subject keyword "${kw}" (${label}) still carries the ${name} marker`,
+      ).toBe(false);
+    }
+  }
+
+  const areaTitles = [
+    ...new Set(documents.map((d) => d.area).filter(Boolean) as string[]),
+  ];
+  for (const title of areaTitles) {
+    for (const { name, re } of ALL_MARKERS) {
+      expect(
+        re.test(title),
+        `area title "${title}" (${label}) still carries the ${name} marker`,
+      ).toBe(false);
+    }
+  }
+}