From 7ed3f24025a953ec461e36f1c6008190208736ec Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Tue, 3 Nov 2026 16:03:13 +0400 Subject: [PATCH] Cleanups - Publishing holds until a real AI provider is configured. The mock provider runs in tests only. - Fact checks run before a post or update goes live. - Entity matching ignores sentence openers and needs two shared facts to merge. - Embedding fallbacks are reported. - Dedup uses stored embeddings and the configured threshold. - Updates add only new facts. - IndexNow calls are checked. The keyword provider filter is wired in. - API keys use sodium and the Gemini key moves to a header. - README matches the WP-Cron queue. - Removes em and en dashes and uppercase labels. --- README.md | 28 ++-- aggregate-it.php | 4 +- assets/css/admin.css | 2 - assets/css/hub.css | 2 +- assets/js/admin.js | 2 +- docs/ARCHITECTURE.md | 60 ++++---- docs/CONVENTIONS.md | 24 ++-- docs/DECISIONS.md | 66 ++++----- docs/EXTENDING.md | 38 ++--- docs/PLAN-custom-sources-activity-hubs.md | 84 +++++------ docs/PLAN.md | 58 ++++---- src/Admin/Admin.php | 112 ++++++++++++--- src/Admin/HubColumns.php | 2 +- src/Admin/PostMetaBox.php | 4 +- src/Admin/RestController.php | 4 +- src/Admin/views/articles.php | 10 +- src/Admin/views/dashboard.php | 4 +- src/Admin/views/settings.php | 8 +- src/Admin/views/sources.php | 14 +- src/Ai/AnthropicProvider.php | 6 +- src/Ai/FactsGuard.php | 38 ++++- src/Ai/GeminiProvider.php | 22 ++- src/Ai/HashEmbedding.php | 8 +- src/Ai/MockProvider.php | 6 +- src/Ai/OpenAiProvider.php | 2 +- src/Ai/ProviderFactory.php | 16 +++ src/Ai/ProviderNotConfigured.php | 11 ++ src/Ai/Rewriter.php | 43 +++++- src/Cluster/Clusterer.php | 11 +- src/Cluster/Deduplicator.php | 162 +++++++++++++--------- src/Entity/DelegationRules.php | 2 +- src/Entity/EntityLinker.php | 2 +- src/Entity/EntityRepository.php | 2 +- src/Entity/EntityResearcher.php | 4 +- src/Entity/EntityResolver.php | 4 +- src/Entity/HubRenderer.php | 2 +- src/Keyword/KeywordStrategy.php | 35 ++++- src/Maintenance/GlobalRulesRefresher.php | 2 +- src/Maintenance/Retention.php | 2 +- src/Pipeline/ComposeStage.php | 54 ++++++-- src/Pipeline/EntityStage.php | 2 +- src/Pipeline/ExtractStage.php | 2 +- src/Pipeline/PaidStage.php | 2 +- src/Pipeline/Stage.php | 2 +- src/Plugin.php | 6 +- src/Publish/CategoryResolver.php | 2 +- src/Publish/FieldMapper.php | 4 +- src/Publish/ImageImporter.php | 4 +- src/Publish/PostFactory.php | 30 +++- src/Publish/RelatedArticles.php | 2 +- src/Publish/Reprocessor.php | 15 +- src/Queue/ItemStore.php | 4 +- src/Queue/QueueWorker.php | 8 +- src/Research/ResearchProvider.php | 2 +- src/Seo/IndexNow.php | 20 ++- src/Seo/SlugGenerator.php | 2 +- src/Settings.php | 23 ++- src/Source/ContentExtractor.php | 10 +- src/Source/HttpFetcher.php | 2 +- src/Source/Importer.php | 4 +- src/Source/Parser/ScraperParser.php | 4 +- src/Source/RateLimited.php | 2 +- src/Source/Scrape/SelectorAssistant.php | 2 +- src/Source/Source.php | 2 +- src/Support/ActivityLog.php | 2 +- src/Support/Crypto.php | 81 +++++++++-- src/Support/Vector.php | 7 +- src/Vector/VectorStore.php | 37 ++++- tests/BootSmokeTest.php | 2 +- tests/CryptoTest.php | 37 +++++ tests/DeduplicatorTest.php | 64 ++++----- tests/FactsGuardTest.php | 9 ++ tests/KeywordStrategyTest.php | 36 +++++ tests/VectorTest.php | 4 + tests/bootstrap.php | 1 + 75 files changed, 952 insertions(+), 444 deletions(-) create mode 100644 src/Ai/ProviderNotConfigured.php create mode 100644 tests/CryptoTest.php diff --git a/README.md b/README.md index 41fa7c7..742163a 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ **An SEO content engine that uses RSS as raw material.** Aggregate It imports RSS/Atom feeds, faithfully rewrites the content with AI (no -fact changes — news stays news), deduplicates stories semantically, and maintains +fact changes, news stays news), deduplicates stories semantically, and maintains *living* topic pages that stay up to date as new sources arrive. It automatically extracts entities (companies, people, products…), resolves them against your custom post types, creates researched hub pages, and builds an internal-link graph @@ -15,30 +15,30 @@ fuel happens to be RSS. - **License:** GPL-2.0-or-later - **Distribution:** GitHub releases + Plugin Update Checker self-update (matches `crm-connect`); WordPress.org listing optional later. Freemium add-ons via provider seams. - **Slug:** `aggregate-it` -- **Conventions:** follows the `crm-connect` house style — see [`docs/CONVENTIONS.md`](docs/CONVENTIONS.md) -- **Status:** planning / pre-implementation +- **Conventions:** follows the `crm-connect` house style, see [`docs/CONVENTIONS.md`](docs/CONVENTIONS.md) +- **Status:** implemented (phases 0-4 built, see [`docs/PLAN.md`](docs/PLAN.md)) ## What makes it different | Capability | Why it matters for SEO | |---|---| -| **Living canonical posts** — one authoritative URL per story, updated over time | Concentrates link equity + sends `dateModified` freshness signals instead of splitting equity across duplicates | -| **Faithful rewrite** — reword + strip promo junk, never alter facts | Original prose without the hallucination/legal risk; a documented selling point | -| **Novelty gate** — only update when genuinely new facts arrive | Avoids the thin-rewrite pattern that triggers Google's Scaled Content Abuse policy | -| **Entity hub engine** — auto-match/create CPT entities + contextual backlinks | Builds hub-and-spoke topical authority + machine-readable `about`/`sameAs` schema | -| **Native-rich schema** — Article/NewsArticle + entity + citation graph | The durable moat thin-rewrite competitors can't replicate | +| **Living canonical posts**: one authoritative URL per story, updated over time | Concentrates link equity + sends `dateModified` freshness signals instead of splitting equity across duplicates | +| **Faithful rewrite**: reword + strip promo junk, never alter facts | Original prose without the hallucination/legal risk; a documented selling point | +| **Novelty gate**: only update when genuinely new facts arrive | Avoids the thin-rewrite pattern that triggers Google's Scaled Content Abuse policy | +| **Entity hub engine**: auto-match/create CPT entities + contextual backlinks | Builds hub-and-spoke topical authority + machine-readable `about`/`sameAs` schema | +| **Native-rich schema**: Article/NewsArticle + entity + citation graph | The durable moat thin-rewrite competitors can't replicate | ## Documentation -- [`docs/DECISIONS.md`](docs/DECISIONS.md) — the resolved design decisions and their rationale -- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) — data model, pipeline, subsystems, config schema, extension API -- [`docs/CONVENTIONS.md`](docs/CONVENTIONS.md) — engineering house style, ported from `crm-connect` -- [`docs/EXTENDING.md`](docs/EXTENDING.md) — provider/hook seams for premium add-ons -- [`docs/PLAN.md`](docs/PLAN.md) — phased implementation plan (phases 0–4 built) +- [`docs/DECISIONS.md`](docs/DECISIONS.md): the resolved design decisions and their rationale +- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md): data model, pipeline, subsystems, config schema, extension API +- [`docs/CONVENTIONS.md`](docs/CONVENTIONS.md): engineering house style, ported from `crm-connect` +- [`docs/EXTENDING.md`](docs/EXTENDING.md): provider/hook seams for premium add-ons +- [`docs/PLAN.md`](docs/PLAN.md): phased implementation plan (phases 0-4 built) ## Operating profile (chosen constraints) -- **Scale:** single site, moderate volume — Action Scheduler, no external worker +- **Scale:** single site, moderate volume. A claim-based database queue driven by WP-Cron, no external worker - **Editorial:** fully automatic publish; manual edits after the fact - **AI:** cheapest-viable models behind a swappable, bring-your-own-key adapter - **Content stance:** full faithful rewrite, facts preserved, promo stripped diff --git a/aggregate-it.php b/aggregate-it.php index 85d913f..b74c64d 100644 --- a/aggregate-it.php +++ b/aggregate-it.php @@ -2,12 +2,12 @@ /** * Plugin Name: Aggregate It * Plugin URI: https://github.com/nmbrthirteen/aggregate-it - * Description: An SEO content engine that uses RSS as raw material — it imports feeds, faithfully rewrites them with AI, deduplicates stories into living topic pages, and auto-builds entity hubs with an internal-link graph. + * Description: An SEO content engine that uses RSS as raw material. It imports feeds, faithfully rewrites them with AI, deduplicates stories into living topic pages, and auto-builds entity hubs with an internal-link graph. * * Features: * - Import RSS/Atom and JSON feeds on a per-feed schedule, with categories, tags, include/exclude keywords, and per-feed length and publish overrides. * - Bulk-add feeds, OPML import (paste or upload), and a global keyword/domain blacklist. - * - Faithful AI rewrite that keeps the facts and strips promo — choose Google Gemini, OpenAI, or Claude with your own key. + * - Faithful AI rewrite that keeps the facts and strips promo, choose Google Gemini, OpenAI, or Claude with your own key. * - Model, article-length, and writing-style controls, a daily spend cap, and a one-click connection test. * - Semantic deduplication into one canonical post per story, a novelty gate, and dated "In the news" timelines. * - Automatic entity hub pages with schema.org types, sameAs links, optional Wikipedia enrichment, an internal-link graph, and related articles. diff --git a/assets/css/admin.css b/assets/css/admin.css index 3ee6b07..020c091 100644 --- a/assets/css/admin.css +++ b/assets/css/admin.css @@ -438,8 +438,6 @@ display: block; color: #646970; font-size: 11px; - text-transform: uppercase; - letter-spacing: .02em; } .aggregate-it .ai-hub-feed-link { diff --git a/assets/css/hub.css b/assets/css/hub.css index 4cff221..f51ac43 100644 --- a/assets/css/hub.css +++ b/assets/css/hub.css @@ -1,4 +1,4 @@ -/* Aggregate It — entity hub front-end styles. Neutral + theme-agnostic. */ +/* Aggregate It entity hub front-end styles. Neutral + theme-agnostic. */ .aggregate-it-details { border-collapse: collapse; width: 100%; diff --git a/assets/js/admin.js b/assets/js/admin.js index 4de7440..92957ce 100644 --- a/assets/js/admin.js +++ b/assets/js/admin.js @@ -143,7 +143,7 @@ } tbody.innerHTML = ''; if ( ! rows.length ) { - tbody.innerHTML = 'No articles yet — add a feed to get started.'; + tbody.innerHTML = 'No articles yet. Add a feed to get started.'; return; } rows.forEach( function ( r ) { diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b9be356..8f9230d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -3,7 +3,7 @@ How Aggregate It is put together: the pipeline, the data model, the subsystems, the configuration schema, and the extension API that the freemium seams depend on. -Read [`DECISIONS.md`](DECISIONS.md) first — this document assumes those choices. +Read [`DECISIONS.md`](DECISIONS.md) first; this document assumes those choices. --- @@ -32,7 +32,7 @@ Read [`DECISIONS.md`](DECISIONS.md) first — this document assumes those choice Every stage is a job on the custom claim-based queue (the crm-connect pattern). ``` -The plugin is the runtime *and* the CMS — no external worker. A **custom claim-based DB +The plugin is the runtime *and* the CMS: no external worker. A **custom claim-based DB queue** (see [`CONVENTIONS.md`](CONVENTIONS.md) §5) drives every stage: atomic `claim_token` claiming, stale-claim recovery, exponential backoff, dead-letter, auto-pause, and a non-blocking "nudge" for low latency. The only outbound calls are: AI @@ -42,7 +42,7 @@ provider, optional research/keyword providers, source-site fetches, and IndexNow ## 2. The pipeline (item state machine) -Each row in `ai_items` advances through explicit, **idempotent, resumable** states — +Each row in `ai_items` advances through explicit, **idempotent, resumable** states, **one stage per queue claim**, never the whole pipeline in a single request. A monolithic "import + rewrite" call is the anti-pattern we are avoiding. @@ -56,14 +56,14 @@ monolithic "import + rewrite" call is the anti-pattern we are avoiding. | ↳ `updated` | Processing (matched cluster) | Novelty gate; if new facts → append dated section, bump `dateModified` | cheap | | `entity_linked` | Entity | Resolve/create entities, inject first-mention links, write relationships | optional paid | | `published` | Publish | Slug, meta via adapter, JSON-LD, IndexNow ping | free | -| `failed` | — | Retried w/ exponential backoff; permanent failures → dead-letter + log | — | +| `failed` | - | Retried w/ exponential backoff; permanent failures → dead-letter + log | - | **Idempotency:** every stage keys off `content_hash`; re-running a stage is safe. **Re-runs:** a post can be re-rewritten (new prompt/model) without re-importing. **Pause:** when the daily spend cap trips, the queue's transient auto-pause halts paid stages (mirrors crm-connect's auto-pause); free stages continue. **Batch API:** the rewrite stage may submit a batch job and set `next_attempt_at` to a -poll time — the claim loop polls until ready, then advances. No long blocking calls. +poll time; the claim loop polls until ready, then advances. No long blocking calls. --- @@ -172,7 +172,7 @@ CREATE TABLE {prefix}_ai_log ( At single-site moderate volume the live vector set is thousands, not millions. Cosine over a few thousand packed float32 rows in PHP is sub-millisecond-per-compare and runs inside a queue job, not a web request. External vector DBs -(Pinecone/Qdrant) are explicitly avoided — they'd break "activate and it just works" +(Pinecone/Qdrant) are explicitly avoided; they'd break "activate and it just works" and the OSS no-paid-dependency rule. A `pgvector`/scale path can be added behind the same `VectorStore` interface later. @@ -184,7 +184,7 @@ same `VectorStore` interface later. - Feed parsing via WordPress core `SimplePie` (already bundled) for RSS/Atom; JSON Feed handled separately. -- **Full-content extraction:** `ContentExtractor` — feed-content-if-full-enough, else +- **Full-content extraction:** `ContentExtractor`: feed-content-if-full-enough, else fetch + GPL-compatible readability port (no JS). `HttpFetcher` enforces `robots.txt`, per-domain rate limit, honest UA, response caching. - Syntactic dedup by `(source_id, guid)` unique key + `content_hash`. @@ -192,43 +192,43 @@ same `VectorStore` interface later. ### 4.2 Processing - `Embedder` (provider adapter) → `ai_vectors`. -- `Clusterer` — 3-signal: cosine ≥ threshold **∧** shared primary entity **∧** within +- `Clusterer`: 3-signal: cosine ≥ threshold **∧** shared primary entity **∧** within `window_until`. Borderline (near threshold) → log, treat as new cluster. -- `Rewriter` — the **single structured call**. Strict JSON schema output. Prompt is +- `Rewriter`: the **single structured call**. Strict JSON schema output. Prompt is template-driven (per-feed overridable), enforces faithful-rewrite + strip-promo + keyword-natural rules. -- `FactsGuard` — extract numbers/dates/proper-nouns from input & output; any in output +- `FactsGuard`: extract numbers/dates/proper-nouns from input & output; any in output not in input → flag on the item (publishes, but visible in the dashboard). -- `NoveltyGate` — diff new item's facts against `cluster.fact_set`; no new facts → +- `NoveltyGate`: diff new item's facts against `cluster.fact_set`; no new facts → suppress; new facts → append + merge into `fact_set`. ### 4.3 Entity engine -- `EntityExtractor` — comes from the rewrite call's `entities[]` (no separate NER call). -- `EntityResolver` — normalize → exact/alias → type-constrained fuzzy+embedding → +- `EntityExtractor`: comes from the rewrite call's `entities[]` (no separate NER call). +- `EntityResolver`: normalize → exact/alias → type-constrained fuzzy+embedding → confidence band (link / create / skip-and-log). Batch-dedup guard via a per-run pending-creation set. -- `EntityResearcher` — pluggable. Default: stub from in-article context (zero deps). +- `EntityResearcher`: pluggable. Default: stub from in-article context (zero deps). With a research provider: cited external facts + `sameAs`. -- `EntityLinker` — first-mention contextual link, capped per post, writes the +- `EntityLinker`: first-mention contextual link, capped per post, writes the relationship taxonomy. -- `HubRenderer` — entity CPT template: description + cited facts + `sameAs` + +- `HubRenderer`: entity CPT template: description + cited facts + `sameAs` + auto-growing related-posts list + Organization/Person/Product schema. -- `EntityMerge` — admin tool: merge B→A, repoint links/aliases/relationships/schema. +- `EntityMerge`: admin tool: merge B→A, repoint links/aliases/relationships/schema. ### 4.4 Publish / SEO -- `SlugGenerator` — from locked primary keyword. -- `SeoAdapter` — strategy per active plugin (Yoast / Rank Math / SEOPress / native +- `SlugGenerator`: from locked primary keyword. +- `SeoAdapter`: strategy per active plugin (Yoast / Rank Math / SEOPress / native fallback): writes title + meta + focus keyword; **suppresses the plugin's schema** on our objects. -- `SchemaGraph` — builds the native JSON-LD `@graph`. -- `IndexNow` — pings on publish and on novelty update; bumps `dateModified`. +- `SchemaGraph`: builds the native JSON-LD `@graph`. +- `IndexNow`: pings on publish and on novelty update; bumps `dateModified`. ### 4.5 Cost & observability -- `CostMeter` — per call/stage tokens + USD → `ai_log`; aggregates per feed/day. -- `SpendCap` — hard daily ceiling; trips → paid stages pause, admin notice. +- `CostMeter`: per call/stage tokens + USD → `ai_log`; aggregates per feed/day. +- `SpendCap`: hard daily ceiling; trips → paid stages pause, admin notice. - Dashboard: pipeline funnel, per-feed costs, facts-guard flags, ambiguous-entity queue, original-vs-rewrite diff, feed-health. @@ -280,7 +280,7 @@ Adding support for a new post type = adding a rule. No code. ## 6. Extension API (the freemium seams) Free core must expose clean hooks so premium add-ons (and the community) extend without -forking. Retrofitting these later is painful — they go in from phase 1. +forking. Retrofitting these later is painful; they go in from phase 1. **Provider interfaces** (premium add-ons register implementations): @@ -304,11 +304,11 @@ interface Seo_Adapter { // one per SEO plugin **Key filters/actions** (illustrative): - `aggregate_it_register_ai_provider`, `…_research_provider`, `…_keyword_provider` -- `aggregate_it_rewrite_prompt` — mutate the structured-call prompt/schema -- `aggregate_it_cluster_match` — override/augment the 3-signal decision -- `aggregate_it_entity_resolve` — inject custom resolution -- `aggregate_it_before_publish` / `…_after_update` — hook the lifecycle -- `aggregate_it_schema_graph` — mutate the JSON-LD before output +- `aggregate_it_rewrite_prompt`: mutate the structured-call prompt/schema +- `aggregate_it_cluster_match`: override/augment the 3-signal decision +- `aggregate_it_entity_resolve`: inject custom resolution +- `aggregate_it_before_publish` / `…_after_update`: hook the lifecycle +- `aggregate_it_schema_graph`: mutate the JSON-LD before output Premium add-ons (keyword-data provider, research provider, headless extraction, multilingual, strategic-mode packs) are pure consumers of these seams. @@ -317,7 +317,7 @@ multilingual, strategic-mode packs) are pure consumers of these seams. ## 7. Failure & safety posture -- **Scaled Content Abuse:** mitigated structurally — living posts + novelty gate + +- **Scaled Content Abuse:** mitigated structurally: living posts + novelty gate + entity value + citations = genuine added value, not thin rewrites. - **Hallucination:** faithful-rewrite prompt + deterministic facts-guard; strict cited entity stubs; entities degrade to in-article-only without a research provider. diff --git a/docs/CONVENTIONS.md b/docs/CONVENTIONS.md index 709f1a5..43b123c 100644 --- a/docs/CONVENTIONS.md +++ b/docs/CONVENTIONS.md @@ -41,7 +41,7 @@ Mirror `crm-connect.php` exactly: ## 2. Plugin container (`src/Plugin.php`) - `final class Plugin` **singleton** (`instance()`), private constructor. -- **Manual dependency injection** — no DI container. Services are constructed in the +- **Manual dependency injection**: no DI container. Services are constructed in the constructor; shared collaborators are passed in explicitly. - `boot()` calls `->register()` on each service, loads textdomain, runs `Schema::maybe_upgrade()`, and fires `do_action( 'aggregate_it_booted', $this )`. @@ -72,7 +72,7 @@ final class Importer { ## 4. Database (`src/Database/Schema.php`) - One `Schema` class. **Status/state values are class constants** (`STATE_FETCHED`, - `STATE_EXTRACTED`, …) — never bare strings in queries. + `STATE_EXTRACTED`, …), never bare strings in queries. - `table( string $name )` → `$wpdb->prefix . 'aggregate_it_' . $name`. - `install()` builds tables with **`dbDelta`** + `$wpdb->get_charset_collate()`, then `update_option( 'aggregate_it_db_version', AGGREGATE_IT_VERSION )`. @@ -80,7 +80,7 @@ final class Importer { - `uninstall()` drops tables + deletes options (called from `uninstall.php`). - All timestamps stored UTC via `gmdate( 'Y-m-d H:i:s' )`. Always `$wpdb->prepare`. -## 5. The queue — follow crm-connect's custom claim-based queue (NOT Action Scheduler) +## 5. The queue: follow crm-connect's custom claim-based queue (NOT Action Scheduler) This **supersedes the earlier Action Scheduler decision.** crm-connect's queue is a proven, dependency-free pattern that solves the WP-Cron reliability complaints, and @@ -97,10 +97,10 @@ matching it keeps one mental model across both plugins. Reuse it verbatim in sha - **The "nudge":** a recurring **1-minute** custom cron interval is the floor, but a non-blocking `wp_remote_post` to `admin-ajax.php` (guarded by a `run_token` + `hash_equals`) kicks the worker *immediately* after enqueue for low latency. Reuse - this — it's why the cron-backed queue feels real-time. + this; it's why the cron-backed queue feels real-time. **Adaptations for the heavier AI pipeline** (the one place AI work stresses the -pattern — LLM calls take seconds, not the sub-second HTTP of a CRM upsert): +pattern, LLM calls take seconds, not the sub-second HTTP of a CRM upsert): - The worker advances each `ai_items` row **one stage per claim**, not the whole pipeline in one request. @@ -116,14 +116,14 @@ pattern — LLM calls take seconds, not the sub-second HTTP of a CRM upsert): - **Typed accessor methods per setting** with sane defaults (`daily_spend_cap_usd(): float`, `target_post_type(): string`, …). - **Secrets (AI/provider API keys) stored encrypted** via `Support\Crypto` - (`set_api_key()` encrypts, `api_key()` decrypts) — never plaintext in the DB. + (`set_api_key()` encrypts, `api_key()` decrypts), never plaintext in the DB. ## 7. Support utilities (`src/Support/`) Port these from crm-connect; they're directly reusable: -- **`Crypto`** — `aes-256-cbc`, key derived from `AUTH_KEY` via sha256, base64 envelope, +- **`Crypto`**: `aes-256-cbc`, key derived from `AUTH_KEY` via sha256, base64 envelope, graceful fallback when openssl is absent. For all stored API keys. -- **`EventLog`** — bounded ring buffer (max ~200) in an option, `error/warning/info`. +- **`EventLog`**: bounded ring buffer (max ~200) in an option, `error/warning/info`. Use for admin-facing notices (facts-guard flags, ambiguous entities, paused-queue). Per-stage **cost** accounting goes in the queryable `ai_log` *table*, not here. - **`Json`**, **`Url`**, plus new ones as needed (`Html`/readability glue, `Vector` @@ -145,7 +145,7 @@ public function get(): Ai_Provider { ``` Same shape for `Research_Provider`, `Keyword_Provider`, and the `Seo_Adapter` -(resolved by detecting the active SEO plugin). This **is** the freemium seam — premium +(resolved by detecting the active SEO plugin). This **is** the freemium seam; premium add-ons register via the filter. ## 9. Admin (`src/Admin/`) @@ -161,7 +161,7 @@ add-ons register via the filter. ## 10. Distribution & release -**Primary channel: GitHub releases + PUC self-update** (matches crm-connect) — *not* +**Primary channel: GitHub releases + PUC self-update** (matches crm-connect), *not* WordPress.org for v1. - `bin/release.sh X.Y.Z`: validates semver, bumps the version in the header + the @@ -171,8 +171,8 @@ WordPress.org for v1. install, exclude dev files) and publish the GitHub release asset; PUC consumes it. > **If a WordPress.org listing is added later** (worth it for the SEO-discoverability of -> the plugin itself): PUC must be **conditionally disabled** in wp.org builds — the -> directory forbids third-party update sources — and external-AI-API usage must be +> the plugin itself): PUC must be **conditionally disabled** in wp.org builds; the +> directory forbids third-party update sources, and external-AI-API usage must be > disclosed in `readme.txt` with BYO-key opt-in. Keep the two build targets in mind but > don't carry the cost now. diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 5568700..3f5c170 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -8,50 +8,50 @@ considered, and why. These are the load-bearing choices; everything in - **Scale / runtime:** single WordPress site, moderate volume (dozens of feeds, hundreds of items/day). → A **custom claim-based DB queue** (the `crm-connect` - pattern); no external worker service. *(Supersedes an earlier Action Scheduler lean — + pattern); no external worker service. *(Supersedes an earlier Action Scheduler lean, see D14.)* - **Editorial model:** fully automatic publish. The owner edits anything off after the fact. → No human review queue, but rich observability so problems are findable. - **AI / budget:** cheapest viable. → Small/cheap default models behind a provider-agnostic, bring-your-own-key adapter; batch + caching everywhere. -- **Content stance:** full **faithful** rewrite. News stays news — no fact changes. +- **Content stance:** full **faithful** rewrite. News stays news: no fact changes. Strip promotional/boilerplate junk from the source. - **Overriding goal:** SEO. Every decision below is filtered through "does this rank?" -## D1 — Content model: living canonical posts (B), built via hybrid path +## D1: Content model: living canonical posts (B), built via hybrid path -One authoritative post per *story/topic*, updated over time — not one post per feed +One authoritative post per *story/topic*, updated over time, not one post per feed item. Built incrementally: every item still becomes a post in phase 1, but each carries a `topic_cluster_id` from day one, so the "update existing instead of duplicate" behavior layers in later with **no migration**. -- *Rejected:* item-per-post (classic aggregator) — splits link equity + freshness +- *Rejected:* item-per-post (classic aggregator): splits link equity + freshness across duplicates; the opposite of what we want for SEO. - *Why:* one living URL concentrates equity and emits `dateModified` freshness. -## D2 — Updates: append timeline-style + novelty gate +## D2: Updates: append timeline-style + novelty gate -When new info arrives for an existing story, append a dated section ("Update — +When new info arrives for an existing story, append a dated section ("Update: June 25: …"). Never rewrite the whole post. **And** only update when the new item -carries genuinely new facts (novelty gate) — restated wire copy is suppressed. +carries genuinely new facts (novelty gate); restated wire copy is suppressed. - *Rejected:* wholesale re-synthesis (expensive, churns SEO, mangle risk on a cheap model); structured section-merge (highest complexity + mangle risk). - *Why:* cheapest, lowest blast radius under full-auto, preserves facts and URL stability; novelty gate is what keeps it from becoming thin-content spam. -## D3 — Clustering: 3-signal conservative match +## D3: Clustering: 3-signal conservative match Two items are "the same story" only if **all three** hold: semantic similarity (embedding cosine over a configurable threshold) **and** shared primary entity **and** within a rolling time window (default ~7 days, per-feed configurable). -- *Rejected:* similarity-only — merges different stories about the same subject. +- *Rejected:* similarity-only: merges different stories about the same subject. - *Why:* under full-auto a **false merge corrupts a live post**; a false split just leaves a stray duplicate. Bias hard toward not merging. Borderline cases are logged, not silently merged. -## D4 — URLs & post type: flat keyword slugs, configurable target +## D4: URLs & post type: flat keyword slugs, configurable target Canonical posts go to a **configurable target post type (default `post`)**. URLs are **flat, keyword-rich slugs with no dates** (`/ai-generated-keyword-slug/`), generated @@ -62,18 +62,18 @@ comes from the entity internal-link graph, not deep URL nesting. keyword. Google News/Discover targeting deferred (compatible, but extra requirements not taken on now). -## D5 — SEO metadata: hybrid adapter +## D5: SEO metadata: hybrid adapter Detect the active SEO plugin (Yoast / Rank Math / SEOPress) and write the AI-generated **target keyword, SEO title, meta description** into *its* fields. If none is installed, a **minimal native fallback** outputs title + meta only. -- *Rejected:* fully native SEO module — duplicate/competing meta tags when the user +- *Rejected:* fully native SEO module: duplicate/competing meta tags when the user also runs Yoast (they will). - *Principle:* an SEO engine generates the *inputs*; let the user's chosen plugin render the ``. -## D6 — Structured data: native-rich JSON-LD +## D6: Structured data: native-rich JSON-LD Schema is the one place we go native (plugins can't build it from our data). One clean JSON-LD `@graph`: `Article`/`NewsArticle` (per-source toggle), always @@ -85,19 +85,19 @@ adapter **suppresses the plugin's competing Article schema** on our objects. - *Why:* the entity + citation graph is the durable SEO moat and the "added value" proof against Scaled Content Abuse. -## D7 — Keyword targeting: 3 layers, auto-infer default +## D7: Keyword targeting: 3 layers, auto-infer default -1. **Default — AI-inferred** primary keyword per cluster (volume mode: publish all). -2. **Optional — keyword strategy list** (global/per-feed) with a **strategic-mode +1. **Default: AI-inferred** primary keyword per cluster (volume mode: publish all). +2. **Optional: keyword strategy list** (global/per-feed) with a **strategic-mode toggle** (only publish clusters that match a target keyword). -3. **Optional/premium — pluggable keyword-data provider** (DataForSEO/Semrush/Ahrefs, +3. **Optional/premium: pluggable keyword-data provider** (DataForSEO/Semrush/Ahrefs, BYO key) for volume + difficulty, to prioritize/skip clusters. -Guardrails: **keyword locked at cluster creation** (never recomputed — appends must +Guardrails: **keyword locked at cluster creation** (never recomputed, appends must not churn the slug); rewrite prompt treats the keyword as "include naturally, never stuff." -## D8 — Full-content extraction: hybrid + polite +## D8: Full-content extraction: hybrid + polite Use feed content when it's full enough (length heuristic); otherwise fetch the source URL and run **server-side readability extraction (no JS)**. **Polite by default:** @@ -109,7 +109,7 @@ bar**, else skip + log. **Never auto-publish a thin stub.** protects the user's server IP and the plugin's reputation. Headless rendering for JS-heavy/paywalled sites is a deferred premium enhancement. -## D9 — Entity engine: conservative resolution + graceful degradation +## D9: Entity engine: conservative resolution + graceful degradation Per extracted mention, **type-constrained** matching within the target CPT: - **Strong match** (exact/alias or high score) → link. @@ -120,53 +120,53 @@ Per extracted mention, **type-constrained** matching within the target CPT: Creation degrades gracefully: a stub is built from **in-article context** with **zero external dependency**; rich research (cited external facts + `sameAs`) is a **pluggable -BYO-key provider** (premium). Stubs are strict — only cited fields, no hallucinated +BYO-key provider** (premium). Stubs are strict, only cited fields, no hallucinated fills. Ship an **entity merge tool** (repoints links/aliases/schema) as the cleanup valve. - *Why:* a clean sparse graph outranks a dense corrupted one. Duplicate entities are recoverable (merge); wrong links corrupt published posts + schema. -## D10 — Entity linking & hubs +## D10: Entity linking & hubs Contextual in-body link on the **first mention only**, natural-name anchor, **capped links per post**. Bidirectional relationship (postmeta/taxonomy) powers schema + hub. **Hubs are pillar pages:** AI description + cited facts + `sameAs` + an -**auto-growing related-posts list** — every new post about an entity strengthens its +**auto-growing related-posts list**, every new post about an entity strengthens its hub (the topical-authority + freshness flywheel). -## D11 — Freshness & indexing +## D11: Freshness & indexing Sitemaps **delegated to the SEO plugin** (ensure entity CPTs included). **IndexNow** -built in (free) — ping on publish and on a real novelty update. Freshness loop: novelty +built in (free), ping on publish and on a real novelty update. Freshness loop: novelty append → bump `dateModified` → update sitemap `lastmod` → fire IndexNow. No faked instant-index API for normal content. -## D12 — Cost strategy +## D12: Cost strategy **One structured AI call per item** returns `{rewritten_body, seo_title, -meta_description, slug, primary_keyword, entities[], facts[]}` — collapses +meta_description, slug, primary_keyword, entities[], facts[]}`, collapses rewrite+SEO+NER+fact-extraction into one round trip. Embeddings are a separate cheap call. **Batch API** for non-urgent work (~50% off). **Content-hash + prompt caching.** **Per-feed/per-stage cost tracking** with a **hard daily spend cap** that pauses the -queue. **Deterministic facts-guard** (numbers/dates/proper-noun subset check — no +queue. **Deterministic facts-guard** (numbers/dates/proper-noun subset check, no extra AI call) flags invented facts. -## D13 — Images & EEAT +## D13: Images & EEAT Featured image **downloaded into the media library** (never hotlinked); import is **configurable** (off / import / import-if-licensed), default conservative due to copyright. **AI-generated alt text** (image SEO + a11y). **Author/EEAT:** configurable -real author with bio + `author` schema — **no fabricated human personas**. Optional +real author with bio + `author` schema, **no fabricated human personas**. Optional "AI-assisted" disclosure toggle. -## D14 — Engineering conventions: match `crm-connect` +## D14: Engineering conventions: match `crm-connect` The author's existing `crm-connect` plugin sets the house style; Aggregate It mirrors it for consistency. Full detail in [`CONVENTIONS.md`](CONVENTIONS.md). The two choices that *changed* prior decisions: - **Queue: custom claim-based DB queue, not Action Scheduler.** crm-connect runs a - proven dependency-free queue — atomic `claim_token` claiming, stale-claim recovery, + proven dependency-free queue, atomic `claim_token` claiming, stale-claim recovery, exponential backoff, dead-letter, transient auto-pause, and a non-blocking "nudge" (admin-ajax self-POST) that makes a 1-minute cron feel real-time. This answers the WP-Cron reliability concern *without* Action Scheduler, and keeps one diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index b1b1912..8133cf2 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -1,25 +1,25 @@ # Extending Aggregate It Aggregate It's core is free and provider-agnostic. Every paid/external capability is a -seam you (or a premium add-on) plug into via a filter — no core fork required. This is +seam you (or a premium add-on) plug into via a filter, no core fork required. This is also the freemium boundary: the free plugin ships the engine; add-ons ship providers. ## AI provider (rewrite + embeddings) -The one required capability. Four providers ship in core — pick one in Settings, no +The one required capability. Four providers ship in core; pick one in Settings, no filter needed: -- **`MockProvider`** — deterministic, free, the default. Good for testing the pipeline. -- **`GeminiProvider`** — cheapest live option. `generateContent` JSON mode + native +- **`MockProvider`**: deterministic and free, for the test suite only. It echoes the source text, so publishing refuses to run with it; the default is "not configured" until a real provider and key are saved. +- **`GeminiProvider`**: cheapest live option. `generateContent` JSON mode + native `text-embedding-004`. Default model `gemini-2.0-flash-lite`. -- **`OpenAiProvider`** — Chat Completions JSON mode + native `text-embedding-3-small`. +- **`OpenAiProvider`**: Chat Completions JSON mode + native `text-embedding-3-small`. Default model `gpt-4o-mini`. -- **`AnthropicProvider`** — Claude Messages API (structured outputs). Default Haiku 4.5. +- **`AnthropicProvider`**: Claude Messages API (structured outputs). Default Haiku 4.5. Anthropic has no embeddings endpoint, so it uses Voyage AI when a Voyage key is set, else a local lexical fallback. All call their HTTP APIs via the WP HTTP layer (no SDK bundling). The model is a free -Settings field — blank uses the provider's cheapest default. +Settings field; blank uses the provider's cheapest default. To register a *different* provider (a local model, Azure, etc.): @@ -44,14 +44,14 @@ interface AiProvider { } ``` -The structured schema is the one from `Ai\Rewriter::schema()` — +The structured schema is the one from `Ai\Rewriter::schema()`, `rewritten_body, seo_title, meta_description, slug, primary_keyword, entities[], facts[]`. Report real `tokens`/`cost_usd` so the spend cap and dashboard work. Use the Anthropic Batch API for the ~50% discount where latency allows. ## Research provider (entity enrichment, optional) -Absent by default — entities are built from in-article context only. Register one to add +Absent by default; entities are built from in-article context only. Register one to add cited external facts + `sameAs`: ```php @@ -72,21 +72,21 @@ Implement `AggregateIt\Keyword\KeywordProvider::metrics( array $keywords )` → ## Content / prompt / schema hooks -- `aggregate_it_rewrite_prompt` ( $prompt, $title, $content, $target_keyword ) — mutate the rewrite instruction. -- `aggregate_it_rewrite_schema` ( $schema ) — extend the structured-output contract. -- `aggregate_it_extract_html` ( null, $html ) — return a string to override the built-in readability heuristic with a real library. -- `aggregate_it_schema_graph` ( $graph, $post_id ) — mutate the JSON-LD before output. -- `aggregate_it_entity_post_types` ( $types ) — declare extra entity CPTs for the dashboard count. +- `aggregate_it_rewrite_prompt` ( $prompt, $title, $content, $target_keyword ): mutate the rewrite instruction. +- `aggregate_it_rewrite_schema` ( $schema ): extend the structured-output contract. +- `aggregate_it_extract_html` ( null, $html ): return a string to override the built-in readability heuristic with a real library. +- `aggregate_it_schema_graph` ( $graph, $post_id ): mutate the JSON-LD before output. +- `aggregate_it_entity_post_types` ( $types ): declare extra entity CPTs for the dashboard count. ## Lifecycle -- `aggregate_it_publish_ping` ( $post_id ) — fired when a post is published or updated (IndexNow listens here). -- `aggregate_it_spend_cap_reached` ( $cap ) — daily spend ceiling hit. -- `aggregate_it_dead_letter` ( $item, $exception ) — an item failed permanently. -- `aggregate_it_booted` ( $plugin ) — container ready. +- `aggregate_it_publish_ping` ( $post_id ): fired when a post is published or updated (IndexNow listens here). +- `aggregate_it_spend_cap_reached` ( $cap ): daily spend ceiling hit. +- `aggregate_it_dead_letter` ( $item, $exception ): an item failed permanently. +- `aggregate_it_booted` ( $plugin ): container ready. ## Notes for premium add-ons - Ship as a separate plugin that registers the filters above on `plugins_loaded`/`init`. -- The free core never hard-depends on any add-on — every seam degrades gracefully. +- The free core never hard-depends on any add-on: every seam degrades gracefully. - Keep GPL-2.0-or-later for WordPress.org compatibility. diff --git a/docs/PLAN-custom-sources-activity-hubs.md b/docs/PLAN-custom-sources-activity-hubs.md index cd18bdb..02f9a8b 100644 --- a/docs/PLAN-custom-sources-activity-hubs.md +++ b/docs/PLAN-custom-sources-activity-hubs.md @@ -2,34 +2,34 @@ Three independent but mutually reinforcing features. Build order below is by dependency, not by the order they were requested. Everything here is -config-driven — no site-specific code, no hardcoded selectors or post types. +config-driven, no site-specific code, no hardcoded selectors or post types. > **STATUS: complete.** Phase 0 (v0.1.37), Phase 1 + scraper follow-ups (v0.1.38, -> v0.1.39), Phase 2 (v0.1.40), and final polish — live extraction preview + stub-hub -> review queue — all shipped. PRs #1–#5 merged. +> v0.1.39), Phase 2 (v0.1.40), and final polish, live extraction preview + stub-hub +> review queue, all shipped. PRs #1-#5 merged. Guiding constraints (from project memory + this request): - Generalize everything; nothing hardcoded for any specific site. -- Observability over silent fallbacks — every automated decision is recorded. +- Observability over silent fallbacks: every automated decision is recorded. - Clean/minimal UI, no decorative code comments. --- -## Verified ground-truth (2nd research pass) — corrections that change the build +## Verified ground-truth (2nd research pass): corrections that change the build These were checked against the actual source; several invalidate the obvious-but-wrong design: 1. **Live pipeline order is `fetched → extracted → embedded → clustered → entity_linked → published`** (`Pipeline::default_order()`). `STATE_REWRITTEN` - exists in `Schema` but is **not in the active order** — the AI rewrite happens + exists in `Schema` but is **not in the active order**; the AI rewrite happens *inside* the `clustered` stage (`ComposeStage`), which is the real publish step. 2. **`PassthroughStage` is a no-op Phase-0 placeholder** (advances state, does - nothing). It is NOT a "publish structured data verbatim" mechanism — do not + nothing). It is NOT a "publish structured data verbatim" mechanism; do not reuse it for passthrough scraping. 3. **`PostFactory::create()` is hard-coupled to AI `$structured` output** (`seo_title`, `slug`, `rewritten_body`, `facts`) and to the **global** - `Settings::target_post_type()`. There is **no per-source post type** today — it + `Settings::target_post_type()`. There is **no per-source post type** today; it is a genuine gap a scraper must fill (add `PostFactory::create_mapped()`). 4. **Robots.txt + throttle will self-sabotage a naive scraper.** `HttpFetcher::fetch()` *throws* on a robots.txt disallow, and `throttle()` *throws* `RateLimited` if the @@ -40,11 +40,11 @@ design: deferred to a per-item pipeline stage** where `RateLimited → defer` already works. This mirrors how RSS already splits lightweight import from heavy `ExtractStage`. There is an SSRF override filter (`aggregate_it_allow_private_hosts`) but **no - robots override** — a per-source robots toggle (with a visible ToS warning) must + robots override**, a per-source robots toggle (with a visible ToS warning) must be added. 5. **No CSS-selector or HTML5 library is present**; runtime deps are zero (only `php >=8.0`), and the plugin autoloads from `src/` when `vendor/` is absent - (`aggregate-it.php:33-51`). Recommended: stay dependency-free — use + (`aggregate-it.php:33-51`). Recommended: stay dependency-free, use `DOMDocument`/`DOMXPath` (already used in `ContentExtractor`) and accept **XPath** plus a small CSS→XPath converter for the common subset. Adding `symfony/dom-crawler` is the fallback only if the release workflow is confirmed to @@ -54,12 +54,12 @@ Reusable as-is: **`EntityRegistrar::register_post_types()` is the exact template for registering a scraper's target CPT from config; **`HttpFetcher`** gives SSRF guard, robots, per-host throttle, UA, and caching; the **`Entry` format** (`{guid,url,title,content,image,date}`) and `Importer::ingest()`'s guid/hash dedup -work unchanged — but custom scraped fields (location, venue, end_date) need a new +work unchanged, but custom scraped fields (location, venue, end_date) need a new `fields` payload threaded through `ItemStore::enqueue()` into `Item->flags`. --- -## Phase 0 — Activity Log (foundation) — IMPLEMENTED +## Phase 0: Activity Log (foundation), IMPLEMENTED Rationale: a durable "what changed from what" record is the substrate that makes both the scraper and Topic Hubs legible. Build it first; the other phases emit @@ -76,13 +76,13 @@ deletes only zero-cost rows so spend history survives. New `ActivityLogTest`; fu suite 64 green. Migration: gated on a new `Schema::DB_VERSION` (not the plugin version), so the -dbDelta column-add runs on the next boot of any existing install — independent of a +dbDelta column-add runs on the next boot of any existing install, independent of a release. Hardened after an adversarial review pass: activity writes that fail `error_log` once instead of vanishing silently; `wp_json_encode` failures store NULL; the 5s live refresh pauses while a row is expanded or text is selected; the Live toggle disables on paged views; JS-built labels are i18n'd. Note (volume): the old `EventLog` capped at 200 rows; the activity log now writes ~1 info row per extracted -item, pruned by `Retention` at `retention_days` (default 90) — deliberate trade for +item, pruned by `Retention` at `retention_days` (default 90), deliberate trade for durability/observability. ### Problem with today's logging @@ -110,19 +110,19 @@ New table `wp_aggregate_it_activity` (migration in `src/Database/Schema.php`): Index on (created_at), (item_id), (source_id), (type). ### Code -- `src/Support/ActivityLog.php` — `record(type, summary, detail, ctx)`, +- `src/Support/ActivityLog.php`: `record(type, summary, detail, ctx)`, `query(filters, paging)`, `prune(keep_days)`. Keep `EventLog` as a thin back-compat shim that forwards to ActivityLog, or fold its call sites over. - Instrument transitions at the single choke point `ItemStore::advance()` plus each stage where a transformation has a meaningful before/after: - - `ExtractStage` — feed length → extracted length, image source chosen/missed. - - `ClusterStage`/`Clusterer` — new story vs merged into #N, similarity score, + - `ExtractStage`: feed length → extracted length, image source chosen/missed. + - `ClusterStage`/`Clusterer`: new story vs merged into #N, similarity score, shared fact (the "close call" data already exists). - - `ComposeStage` — original feed title → SEO title; raw → rewritten length; + - `ComposeStage`: original feed title → SEO title; raw → rewritten length; **suppression + reason** (thin / keyword / strategic) which is currently silent. - - `EntityStage` — entities detected → linked / created / skipped + score. - - `ImageImporter` — which candidate won. + - `EntityStage`: entities detected → linked / created / skipped + score. + - `ImageImporter`: which candidate won. - Retention: hook into `src/Maintenance/Retention.php` to prune by age/row cap. ### UI @@ -135,7 +135,7 @@ Index on (created_at), (item_id), (source_id), (type). --- -## Phase 1 — Custom Website Sources (generalized scraper) +## Phase 1: Custom Website Sources (generalized scraper) **Feature-complete on `feature/custom-website-sources`** (PR #2): - Engine: `CssToXpath`, `FieldExtractor`, `ScraperParser` (one-fetch list + sitemap @@ -159,7 +159,7 @@ per-source robots-override toggle; a live extraction preview in the form. The centerpiece. Lets a user point at any HTML site (e.g. igamingcalendar.com), extract repeating items, and map fields onto a chosen post type. igaming is -server-rendered HTML with a sitemap but **no RSS / JSON / JSON-LD** — so an HTML +server-rendered HTML with a sitemap but **no RSS / JSON / JSON-LD**, so an HTML scraper is required, and feeds/structured-data are an accuracy bonus when present. ### 1a. Source type + parser factory @@ -171,30 +171,30 @@ scraper is required, and feeds/structured-data are an accuracy bonus when presen - Add `src/Source/Parser/ScraperParser.php`. - `Importer::parse()` becomes a dispatch on `source_type` (the existing seam). -### 1b. Discovery — how items are found (config) +### 1b. Discovery: how items are found (config) Stored in `Source::settings['scrape']['discovery']`: - `mode: "list" | "sitemap"` -- list mode: `list_urls[]`, `item_selector` (the repeating row), `detail` (bool — +- list mode: `list_urls[]`, `item_selector` (the repeating row), `detail` (bool, follow each row's link to a detail page for richer fields). -- sitemap mode: `sitemap_url`, `url_filter` (regex) — each matching URL is an item. +- sitemap mode: `sitemap_url`, `url_filter` (regex): each matching URL is an item. `src/Source/Scrape/Discovery.php` resolves a discovery config into a list of item handles (inline row DOM and/or detail URL). **Runs at import time with a single -`HttpFetcher::fetch()`** (the listing or sitemap page) — per finding #4, detail-page +`HttpFetcher::fetch()`** (the listing or sitemap page), per finding #4, detail-page fetches must NOT happen here. Each handle becomes an `Entry` whose `url` is the detail link and whose inline row fields ride in a new `fields` payload. The detail fetch + richer extraction is deferred to `ExtractStage` (per item, throttle-safe). Scraping a path blocked by robots.txt requires the per-source robots-override toggle -(finding #4) — surfaced in the UI with a ToS warning, implemented as an +(finding #4), surfaced in the UI with a ToS warning, implemented as an `aggregate_it_respect_robots` filter check keyed by source. -### 1c. Extraction — accuracy cascade +### 1c. Extraction: accuracy cascade `src/Source/Scrape/FieldExtractor.php`. For each item, resolve every configured field by trying, in order of accuracy: -1. **Structured data** — schema.org JSON-LD / microdata on the page (most +1. **Structured data**: schema.org JSON-LD / microdata on the page (most accurate when present). -2. **Feed/sitemap metadata** — when the discovery source already carries it. -3. **CSS/XPath selectors** — `{ selector, attr (text/html/href/src/datetime/any +2. **Feed/sitemap metadata**: when the discovery source already carries it. +3. **CSS/XPath selectors**: `{ selector, attr (text/html/href/src/datetime/any attr), regex?, transform? (date-parse, trim) }`. Add a small CSS→XPath converter (or accept XPath directly) over `DOMDocument`/`DOMXPath`. @@ -202,7 +202,7 @@ Field set is open-ended: `title, url, guid, date, image, content` + arbitrary custom fields (`location, venue, end_date, category`, …). Nothing event-specific is hardcoded. -### 1d. Selector configuration — AI-assisted + validated (most accurate practical path) +### 1d. Selector configuration: AI-assisted + validated (most accurate practical path) `src/Source/Scrape/SelectorAssistant.php`: - User pastes a listing URL (and optionally a detail URL). - Fetch sample page(s) via `HttpFetcher` (already has SSRF guard, robots, @@ -213,7 +213,7 @@ is hardcoded. and show the extracted preview; user confirms/edits. Manual selector entry is always available as the floor. - On each scheduled scrape, re-validate; selector drift (rows suddenly empty) - emits a `warning` activity row — the activity log becomes the early-warning + emits a `warning` activity row; the activity log becomes the early-warning system for site changes. ### 1e. Mapping to a post type ("post type connection") @@ -232,18 +232,18 @@ Generic CPT/taxonomy registration from config (reuse the existing `DelegationRules` / `EntityRegistrar` pattern) so a user can define an "Events" post type without code. -### 1f. Processing mode — passthrough by default +### 1f. Processing mode: passthrough by default - Per-source `processing: "passthrough" | "rewrite"`, set at enqueue as `Item->flags['passthrough']`. Scraped structured data defaults to **passthrough**: fields map in verbatim, no AI rewrite, so dates/venues can't be hallucinated. News-style sources opt into `rewrite`. -- Mechanism (corrected — the pipeline is one-stage-per-state and stages route via +- Mechanism (corrected, the pipeline is one-stage-per-state and stages route via their return value): add a cheap `if ($item->flags['passthrough'])` guard to each AI/cost stage so passthrough items skip the expensive work but still traverse the state machine: - `ExtractStage` (fetched): if passthrough, optionally fetch the detail page (throttle-safe; `RateLimited → defer`) and run `FieldExtractor` into - `flags['fields']`; advance — no readability/AI. + `flags['fields']`; advance; no readability/AI. - `EmbedStage` / `ClusterStage`: if passthrough, skip embedding/clustering; advance. - `ComposeStage` (clustered): if passthrough, call **`PostFactory::create_mapped()`** (no `Rewriter`, no `CostMeter`) using the per-source post type + field/meta/ @@ -268,12 +268,12 @@ post type without code. --- -## Phase 2 — Reimagine "Linked Pages" → legible Topic Hubs — SHIPPED (engine reused) +## Phase 2: Reimagine "Linked Pages" → legible Topic Hubs, SHIPPED (engine reused) Landed on `feature/topic-hubs`: renamed "Linked Pages" → **Topic Hubs** (menu, dashboard card, page heading) with a plain-language explainer; `EntityStage` now records every -decision into the Activity log — linked + **created** + the previously-silent **skipped** -(with reason: ambiguous match / no hub for type) — as structured detail; the post editor +decision into the Activity log, linked + **created** + the previously-silent **skipped** +(with reason: ambiguous match / no hub for type), as structured detail; the post editor metabox lists the topic hubs linked from each article. Thresholds were already per-rule config (link 92 / ambiguous 75). Follow-up left: a stub-page review queue. @@ -303,9 +303,9 @@ Same entity-hub engine (`EntityStage`, `DelegationRules`, `EntityRepository`, --- ## Suggested sequencing -1. Phase 0 (Activity log) — small, foundational, immediately useful. -2. Phase 1 (Custom sources) — the headline feature; emits into the log. -3. Phase 2 (Topic Hubs legibility) — mostly UI + surfacing existing data into the log. +1. Phase 0 (Activity log): small, foundational, immediately useful. +2. Phase 1 (Custom sources): the headline feature; emits into the log. +3. Phase 2 (Topic Hubs legibility): mostly UI + surfacing existing data into the log. Each phase ships independently and is backward compatible (new `source_type` defaults to `rss`; activity table is additive; Topic Hubs reuses the existing diff --git a/docs/PLAN.md b/docs/PLAN.md index 90b55d4..e217e52 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -5,9 +5,9 @@ a working aggregator first, then the AI+SEO core that makes it rank, then the en moat, then polish + premium seams. - **Stack:** PHP 8.0+, WordPress 6.4+, custom claim-based queue (the `crm-connect` - pattern — see [`CONVENTIONS.md`](CONVENTIONS.md)), Composer for dev deps. + pattern, see [`CONVENTIONS.md`](CONVENTIONS.md)), Composer for dev deps. - **Standard:** WordPress Coding Standards (PHPCS), GPL-2.0+ only. -- **Conventions:** match `crm-connect` — namespace `AggregateIt\`, singleton `Plugin` +- **Conventions:** match `crm-connect`: namespace `AggregateIt\`, singleton `Plugin` with `register()` services, `Schema`/`dbDelta`, encrypted `Settings`, provider factory-with-filter, GitHub + PUC distribution. Details in `CONVENTIONS.md`. - **Testing:** PHPUnit + WP test suite; integration tests against canned feed fixtures @@ -15,13 +15,13 @@ moat, then polish + premium seams. - **Definition of done per phase:** feature works end-to-end on a real feed, has tests, and the relevant hooks/filters from the extension API are in place. -> The extension-API seams (`ARCHITECTURE.md` §6) are NOT a final phase — each phase +> The extension-API seams (`ARCHITECTURE.md` §6) are NOT a final phase; each phase > wires its own hooks as it lands. Retrofitting them is the one thing that forces a > rewrite later. --- -## Phase 0 — Scaffold (foundation) +## Phase 0: Scaffold (foundation) - Plugin bootstrap (mirror `crm-connect.php`): header w/ GitHub `Plugin URI`, constants, composer autoload + spl fallback, **PUC GitHub updater**, activation→`Schema::install`, @@ -47,7 +47,7 @@ moat, then polish + premium seams. --- -## Phase 1 — Ingestion engine (a working aggregator) — ✅ built +## Phase 1: Ingestion engine (a working aggregator), ✅ built **Goal:** add feeds, import items, deduplicate, land them as posts. No AI yet. @@ -58,7 +58,7 @@ moat, then polish + premium seams. > `aggregate_it_extract_html`), `Pipeline\ExtractStage` (replaces the `fetched` > passthrough; flags thin items), GUID + content-hash dedup in `ItemStore`, and the > Sources admin screen. Items currently flow through to `published` via passthroughs -> (no rewrite yet — that's Phase 2). +> (no rewrite yet; that's Phase 2). - **Sources admin:** CRUD for feeds, per-feed settings, schedule, status, health view. - **Fetch + parse:** SimplePie (RSS/Atom) + JSON Feed; map items → `ai_items`. @@ -71,7 +71,7 @@ moat, then polish + premium seams. plus the nudge (NOT raw WP-Cron for the work itself). - Items publish as plain posts (rewrite is phase 2) so the pipeline is observable. -*Ships:* a polite, reliable RSS aggregator with full-content extraction — already on +*Ships:* a polite, reliable RSS aggregator with full-content extraction, already on par with the baseline tools, minus AI. **Exit criteria:** a real feed imports on schedule; truncated feeds get full content; @@ -79,13 +79,13 @@ duplicates are suppressed; rude-scraping safeguards verified. --- -## Phase 2 — AI + SEO core (the ranking engine) — ✅ built +## Phase 2: AI + SEO core (the ranking engine), ✅ built **Goal:** faithful rewrite, semantic dedup, living canonical posts, full SEO output. > Status: implemented. `Vector\VectorStore` (brute-force cosine), `Pipeline\EmbedStage` > (paid), `Cluster\Clusterer` + `ClusterStage` (3-signal: similarity ∧ time window ∧ -> shared salient fact), `Ai\Rewriter` (single structured call), `Ai\FactsGuard` +> two shared salient facts), `Ai\Rewriter` (single structured call), `Ai\FactsGuard` > (invented-fact + novelty detection), `Keyword\KeywordStrategy` (auto-infer + list + > strategic skip), `Publish\PostFactory` (living posts + dated append), `Seo\Seo` > (Yoast/RankMath/SEOPress meta adapter + schema suppression + native fallback), @@ -94,27 +94,27 @@ duplicates are suppressed; rude-scraping safeguards verified. > cap, and the original-vs-rewrite editor meta box. Verified with the MockProvider. - **`Embedder`** → `ai_vectors`; `VectorStore` with brute-force cosine. -- **`Clusterer`** — 3-signal match (similarity ∧ shared entity ∧ time window); +- **`Clusterer`**: 3-signal match (similarity ∧ shared entity ∧ time window); borderline → log + new cluster. Assign `topic_cluster_id`. -- **`Rewriter`** — the **single structured AI call** → `{body, seo_title, +- **`Rewriter`**: the **single structured AI call** → `{body, seo_title, meta_description, slug, primary_keyword, entities[], facts[]}`. Template-driven prompt (per-feed overridable): faithful rewrite, strip promo, keyword-natural. -- **`FactsGuard`** — deterministic invented-fact detector → flags item. +- **`FactsGuard`**: deterministic invented-fact detector → flags item. - **Living posts (B):** new cluster → create canonical post; matched cluster → **`NoveltyGate`** → append dated section + bump `dateModified` + merge `fact_set`. - **Keyword layer 1** (auto-infer) + **layer 2** scaffold (keyword list + strategic-mode toggle). Keyword **locked at cluster creation**. - **`SlugGenerator`** (flat keyword slug, no dates). -- **`SeoAdapter`** — Yoast / Rank Math / SEOPress strategies + native fallback (title + +- **`SeoAdapter`**: Yoast / Rank Math / SEOPress strategies + native fallback (title + meta); **suppress plugin schema** on our objects. -- **`SchemaGraph`** — native JSON-LD: Article/NewsArticle, `datePublished` + +- **`SchemaGraph`**: native JSON-LD: Article/NewsArticle, `datePublished` + `dateModified`, author, publisher, image, `mainEntityOfPage`, `citation` → sources. (`about`/`mentions`/entity hubs arrive with phase 3.) - **Cost controls live:** batch API path, content-hash + prompt caching, per-feed cost tracking, hard daily spend cap pauses paid stages. - **Diff view:** original vs rewrite in the admin. -*Ships:* an SEO content engine — original faithful posts, one living URL per story, +*Ships:* an SEO content engine: original faithful posts, one living URL per story, freshness signals, correct meta + schema, bounded cost. **Exit criteria:** two near-duplicate feed items produce ONE living post; a genuine @@ -123,7 +123,7 @@ the active SEO plugin with no duplicate schema; spend cap halts paid work on tri --- -## Phase 3 — Entity engine (the moat) — ✅ built +## Phase 3: Entity engine (the moat), ✅ built **Goal:** auto-build the entity hub graph + internal links + entity schema. @@ -139,19 +139,19 @@ the active SEO plugin with no duplicate schema; spend cap halts paid work on tri - **Delegation-rule config** (`ARCHITECTURE.md` §5) + admin UI: map entity type → CPT → field map → match/research/linking settings. -- **`EntityResolver`** — normalize → exact/alias → type-constrained fuzzy+embedding → +- **`EntityResolver`**: normalize → exact/alias → type-constrained fuzzy+embedding → band (link / create / skip-and-log); **batch-dedup guard**. -- **`EntityResearcher`** — default in-article stub (zero deps); pluggable provider path +- **`EntityResearcher`**: default in-article stub (zero deps); pluggable provider path for cited external facts + `sameAs`. Strict cited stubs. -- **`EntityLinker`** — first-mention contextual links, capped per post; relationship +- **`EntityLinker`**: first-mention contextual links, capped per post; relationship taxonomy (bidirectional). -- **`HubRenderer`** — pillar template: description + cited facts + `sameAs` + +- **`HubRenderer`**: pillar template: description + cited facts + `sameAs` + auto-growing related-posts list; Organization/Person/Product schema. - **Schema upgrade:** add `about`/`mentions` to post graph now that hubs exist. -- **`EntityMerge`** admin tool — merge B→A, repoint everything. +- **`EntityMerge`** admin tool: merge B→A, repoint everything. - **Ambiguous-entity queue** in the dashboard. -*Ships:* the durable SEO advantage — topical authority via auto hub-and-spoke linking + +*Ships:* the durable SEO advantage: topical authority via auto hub-and-spoke linking + machine-readable entity/citation graph. **Exit criteria:** a mentioned company links to an existing hub when present, creates a @@ -161,7 +161,7 @@ related list. --- -## Phase 4 — Polish + premium seams — ✅ built +## Phase 4: Polish + premium seams, ✅ built **Goal:** the indexing/freshness loop, media, EEAT, ops maturity, and the paid hooks. @@ -171,9 +171,9 @@ related list. > tracking in `Importer` (consecutive-fail → dead) + admin notice, a full **Settings** > screen (provider/key, spend cap, post type, author, images, IndexNow, clustering, > keywords, feeds), and `docs/EXTENDING.md` documenting every provider/hook seam. All -> phases 0–4 complete. +> phases 0-4 complete. -- **`IndexNow`** — ping on publish + novelty update; sitemap `lastmod` via SEO plugin. +- **`IndexNow`**: ping on publish + novelty update; sitemap `lastmod` via SEO plugin. - **Images:** featured-image import (configurable off/import/if-licensed, default conservative) + **AI alt text**. - **EEAT:** configurable author (bio + `author` schema); optional AI-assisted @@ -182,7 +182,7 @@ related list. - **Observability dashboard:** pipeline funnel, per-feed cost, facts-guard flags, ambiguous-entity queue, feed health. - **Premium provider seams finalized & documented:** keyword-data provider (layer 3), - research provider, headless extraction, multilingual, strategic-mode packs — all as + research provider, headless extraction, multilingual, strategic-mode packs, all as consumers of the phase-0…3 interfaces. *Ships:* production-grade ops + fast indexing + the freemium boundary. @@ -200,7 +200,7 @@ provider registers through the public interface with zero core changes. - **Cost discipline:** no feature added without its `CostMeter` accounting. - **WordPress.org readiness:** external-API disclosure in `readme.txt`, BYO-key opt-in, no obfuscation, sanitize/escape/nonce everywhere, GPL-compatible deps only. -- **Docs:** keep `DECISIONS.md` authoritative — if a decision changes during build, +- **Docs:** keep `DECISIONS.md` authoritative: if a decision changes during build, amend it there first, then the code. --- @@ -208,10 +208,10 @@ provider registers through the public interface with zero core changes. ## Open items to verify at build time 1. **Readability library:** confirm a maintained, **GPL-compatible** PHP readability - port (license check is a hard gate — Apache-2.0 is compatible with GPLv3 but not + port (license check is a hard gate; Apache-2.0 is compatible with GPLv3 but not GPLv2-only; pick accordingly given GPL-2.0-**or-later**). 2. **Cheapest viable model IDs + batch availability** for the default `Ai_Provider` - (decided at implementation, behind the adapter — not baked into core). + (decided at implementation, behind the adapter, not baked into core). 3. **Embedding model + dims** (sets `ai_vectors.dims` and the cosine packing format). 4. **SEO-plugin meta keys** for Yoast / Rank Math / SEOPress current versions (adapter strategies); confirm each exposes a schema-suppression filter. diff --git a/src/Admin/Admin.php b/src/Admin/Admin.php index cfc40a9..f88f6c6 100644 --- a/src/Admin/Admin.php +++ b/src/Admin/Admin.php @@ -2,6 +2,7 @@ namespace AggregateIt\Admin; +use AggregateIt\Ai\ProviderFactory; use AggregateIt\Database\Schema; use AggregateIt\Pipeline\Pipeline; use AggregateIt\Plugin; @@ -49,6 +50,47 @@ public function register(): void { add_action( 'admin_post_aggregate_it_clear_logs', [ $this, 'handle_clear_logs' ] ); add_action( 'admin_post_aggregate_it_reset', [ $this, 'handle_reset' ] ); add_action( 'admin_notices', [ $this, 'feed_health_notice' ] ); + add_action( 'admin_notices', [ $this, 'provider_notice' ] ); + add_action( 'admin_notices', [ $this, 'embedding_notice' ] ); + } + + public function provider_notice(): void { + if ( ! current_user_can( 'manage_options' ) ) { + return; + } + $reason = get_transient( ProviderFactory::BLOCKED_TRANSIENT ); + if ( ! $reason ) { + return; + } + $message = $reason === 'undecryptable' + ? __( 'Aggregate It: the saved AI API key cannot be decrypted, so articles are on hold. Enter the key again in Settings.', 'aggregate-it' ) + : __( 'Aggregate It: no AI service is configured, so articles are on hold. Choose a service and add its API key in Settings.', 'aggregate-it' ); + printf( + '

%s %s

', + esc_html( $message ), + esc_url( admin_url( 'admin.php?page=' . self::SLUG . '-settings' ) ), + esc_html__( 'Open settings', 'aggregate-it' ) + ); + } + + public function embedding_notice(): void { + if ( ! current_user_can( 'manage_options' ) ) { + return; + } + $reason = get_transient( 'aggregate_it_embed_degraded' ); + if ( ! $reason ) { + return; + } + printf( + '

%s

', + esc_html( + sprintf( + /* translators: %s: why embeddings failed */ + __( 'Aggregate It: embeddings are failing (%s). Duplicate detection is degraded, so similar articles may publish as separate posts.', 'aggregate-it' ), + (string) $reason + ) + ) + ); } public function feed_health_notice(): void { @@ -153,7 +195,7 @@ public function assets( string $hook ): void { 'refreshing' => __( 'Refreshing…', 'aggregate-it' ), 'seeded' => __( 'Sample articles added.', 'aggregate-it' ), 'running' => __( 'Working on it…', 'aggregate-it' ), - 'started' => __( 'Started — new posts will appear shortly.', 'aggregate-it' ), + 'started' => __( 'Started. New posts will appear shortly.', 'aggregate-it' ), 'resumed' => __( 'Back up and running.', 'aggregate-it' ), 'failed' => __( 'Something went wrong. Please try again.', 'aggregate-it' ), 'expired' => __( 'Your session expired. Reload the page to continue.', 'aggregate-it' ), @@ -179,7 +221,7 @@ public function render_dashboard(): void { $s = $this->plugin->settings(); $setup = [ - 'provider' => $s->provider_key() !== 'mock' && $s->api_key() !== '', + 'provider' => $this->plugin->providers()->blocked_reason() === null && $s->api_key() !== '', 'feeds' => (bool) $this->plugin->sources()->all(), 'types' => (bool) $this->plugin->rules()->post_types(), ]; @@ -325,9 +367,14 @@ public function handle_rewrite_article(): void { $id = (int) ( $_REQUEST['id'] ?? 0 ); $this->guard( 'aggregate_it_rewrite_article_' . $id ); - $item = $this->plugin->items()->find( $id ); + $item = $this->plugin->items()->find( $id ); + $reproc = $this->plugin->reprocessor(); + if ( $reproc->blocked() ) { + $this->flash( __( 'Rewrite skipped: no AI provider is configured. Add an API key in Settings, then try again.', 'aggregate-it' ), 'error' ); + $this->redirect( self::SLUG . '-articles', '', $this->articles_context() ); + } if ( $item && ! empty( $item->post_id ) ) { - $this->plugin->reprocessor()->reprocess( (int) $item->post_id ); + $reproc->reprocess( (int) $item->post_id ); } $this->redirect( self::SLUG . '-articles', 'rewritten', $this->articles_context() ); @@ -345,6 +392,10 @@ public function handle_bulk_articles(): void { $items = $this->plugin->items(); $reproc = $action === 'rewrite' ? $this->plugin->reprocessor() : null; + if ( $reproc && $reproc->blocked() ) { + $this->flash( __( 'Rewrite skipped: no AI provider is configured. Add an API key in Settings, then try again.', 'aggregate-it' ), 'error' ); + $this->redirect( self::SLUG . '-articles', '', $this->articles_context() ); + } $refreshed = 0; $busy = 0; @@ -365,7 +416,7 @@ public function handle_bulk_articles(): void { $refreshed++; } } catch ( \Throwable $e ) { - $busy++; // rate-limited / transient — re-run to retry + $busy++; // rate-limited / transient, re-run to retry } } break; @@ -397,7 +448,7 @@ public function handle_bulk_articles(): void { $msg = sprintf( _n( 'Refreshed %d featured image.', 'Refreshed %d featured images.', $refreshed, 'aggregate-it' ), $refreshed ); if ( $busy ) { /* translators: %d: number skipped */ - $msg .= ' ' . sprintf( _n( '%d was skipped because the source was busy — run it again to retry.', '%d were skipped because the source was busy — run it again to retry.', $busy, 'aggregate-it' ), $busy ); + $msg .= ' ' . sprintf( _n( '%d was skipped because the source was busy. Run it again to retry.', '%d were skipped because the source was busy. Run it again to retry.', $busy, 'aggregate-it' ), $busy ); } $this->flash( $msg ); $this->redirect( self::SLUG . '-articles', '', $this->articles_context() ); @@ -725,7 +776,7 @@ public function handle_save_rule(): void { $this->guard( 'aggregate_it_save_rule' ); // One field carries both the entity type the AI detects and the post type the - // hubs live under — keep it dead simple: "Company" → type "company", CPT "company". + // hubs live under. Keep it dead simple: "Company" → type "company", CPT "company". $name = sanitize_text_field( wp_unslash( $_POST['type_name'] ?? '' ) ); $slug = sanitize_key( $name !== '' ? $name : ( $_POST['target_cpt'] ?? '' ) ); $entity_type = sanitize_key( wp_unslash( $_POST['entity_type'] ?? '' ) ) ?: $slug; @@ -896,7 +947,7 @@ public function handle_save_settings(): void { $settings->update( [ 'brand_name' => sanitize_text_field( wp_unslash( $_POST['brand_name'] ?? '' ) ), - 'provider' => sanitize_key( wp_unslash( $_POST['provider'] ?? 'mock' ) ), + 'provider' => sanitize_key( wp_unslash( $_POST['provider'] ?? 'none' ) ), 'ai_model' => sanitize_text_field( wp_unslash( $_POST['ai_model'] ?? '' ) ), 'max_output_tokens' => max( 1024, (int) ( $_POST['max_output_tokens'] ?? 8000 ) ), 'target_post_type' => sanitize_key( wp_unslash( $_POST['target_post_type'] ?? 'post' ) ), @@ -926,16 +977,25 @@ public function handle_save_settings(): void { ] ); - $api_key = trim( (string) wp_unslash( $_POST['api_key'] ?? '' ) ); - if ( $api_key !== '' ) { - $settings->set_api_key( $api_key ); - } + try { + $api_key = trim( (string) wp_unslash( $_POST['api_key'] ?? '' ) ); + if ( $api_key !== '' ) { + $settings->set_api_key( $api_key ); + } + + $voyage_key = trim( (string) wp_unslash( $_POST['voyage_api_key'] ?? '' ) ); + if ( $voyage_key !== '' ) { + $settings->set_voyage_api_key( $voyage_key ); + } - $voyage_key = trim( (string) wp_unslash( $_POST['voyage_api_key'] ?? '' ) ); - if ( $voyage_key !== '' ) { - $settings->set_voyage_api_key( $voyage_key ); + $settings->upgrade_key_encryption(); + } catch ( \RuntimeException $e ) { + $this->redirect( self::SLUG . '-settings', 'key_not_stored' ); } + delete_transient( ProviderFactory::BLOCKED_TRANSIENT ); + delete_transient( 'aggregate_it_embed_degraded' ); + $this->redirect( self::SLUG . '-settings', 'saved' ); } @@ -1071,11 +1131,19 @@ public function handle_reset(): void { break; case 'dupes_preview': $preview = $this->plugin->deduplicator()->run( true ); + if ( ! $preview['complete'] ) { + $this->flash( $this->dedupe_progress( $preview ) ); + break; + } /* translators: 1: duplicate posts, 2: stories */ $this->flash( sprintf( __( 'Preview: %1$d duplicate posts across %2$d stories. See the Activity log for examples. Nothing was changed.', 'aggregate-it' ), $preview['duplicates'], $preview['groups'] ) ); break; case 'dupes': $result = $this->plugin->deduplicator()->run( false ); + if ( ! $result['complete'] ) { + $this->flash( $this->dedupe_progress( $result ) ); + break; + } /* translators: %d: number of duplicate posts trashed */ $this->flash( sprintf( _n( 'Trashed %d duplicate post, keeping the oldest of its story.', 'Trashed %d duplicate posts, keeping the oldest of each story.', $result['duplicates'], 'aggregate-it' ), $result['duplicates'] ) ); break; @@ -1086,6 +1154,16 @@ public function handle_reset(): void { $this->redirect( self::SLUG . '-settings', '', [ 'tab' => 'tools' ] ); } + /** @param array{checked:int,total:int} $progress */ + private function dedupe_progress( array $progress ): string { + return sprintf( + /* translators: 1: posts checked so far, 2: total posts */ + __( 'Checked %1$d of %2$d posts. Run this again to continue; nothing is trashed until every post is checked.', 'aggregate-it' ), + $progress['checked'], + $progress['total'] + ); + } + private function flash( string $message, string $type = 'success' ): void { set_transient( 'aggregate_it_flash', @@ -1134,7 +1212,7 @@ private function upload_error_message( int $error ): string { }; } - /** @return array settings safe to export — secrets are never written out */ + /** @return array settings safe to export. Secrets are never written out */ private function exportable_settings(): array { $all = $this->plugin->settings()->all(); unset( $all['api_key'], $all['voyage_api_key'] ); @@ -1185,7 +1263,7 @@ private function import_wprss_config( array $data ): string { } $save_images = $ftp['save_images_locally'] ?? null; if ( 'true' === $save_images || true === $save_images || '1' === (string) $save_images ) { - // Only ever turn import on from an import — never silently disable featured images + // Only ever turn import on from an import. Never silently disable featured images // (aggregate-it has no hotlink mode, so 'off' means no image at all). $map['image_mode'] = 'import'; } diff --git a/src/Admin/HubColumns.php b/src/Admin/HubColumns.php index 1c45f64..26d4f9c 100644 --- a/src/Admin/HubColumns.php +++ b/src/Admin/HubColumns.php @@ -61,7 +61,7 @@ public function render( string $column, int $post_id ): void { } elseif ( get_post_meta( $post_id, '_ai_canonical_name', true ) ) { printf( '%s', esc_attr( $base ), esc_html__( 'Topic hub', 'aggregate-it' ) ); } else { - echo '—'; + echo '-'; } } } diff --git a/src/Admin/PostMetaBox.php b/src/Admin/PostMetaBox.php index 71281da..a240530 100644 --- a/src/Admin/PostMetaBox.php +++ b/src/Admin/PostMetaBox.php @@ -8,7 +8,7 @@ /** * Shows the original source text, source citations, and any flagged invented figures on - * the editor for generated posts — the "what did the AI change" view. + * the editor for generated posts: the "what did the AI change" view. */ final class PostMetaBox { @@ -21,7 +21,7 @@ public function register(): void { public function add(): void { add_meta_box( 'aggregate-it-source', - __( 'Aggregate It — where this came from', 'aggregate-it' ), + __( 'Aggregate It: where this came from', 'aggregate-it' ), [ $this, 'render' ], $this->settings->target_post_type(), 'normal', diff --git a/src/Admin/RestController.php b/src/Admin/RestController.php index cc39d56..005c6bf 100644 --- a/src/Admin/RestController.php +++ b/src/Admin/RestController.php @@ -251,7 +251,7 @@ public function suggest_selectors( WP_REST_Request $request ): WP_REST_Response } $provider = $this->plugin->providers()->get(); - if ( $this->plugin->settings()->provider_key() !== 'mock' && $provider->key() === 'mock' ) { + if ( $this->plugin->providers()->blocked_reason() !== null ) { return new WP_REST_Response( [ 'ok' => false, 'error' => __( 'Add an AI API key in Settings first.', 'aggregate-it' ) ], 200 ); } @@ -285,7 +285,7 @@ public function resume(): WP_REST_Response { public function test(): WP_REST_Response { try { $provider = $this->plugin->providers()->get(); - if ( $this->plugin->settings()->provider_key() !== 'mock' && $provider->key() === 'mock' ) { + if ( $this->plugin->providers()->blocked_reason() !== null ) { return new WP_REST_Response( [ 'ok' => false, 'error' => __( 'No API key saved for the selected service.', 'aggregate-it' ) ], 200 ); } $schema = [ diff --git a/src/Admin/views/articles.php b/src/Admin/views/articles.php index d3cdc12..d372ec4 100644 --- a/src/Admin/views/articles.php +++ b/src/Admin/views/articles.php @@ -35,9 +35,9 @@ if ( $row->state === 'published' ) { if ( ! empty( $flags['suppressed'] ) ) { $reason = [ - 'thin' => __( 'Skipped — too short', 'aggregate-it' ), - 'no-novelty' => __( 'Skipped — duplicate', 'aggregate-it' ), - 'no-keyword-match' => __( 'Skipped — no keyword match', 'aggregate-it' ), + 'thin' => __( 'Skipped: too short', 'aggregate-it' ), + 'no-novelty' => __( 'Skipped: duplicate', 'aggregate-it' ), + 'no-keyword-match' => __( 'Skipped: no keyword match', 'aggregate-it' ), ]; return [ $reason[ $flags['suppressed'] ] ?? __( 'Skipped', 'aggregate-it' ), 'post-state' ]; } @@ -201,7 +201,7 @@ - source_id ] ?? '—' ); ?> + source_id ] ?? '-' ); ?> state === 'dead_letter' ) : ?> last_error ) : ?> @@ -215,7 +215,7 @@  ·  - — + - created_at ); ?> diff --git a/src/Admin/views/dashboard.php b/src/Admin/views/dashboard.php index 3d10207..0cc7e8f 100644 --- a/src/Admin/views/dashboard.php +++ b/src/Admin/views/dashboard.php @@ -14,7 +14,7 @@
- + @@ -78,7 +78,7 @@
- — + -
diff --git a/src/Admin/views/settings.php b/src/Admin/views/settings.php index b28e337..2911268 100644 --- a/src/Admin/views/settings.php +++ b/src/Admin/views/settings.php @@ -17,6 +17,10 @@

+ +

+ +
@@ -27,7 +31,7 @@ + placeholder="voyage_api_key() !== '' ? esc_attr__( '•••••••• (saved)', 'aggregate-it' ) : esc_attr__( 'optional: only if you use Claude', 'aggregate-it' ); ?>"> diff --git a/src/Admin/views/sources.php b/src/Admin/views/sources.php index ed8f66c..32399d7 100644 --- a/src/Admin/views/sources.php +++ b/src/Admin/views/sources.php @@ -194,7 +194,7 @@ > -

+

@@ -319,7 +319,7 @@ + placeholder=""> @@ -428,7 +428,7 @@ function apply() { return; } fill( data.suggestion || {} ); - if ( status ) { status.textContent = 'Detected — showing a sample of what will be scraped…'; } + if ( status ) { status.textContent = 'Detected. Showing a sample of what will be scraped…'; } var pv = document.getElementById( 'ai-preview-btn' ); if ( pv ) { pv.click(); } } ).catch( function () { @@ -493,7 +493,7 @@ function renderPreview( box, rows ) { if ( ! box ) { return; } box.innerHTML = ''; if ( ! rows.length ) { - box.textContent = 'No items matched — try Edit selectors below.'; + box.textContent = 'No items matched. Try Edit selectors below.'; return; } var grid = document.createElement( 'div' ); @@ -698,7 +698,7 @@ function fillMetaKeys() { $health = $source->health; $health_text = ! empty( $health['last_error'] ) ? esc_html( sprintf( /* translators: %s: error message */ __( 'Problem: %s', 'aggregate-it' ), $health['last_error'] ) ) - : ( isset( $health['last_imported'] ) ? sprintf( /* translators: %d: article count */ esc_html__( '%d new articles last time', 'aggregate-it' ), (int) $health['last_imported'] ) : '—' ); + : ( isset( $health['last_imported'] ) ? sprintf( /* translators: %d: article count */ esc_html__( '%d new articles last time', 'aggregate-it' ), (int) $health['last_imported'] ) : '-' ); $import_url = wp_nonce_url( admin_url( 'admin-post.php?action=aggregate_it_import_now&id=' . $source->id ), 'aggregate_it_import_now_' . $source->id ); $delete_url = wp_nonce_url( admin_url( 'admin-post.php?action=aggregate_it_delete_source&id=' . $source->id ), 'aggregate_it_delete_source_' . $source->id ); $edit_url = admin_url( 'admin.php?page=aggregate-it-sources&edit=' . $source->id ); @@ -706,7 +706,7 @@ function fillMetaKeys() { - title ?: '—' ); ?> + title ?: '-' ); ?>
| | @@ -715,7 +715,7 @@ function fillMetaKeys() { url ); ?> status ); ?> - last_checked ?: '—' ); ?> + last_checked ?: '-' ); ?> diff --git a/src/Ai/AnthropicProvider.php b/src/Ai/AnthropicProvider.php index 4770830..e0f9482 100644 --- a/src/Ai/AnthropicProvider.php +++ b/src/Ai/AnthropicProvider.php @@ -97,7 +97,7 @@ public function structured( string $prompt, array $schema, array $opts = [] ): a public function embed( string $text ): array { $voyage_key = $this->settings->voyage_api_key(); if ( $voyage_key === '' ) { - return $this->hash_embedding( $text ); + return $this->degraded_embedding( $text, 'no Voyage AI key is set; Claude has no embeddings of its own' ); } $response = wp_remote_post( @@ -113,13 +113,13 @@ public function embed( string $text ): array { ); if ( is_wp_error( $response ) ) { - return $this->hash_embedding( $text ); + return $this->degraded_embedding( $text, $response->get_error_message() ); } $data = json_decode( (string) wp_remote_retrieve_body( $response ), true ); $vector = $data['data'][0]['embedding'] ?? null; if ( ! is_array( $vector ) ) { - return $this->hash_embedding( $text ); + return $this->degraded_embedding( $text, (string) ( $data['detail'] ?? $data['error']['message'] ?? 'no vector in response' ) ); } return [ diff --git a/src/Ai/FactsGuard.php b/src/Ai/FactsGuard.php index bbce39d..a33af5a 100644 --- a/src/Ai/FactsGuard.php +++ b/src/Ai/FactsGuard.php @@ -12,6 +12,15 @@ */ final class FactsGuard { + /** Common words that are capitalized only because they open a sentence or clause. */ + private const STOPWORDS = [ + 'a', 'an', 'the', 'this', 'that', 'these', 'those', 'it', 'its', 'he', 'she', 'they', 'we', 'you', 'his', 'her', 'their', 'our', + 'in', 'on', 'at', 'for', 'with', 'from', 'by', 'to', 'of', 'as', 'after', 'before', 'during', 'while', 'when', 'where', + 'and', 'but', 'or', 'so', 'if', 'then', 'also', 'however', 'meanwhile', 'according', 'there', 'here', 'what', 'who', + 'is', 'are', 'was', 'were', 'will', 'has', 'have', 'had', 'not', 'no', 'yes', 'some', 'many', 'most', 'other', 'one', 'all', + 'today', 'yesterday', 'tomorrow', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday', + ]; + /** @return string[] normalized salient tokens */ public function salient( string $text ): array { $tokens = []; @@ -23,10 +32,21 @@ public function salient( string $text ): array { } } - // Capitalized multi-word proper nouns (Acme Corp, Tim Cook). - if ( preg_match_all( '/\b([A-Z][a-zA-Z0-9]+(?:\s+[A-Z][a-zA-Z0-9]+){0,3})\b/', $text, $m ) ) { - foreach ( $m[1] as $name ) { - $tokens[] = strtolower( $name ); + // Capitalized proper nouns (Acme Corp, Tim Cook). A lone capitalized word that opens + // a sentence is just grammar ("Yesterday", "Shares"), so it is not an entity. + if ( preg_match_all( '/\b([A-Z][a-zA-Z0-9]+(?:\s+[A-Z][a-zA-Z0-9]+){0,3})\b/', $text, $m, PREG_OFFSET_CAPTURE ) ) { + foreach ( $m[1] as [ $run, $offset ] ) { + $words = preg_split( '/\s+/', $run ) ?: []; + while ( $words && in_array( strtolower( $words[0] ), self::STOPWORDS, true ) ) { + array_shift( $words ); + } + if ( ! $words ) { + continue; + } + if ( count( $words ) === 1 && $this->starts_sentence( $text, (int) $offset ) ) { + continue; + } + $tokens[] = strtolower( implode( ' ', $words ) ); } } @@ -34,7 +54,7 @@ public function salient( string $text ): array { } /** - * Numeric/date tokens present in the rewrite but absent from the source — likely + * Numeric/date tokens present in the rewrite but absent from the source, likely * fabrications. Proper nouns are excluded (rewrites legitimately rephrase names). * * @return string[] @@ -55,6 +75,14 @@ public function novel( string $new, array $known ): array { return array_values( array_diff( $this->salient( $new ), $known ) ); } + private function starts_sentence( string $text, int $offset ): bool { + $before = rtrim( substr( $text, 0, $offset ), " \t\"'\u{201C}\u{2018}(" ); + if ( $before === '' ) { + return true; + } + return strpbrk( substr( $before, -1 ), ".!?:\n" ) !== false; + } + /** @return string[] */ private function numbers( string $text ): array { if ( ! preg_match_all( '/\d[\d,.]*\%?/', $text, $m ) ) { diff --git a/src/Ai/GeminiProvider.php b/src/Ai/GeminiProvider.php index e18b3bf..5d07fac 100644 --- a/src/Ai/GeminiProvider.php +++ b/src/Ai/GeminiProvider.php @@ -6,7 +6,7 @@ /** * Google Gemini provider. Structured output via generateContent with a JSON response - * mime type; embeddings via text-embedding-004. Default model gemini-2.0-flash-lite — + * mime type; embeddings via text-embedding-004. Default model gemini-2.0-flash-lite, * among the cheapest text models available. */ final class GeminiProvider extends BaseHttpProvider { @@ -31,10 +31,10 @@ public function structured( string $prompt, array $schema, array $opts = [] ): a $model = $this->settings->ai_model() ?: self::DEFAULT_MODEL; $response = wp_remote_post( - self::BASE . rawurlencode( $model ) . ':generateContent?key=' . rawurlencode( $this->settings->api_key() ), + self::BASE . rawurlencode( $model ) . ':generateContent', [ 'timeout' => 120, - 'headers' => [ 'content-type' => 'application/json' ], + 'headers' => $this->headers(), 'body' => wp_json_encode( [ 'contents' => [ [ 'parts' => [ [ 'text' => $prompt . $this->schema_hint( $schema ) ] ] ] ], @@ -59,10 +59,10 @@ public function structured( string $prompt, array $schema, array $opts = [] ): a public function embed( string $text ): array { $response = wp_remote_post( - self::BASE . self::EMBED_MODEL . ':embedContent?key=' . rawurlencode( $this->settings->api_key() ), + self::BASE . self::EMBED_MODEL . ':embedContent', [ 'timeout' => 30, - 'headers' => [ 'content-type' => 'application/json' ], + 'headers' => $this->headers(), 'body' => wp_json_encode( [ 'model' => 'models/' . self::EMBED_MODEL, @@ -73,13 +73,13 @@ public function embed( string $text ): array { ); if ( is_wp_error( $response ) ) { - return $this->hash_embedding( $text ); + return $this->degraded_embedding( $text, $response->get_error_message() ); } $data = json_decode( (string) wp_remote_retrieve_body( $response ), true ); $vector = $data['embedding']['values'] ?? null; if ( ! is_array( $vector ) ) { - return $this->hash_embedding( $text ); + return $this->degraded_embedding( $text, (string) ( $data['error']['message'] ?? 'no vector in response' ) ); } return [ @@ -89,6 +89,14 @@ public function embed( string $text ): array { ]; } + /** The key travels in a header so it never lands in URLs, proxy logs, or error messages. */ + private function headers(): array { + return [ + 'content-type' => 'application/json', + 'x-goog-api-key' => $this->settings->api_key(), + ]; + } + private function decode( $response ): array { if ( is_wp_error( $response ) ) { throw new \RuntimeException( 'Gemini request failed: ' . $response->get_error_message() ); diff --git a/src/Ai/HashEmbedding.php b/src/Ai/HashEmbedding.php index 1701b56..1dc53fb 100644 --- a/src/Ai/HashEmbedding.php +++ b/src/Ai/HashEmbedding.php @@ -7,17 +7,17 @@ trait HashEmbedding { /** - * Hash fallback, but warn (once an hour) that real embeddings are down. Silent fallback - * here makes every article look unique, which breaks story de-duplication — so it must + * Hash fallback, but warn (once an hour, and in an admin notice) that real embeddings are down. Silent fallback + * here makes every article look unique, which breaks story de-duplication, so it must * be visible, not swallowed. * * @return array{vector:float[],tokens:int,cost_usd:float} */ protected function degraded_embedding( string $text, string $reason ): array { if ( ! get_transient( 'aggregate_it_embed_degraded' ) ) { - set_transient( 'aggregate_it_embed_degraded', 1, HOUR_IN_SECONDS ); + set_transient( 'aggregate_it_embed_degraded', $reason, HOUR_IN_SECONDS ); \AggregateIt\Support\EventLog::warning( - sprintf( 'Embeddings are failing (%s). Duplicate detection is degraded — similar articles may publish as separate posts. Check the AI provider API key and plan.', $reason ) + sprintf( 'Embeddings are failing (%s). Duplicate detection is degraded. Similar articles may publish as separate posts. Check the AI provider API key and plan.', $reason ) ); } return $this->hash_embedding( $text ); diff --git a/src/Ai/MockProvider.php b/src/Ai/MockProvider.php index 93f003e..ca1d102 100644 --- a/src/Ai/MockProvider.php +++ b/src/Ai/MockProvider.php @@ -5,9 +5,9 @@ defined( 'ABSPATH' ) || exit; /** - * Deterministic, zero-cost provider used for tests and for running the pipeline before - * a real provider is configured. Output is derived from the input so the same text - * always yields the same result. + * Deterministic, zero-cost provider for tests. Output is derived from the input so the + * same text always yields the same result. It returns the source verbatim as the + * "rewrite", so ProviderFactory::blocked_reason() keeps it out of publishing. */ final class MockProvider implements AiProvider { diff --git a/src/Ai/OpenAiProvider.php b/src/Ai/OpenAiProvider.php index 0e7495a..10c85f6 100644 --- a/src/Ai/OpenAiProvider.php +++ b/src/Ai/OpenAiProvider.php @@ -84,7 +84,7 @@ private function chat( string $model, string $message, array $response_format ): /** * Rewrite our schema into the subset OpenAI's strict mode accepts: every object must list * all properties as required and forbid extras, and string/number constraints it rejects - * (maxLength and friends) are dropped — length is still guided by the prompt. + * (maxLength and friends) are dropped; length is still guided by the prompt. * * @param array $schema * @return array diff --git a/src/Ai/ProviderFactory.php b/src/Ai/ProviderFactory.php index 6109ee3..13776c9 100644 --- a/src/Ai/ProviderFactory.php +++ b/src/Ai/ProviderFactory.php @@ -8,6 +8,9 @@ final class ProviderFactory { + /** Transient holding why publishing is on hold; the admin notice reads it. */ + public const BLOCKED_TRANSIENT = 'aggregate_it_provider_blocked'; + private ?AiProvider $resolved = null; public function __construct( private Settings $settings ) {} @@ -19,6 +22,19 @@ public function get(): AiProvider { return $this->resolved; } + /** + * Why publishing must not run, or null when a real provider is usable. MockProvider + * echoes the source text as a "rewrite", so it is allowed only under the test suite. + * + * @return 'unconfigured'|'undecryptable'|null + */ + public function blocked_reason(): ?string { + if ( $this->get()->key() !== 'mock' || ( defined( 'AGGREGATE_IT_TESTING' ) && AGGREGATE_IT_TESTING ) ) { + return null; + } + return $this->settings->api_key_unreadable() ? 'undecryptable' : 'unconfigured'; + } + private function make(): AiProvider { $key = $this->settings->provider_key(); diff --git a/src/Ai/ProviderNotConfigured.php b/src/Ai/ProviderNotConfigured.php new file mode 100644 index 0000000..96f536e --- /dev/null +++ b/src/Ai/ProviderNotConfigured.php @@ -0,0 +1,11 @@ +providers->get()->structured( $prompt, $this->schema() ); } + /** + * Short update note for a story that already has a live post: only the facts the post + * lacks, never a second full article. + * + * @param string[] $novel facts or names the live post does not yet cover + * @return array{result:array,tokens:int,cost_usd:float} + */ + public function rewrite_update( string $title, string $content, array $novel ): array { + $rules = [ + 'An article on this story is already published. Write ONLY a short update note covering what is new in the article below.', + 'Cover just the new facts. Do not repeat background the published article already has.', + 'Keep it to one short paragraph of 2-4 sentences, under 80 words.', + 'PRESERVE every number, name, date, and quote you use exactly. Never invent or alter a fact.', + 'Neutral, factual tone. Do not mention sources, the original article, or that this is an update note.', + 'Return ONLY the structured object, with the note in "rewritten_body".', + ]; + if ( $novel ) { + $rules[] = 'Details that are new: ' . implode( ', ', array_slice( $novel, 0, 20 ) ) . '.'; + } + + $prompt = "You are a faithful news editor.\n\n" + . implode( "\n", array_map( static fn ( $r ) => '- ' . $r, $rules ) ) + . "\n\nTITLE: " . $title + . "\n\nARTICLE:\n" . $content; + + $schema = [ + 'type' => 'object', + 'additionalProperties' => false, + 'required' => [ 'rewritten_body' ], + 'properties' => [ 'rewritten_body' => [ 'type' => 'string' ] ], + ]; + + return $this->providers->get()->structured( $prompt, $schema ); + } + /** @param string[] $categories */ private function prompt( string $title, string $content, ?string $target_keyword, ?string $length = null, array $categories = [] ): string { $rules = [ @@ -32,9 +67,9 @@ private function prompt( string $title, string $content, ?string $target_keyword 'PRESERVE every fact verbatim: names, numbers, dates, quotes, and claims. Never invent, alter, or DROP a fact. Fact-keeping is the top priority and overrides brevity.', 'Write naturally and human: vary sentence length and structure, use a plain confident voice, and make it flow. It must NOT read as AI-generated or templated.', 'Avoid AI tells and stock filler. Do not use phrases like "in conclusion", "furthermore", "moreover", "it is worth noting", "in today\'s world", "delve", "navigate the landscape", "robust", "leverage", "boasts", "a testament to", or formulaic intros/outros. No hedging padding.', - 'Be concise WITHOUT losing information. Shorten by tightening wording and cutting filler, repetition, and throat-clearing — never by removing facts. If a sentence carries a fact, keep the fact; only trim the words around it.', + 'Be concise WITHOUT losing information. Shorten by tightening wording and cutting filler, repetition, and throat-clearing, never by removing facts. If a sentence carries a fact, keep the fact; only trim the words around it.', 'REMOVE promotional cruft: calls-to-action, "subscribe", affiliate lines, ads, author self-promotion, "read more" links.', - 'Keep news as news — neutral, factual tone. Do not editorialize. Do not mention sources, the original article, or that this is a rewrite.', + 'Keep news as news: neutral, factual tone. Do not editorialize. Do not mention sources, the original article, or that this is a rewrite.', 'In "entities", list the notable organizations, people, products, and places the article is about (the main subjects, not every passing mention). For each, give its name, a one-word type (company, person, product, place), and a "description": one or two neutral sentences explaining what it is and its role in this story. Keep descriptions factual; do not invent specific figures.', 'Return ONLY the structured object.', ]; @@ -61,7 +96,7 @@ private function prompt( string $title, string $content, ?string $target_keyword ]; $length = $length ?: $this->settings->article_length(); $rules[] = ( $lengths[ $length ] ?? $lengths['auto'] ) - . ( $length === 'auto' ? '' : ' Treat this as a target, not a hard limit — never drop or shorten facts to hit it.' ); + . ( $length === 'auto' ? '' : ' Treat this as a target, not a hard limit. Never drop or shorten facts to hit it.' ); $custom = $this->settings->writing_instructions(); if ( $custom !== '' ) { diff --git a/src/Cluster/Clusterer.php b/src/Cluster/Clusterer.php index ccd7630..22c1572 100644 --- a/src/Cluster/Clusterer.php +++ b/src/Cluster/Clusterer.php @@ -12,12 +12,15 @@ /** * Decides whether an item belongs to an existing story cluster. Conservative by design: * a false merge corrupts a live post, a false split only leaves a duplicate. Requires - * THREE signals to merge — embedding similarity, an in-window cluster, and at least one - * shared salient fact (the pre-rewrite stand-in for "shared primary entity"). Borderline + * THREE signals to merge: embedding similarity, an in-window cluster, and at least two + * shared salient facts (the pre-rewrite stand-in for "shared primary entity"; one shared + * token is too easy to hit by accident). Borderline * matches are logged and treated as new clusters. */ final class Clusterer { + private const MIN_SHARED_FACTS = 2; + public function __construct( private VectorStore $vectors, private ClusterRepository $clusters, @@ -49,11 +52,11 @@ public function match( array $vector, string $content ): ?int { $threshold ) ); - return null; // sorted desc — nothing below the top will clear it either + return null; // sorted desc; nothing below the top will clear it either } $shared = array_intersect( $item_facts, $this->clusters->fact_set( $candidate['owner_id'] ) ); - if ( count( $shared ) >= 1 ) { + if ( count( $shared ) >= self::MIN_SHARED_FACTS ) { return $candidate['owner_id']; } diff --git a/src/Cluster/Deduplicator.php b/src/Cluster/Deduplicator.php index cd7edc6..47232f3 100644 --- a/src/Cluster/Deduplicator.php +++ b/src/Cluster/Deduplicator.php @@ -4,59 +4,67 @@ use AggregateIt\Settings; use AggregateIt\Support\EventLog; +use AggregateIt\Support\Vector; +use AggregateIt\Vector\VectorStore; defined( 'ABSPATH' ) || exit; /** - * Finds (and optionally trashes) duplicate published posts by comparing their text - * directly — bag-of-words cosine over the title (weighted) and body. Deliberately - * independent of the embedding/vector store, which can silently degrade and is what made - * the earlier vector-based cleanup a no-op. Keeps the oldest post of each group; trashes, - * never deletes. A dry run reports what it would do without changing anything. + * Finds (and optionally trashes) duplicate published posts by comparing the stored + * embedding of each post's story against the others, at the same "Duplicate sensitivity" + * the live clusterer uses. Keeps the oldest post of each group; trashes, never deletes. + * A dry run reports what it would do without changing anything. + * + * The pairwise scan is quadratic, so each call works for a bounded time and saves its + * position. Calling run() again continues until every post has been checked; only then + * are duplicates trashed. */ final class Deduplicator { - private const MAX_POSTS = 3000; + private const MAX_POSTS = 3000; + private const TIME_LIMIT = 15; // seconds per call + private const STATE_KEY = 'aggregate_it_dedup_state'; - public function __construct( private Settings $settings ) {} + public function __construct( private Settings $settings, private VectorStore $vectors ) {} /** - * @return array{groups:int,duplicates:int,examples:string[]} + * @return array{groups:int,duplicates:int,examples:string[],complete:bool,checked:int,total:int} */ - public function run( bool $dry_run = false, float $threshold = 0.40 ): array { - $ids = $this->post_ids(); - $vecs = []; - $norms = []; - foreach ( $ids as $id ) { - $tokens = $this->tokens( get_the_title( $id ), (string) get_post_field( 'post_content', $id ) ); - $vecs[ $id ] = $tokens; - $norms[ $id ] = sqrt( array_sum( array_map( static fn ( $v ) => $v * $v, $tokens ) ) ); + public function run( bool $dry_run = false, ?float $threshold = null ): array { + $threshold = $threshold ?? $this->settings->similarity_threshold(); + + $state = get_transient( self::STATE_KEY ); + if ( ! is_array( $state ) || ( $state['dry_run'] ?? null ) !== $dry_run || ( $state['threshold'] ?? null ) !== $threshold ) { + $state = [ + 'dry_run' => $dry_run, + 'threshold' => $threshold, + 'ids' => $this->post_ids(), + 'cursor' => 0, + 'assigned' => [], + ]; } - $assigned = []; - $groups = []; - $count = count( $ids ); - for ( $i = 0; $i < $count; $i++ ) { - $a = $ids[ $i ]; - if ( isset( $assigned[ $a ] ) ) { - continue; - } - for ( $j = $i + 1; $j < $count; $j++ ) { - $b = $ids[ $j ]; - if ( isset( $assigned[ $b ] ) ) { - continue; - } - if ( $this->cosine( $vecs[ $a ], $norms[ $a ], $vecs[ $b ], $norms[ $b ] ) >= $threshold ) { - $assigned[ $b ] = $a; - $groups[ $a ][] = $b; - } - } + $ids = array_map( 'intval', $state['ids'] ); + $embeddings = $this->embeddings( $ids ); + $assigned = array_map( 'intval', $state['assigned'] ); + $cursor = $this->scan( $ids, $embeddings, $threshold, (int) $state['cursor'], time() + self::TIME_LIMIT, $assigned ); + + if ( $cursor < count( $ids ) ) { + $state['cursor'] = $cursor; + $state['assigned'] = $assigned; + set_transient( self::STATE_KEY, $state, HOUR_IN_SECONDS ); + return [ 'groups' => 0, 'duplicates' => 0, 'examples' => [], 'complete' => false, 'checked' => $cursor, 'total' => count( $ids ) ]; } - $duplicates = 0; - $examples = []; + delete_transient( self::STATE_KEY ); + + $groups = []; + foreach ( $assigned as $dup => $keeper ) { + $groups[ $keeper ][] = $dup; + } + + $examples = []; foreach ( $groups as $keeper => $dups ) { - $duplicates += count( $dups ); if ( count( $examples ) < 10 ) { $examples[] = sprintf( '"%s" +%d', get_the_title( $keeper ), count( $dups ) ); } @@ -67,9 +75,42 @@ public function run( bool $dry_run = false, float $threshold = 0.40 ): array { } } + $duplicates = count( $assigned ); $this->log( $dry_run, $duplicates, count( $groups ), $examples ); - return [ 'groups' => count( $groups ), 'duplicates' => $duplicates, 'examples' => $examples ]; + return [ 'groups' => count( $groups ), 'duplicates' => $duplicates, 'examples' => $examples, 'complete' => true, 'checked' => count( $ids ), 'total' => count( $ids ) ]; + } + + /** + * Compares each unassigned post, from $cursor on, against the later ones until done or + * the deadline passes. Posts without a stored embedding never match. + * + * @param int[] $ids post ids, oldest first (the oldest in a group is kept) + * @param array $embeddings post id => embedding + * @param array $assigned duplicate post id => keeper post id, updated in place + * @return int the cursor to resume from; equals count( $ids ) when finished + */ + public function scan( array $ids, array $embeddings, float $threshold, int $cursor, int $deadline, array &$assigned ): int { + $count = count( $ids ); + for ( $i = $cursor; $i < $count; $i++ ) { + if ( time() >= $deadline ) { + return $i; + } + $a = $ids[ $i ]; + if ( isset( $assigned[ $a ] ) || empty( $embeddings[ $a ] ) ) { + continue; + } + for ( $j = $i + 1; $j < $count; $j++ ) { + $b = $ids[ $j ]; + if ( isset( $assigned[ $b ] ) || empty( $embeddings[ $b ] ) ) { + continue; + } + if ( Vector::cosine( $embeddings[ $a ], $embeddings[ $b ] ) >= $threshold ) { + $assigned[ $b ] = $a; + } + } + } + return $count; } /** @return int[] published plugin posts, oldest first */ @@ -89,39 +130,28 @@ private function post_ids(): array { return array_map( 'intval', $query->posts ); } - /** @return array token => count, title weighted so headlines drive the match */ - private function tokens( string $title, string $content ): array { - $text = strtolower( wp_strip_all_tags( $title . ' ' . $title . ' ' . $title . ' ' . $content ) ); - $words = preg_split( '/[^a-z0-9]+/', $text ) ?: []; - - $counts = []; - foreach ( $words as $word ) { - if ( strlen( $word ) < 4 ) { - continue; - } - $counts[ $word ] = ( $counts[ $word ] ?? 0 ) + 1; - } - return $counts; - } - /** - * @param array $a - * @param array $b + * @param int[] $post_ids + * @return array post id => embedding of the post's story */ - private function cosine( array $a, float $na, array $b, float $nb ): float { - if ( $na === 0.0 || $nb === 0.0 ) { - return 0.0; - } - if ( count( $b ) < count( $a ) ) { - [ $a, $b ] = [ $b, $a ]; + private function embeddings( array $post_ids ): array { + $cluster_of = []; + foreach ( $post_ids as $post_id ) { + $cluster_id = (int) get_post_meta( $post_id, '_ai_cluster_id', true ); + if ( $cluster_id > 0 ) { + $cluster_of[ $post_id ] = $cluster_id; + } } - $dot = 0.0; - foreach ( $a as $token => $count ) { - if ( isset( $b[ $token ] ) ) { - $dot += $count * $b[ $token ]; + + $by_cluster = $this->vectors->get_many( 'cluster', array_values( array_unique( $cluster_of ) ) ); + + $embeddings = []; + foreach ( $cluster_of as $post_id => $cluster_id ) { + if ( isset( $by_cluster[ $cluster_id ] ) ) { + $embeddings[ $post_id ] = $by_cluster[ $cluster_id ]; } } - return $dot / ( $na * $nb ); + return $embeddings; } /** @param string[] $examples */ diff --git a/src/Entity/DelegationRules.php b/src/Entity/DelegationRules.php index 9afc30c..b9f643a 100644 --- a/src/Entity/DelegationRules.php +++ b/src/Entity/DelegationRules.php @@ -7,7 +7,7 @@ /** * The data-driven config that maps an extracted entity type to a custom post type and * tells the engine how to match, research, and link it. Adding support for a new post - * type is adding a rule — no code. Stored as one option. + * type is adding a rule, with no code. Stored as one option. */ final class DelegationRules { diff --git a/src/Entity/EntityLinker.php b/src/Entity/EntityLinker.php index 77f0e8e..6c2c03d 100644 --- a/src/Entity/EntityLinker.php +++ b/src/Entity/EntityLinker.php @@ -6,7 +6,7 @@ /** * Injects a single contextual link on the first mention of each entity (capped per post) - * and records the bidirectional relationship as `_ai_entity` meta — which powers the + * and records the bidirectional relationship as `_ai_entity` meta, which powers the * hub's auto-growing related-posts list and the article's schema about/mentions. */ final class EntityLinker { diff --git a/src/Entity/EntityRepository.php b/src/Entity/EntityRepository.php index 04ad5b2..2aef110 100644 --- a/src/Entity/EntityRepository.php +++ b/src/Entity/EntityRepository.php @@ -96,7 +96,7 @@ public function set_fields( int $id, array $fields ): void { } } - /** Record that a post mentioned this entity — builds the hub's "In the news" timeline. */ + /** Record that a post mentioned this entity; builds the hub's "In the news" timeline. */ public function add_timeline( int $id, int $post_id, string $note ): void { if ( ! $post_id ) { return; diff --git a/src/Entity/EntityResearcher.php b/src/Entity/EntityResearcher.php index 38e1056..d6d15c3 100644 --- a/src/Entity/EntityResearcher.php +++ b/src/Entity/EntityResearcher.php @@ -11,7 +11,7 @@ /** * Builds the data for a new entity stub. Degrades gracefully: with no research provider * the stub is built from in-article context only (zero external dependency); when a - * provider is registered, it adds cited external facts + sameAs. Stubs stay strict — only + * provider is registered, it adds cited external facts + sameAs. Stubs stay strict: only * cited information, never confidently-filled hallucination. */ final class EntityResearcher { @@ -51,7 +51,7 @@ public function fields( array $rule, string $name, string $type, string $context $prompt = sprintf( "For the %s \"%s\", fill in these details from the article and well-known facts. " - . "Leave a field empty if you are not confident — do not guess. Fields: %s.\n\nARTICLE:\n%s", + . "Leave a field empty if you are not confident. Do not guess. Fields: %s.\n\nARTICLE:\n%s", $type !== '' ? $type : 'subject', $name, implode( ', ', $labels ), diff --git a/src/Entity/EntityResolver.php b/src/Entity/EntityResolver.php index 9600943..3d53eb1 100644 --- a/src/Entity/EntityResolver.php +++ b/src/Entity/EntityResolver.php @@ -10,7 +10,7 @@ * Type-constrained, conservative resolution. Exact/alias or high fuzzy similarity links; * clearly novel creates; the ambiguous middle is skipped and logged rather than guessed. * A clean sparse graph outranks a dense corrupted one, and duplicates are recoverable - * via the merge tool — wrong links are not. + * via the merge tool; wrong links are not. */ final class EntityResolver { @@ -55,7 +55,7 @@ public function resolve( array $rule, string $name ): array { } if ( $best_id && $best_score >= $ambiguous_floor ) { - EventLog::warning( sprintf( 'Not sure which page "%s" belongs to (%.0f%% match with page #%d) — skipped it for now.', $name, $best_score, $best_id ) ); + EventLog::warning( sprintf( 'Not sure which page "%s" belongs to (%.0f%% match with page #%d). Skipped it for now.', $name, $best_score, $best_id ) ); return [ 'action' => 'skip', 'entity_id' => null ]; } diff --git a/src/Entity/HubRenderer.php b/src/Entity/HubRenderer.php index e8b11af..cbe244c 100644 --- a/src/Entity/HubRenderer.php +++ b/src/Entity/HubRenderer.php @@ -9,7 +9,7 @@ /** * Renders entity hub pages as pillar pages: Organization/Person/Product schema with * sameAs, plus an auto-growing "Related coverage" list of every post that mentions the - * entity. Each new post strengthens the hub — the topical-authority + freshness flywheel. + * entity. Each new post strengthens the hub: the topical-authority + freshness flywheel. */ final class HubRenderer { diff --git a/src/Keyword/KeywordStrategy.php b/src/Keyword/KeywordStrategy.php index df39dd3..61f83f4 100644 --- a/src/Keyword/KeywordStrategy.php +++ b/src/Keyword/KeywordStrategy.php @@ -8,17 +8,50 @@ /** * Resolves the target keyword for a cluster. Layer 1 (default): use the AI-inferred - * keyword (volume mode — publish everything). Layer 2 (optional): map to a configured + * keyword (volume mode, publish everything). Layer 2 (optional): map to a configured * keyword list; in strategic mode, content that matches no target keyword is skipped. */ final class KeywordStrategy { public function __construct( private Settings $settings ) {} + /** Optional keyword-data add-on registered through `aggregate_it_keyword_provider`. */ + public function provider(): ?KeywordProvider { + $provider = apply_filters( 'aggregate_it_keyword_provider', null, $this->settings ); + return $provider instanceof KeywordProvider ? $provider : null; + } + /** * @return array{keyword:string,skip:bool} */ public function resolve( string $inferred, string $content ): array { + $decision = $this->match( $inferred, $content ); + + if ( ! $decision['skip'] && $this->settings->strategic_mode() && $this->has_no_demand( $decision['keyword'] ) ) { + $decision['skip'] = true; + } + + return $decision; + } + + /** True only when a provider is installed and reports zero search volume for the keyword. */ + private function has_no_demand( string $keyword ): bool { + $provider = $this->provider(); + if ( ! $provider || $keyword === '' ) { + return false; + } + try { + $metrics = $provider->metrics( [ $keyword ] ); + } catch ( \Throwable $e ) { + return false; + } + return isset( $metrics[ $keyword ]['volume'] ) && (int) $metrics[ $keyword ]['volume'] === 0; + } + + /** + * @return array{keyword:string,skip:bool} + */ + private function match( string $inferred, string $content ): array { $list = $this->settings->keyword_list(); if ( ! $list ) { diff --git a/src/Maintenance/GlobalRulesRefresher.php b/src/Maintenance/GlobalRulesRefresher.php index f578c33..5a5f821 100644 --- a/src/Maintenance/GlobalRulesRefresher.php +++ b/src/Maintenance/GlobalRulesRefresher.php @@ -12,7 +12,7 @@ /** * Applies global per-post-type rules to every post of each configured type, reading the * post's own meta as the field values. Runs daily (and on demand after a save) so derived - * meta like status stays correct as dates pass — for any post type, not just scraped ones. + * meta like status stays correct as dates pass, for any post type, not just scraped ones. */ final class GlobalRulesRefresher { diff --git a/src/Maintenance/Retention.php b/src/Maintenance/Retention.php index 0b88d91..716f104 100644 --- a/src/Maintenance/Retention.php +++ b/src/Maintenance/Retention.php @@ -10,7 +10,7 @@ /** * Daily housekeeping: drop old processed/failed item rows and old log rows so the - * tables don't grow without bound. Published posts are never touched — only the + * tables don't grow without bound. Published posts are never touched; only the * internal tracking rows behind the Articles list and the cost log. */ final class Retention { diff --git a/src/Pipeline/ComposeStage.php b/src/Pipeline/ComposeStage.php index 73cdc35..091ef9f 100644 --- a/src/Pipeline/ComposeStage.php +++ b/src/Pipeline/ComposeStage.php @@ -3,6 +3,8 @@ namespace AggregateIt\Pipeline; use AggregateIt\Ai\FactsGuard; +use AggregateIt\Ai\ProviderFactory; +use AggregateIt\Ai\ProviderNotConfigured; use AggregateIt\Ai\Rewriter; use AggregateIt\Cluster\Clusterer; use AggregateIt\Cluster\ClusterRepository; @@ -31,6 +33,7 @@ final class ComposeStage implements PaidStage { public function __construct( + private ProviderFactory $providers, private Rewriter $rewriter, private FactsGuard $facts, private KeywordStrategy $keywords, @@ -102,6 +105,26 @@ private function length( Item $item ): ?string { return $len !== '' ? $len : null; } + /** @throws ProviderNotConfigured */ + private function require_provider( Item $item ): void { + $reason = $this->providers->blocked_reason(); + if ( $reason === null ) { + return; + } + + // Log once per reason so a held queue does not flood the activity log every retry. + if ( get_transient( ProviderFactory::BLOCKED_TRANSIENT ) !== $reason ) { + EventLog::warning( + $reason === 'undecryptable' + ? 'Articles are on hold: the saved AI API key cannot be decrypted. Enter the key again in Settings.' + : 'Articles are on hold: no AI service is configured. Choose a service and add its API key in Settings.' + ); + } + set_transient( ProviderFactory::BLOCKED_TRANSIENT, $reason, DAY_IN_SECONDS ); + + throw new ProviderNotConfigured( sprintf( 'Article #%d held: no usable AI provider (%s).', $item->id, $reason ) ); + } + public function process( Item $item ): string { if ( ! empty( $item->flags['passthrough'] ) ) { return $this->publish_mapped( $item ); @@ -113,6 +136,8 @@ public function process( Item $item ): string { return Schema::STATE_ENTITY_LINKED; } + $this->require_provider( $item ); + // Clusters are created lazily here (compose), so a burst of the same breaking story // can all clear ClusterStage before any cluster exists and each publish a duplicate. // Re-match now that siblings may have created their cluster, folding them into one @@ -157,6 +182,8 @@ private function create( Item $item ): string { } $keyword = $decision['keyword']; + $invented = $this->facts->invented( $content, (string) ( $structured['rewritten_body'] ?? '' ) ); + $vector = $this->vectors->get( 'item', $item->id ); $cluster_id = null; @@ -175,18 +202,17 @@ private function create( Item $item ): string { $item->flags['entities'] = (array) ( $structured['entities'] ?? [] ); - $post_id = $this->posts->create( $cluster_id, $item, $structured, $keyword, [ $item->url ] ); + $post_id = $this->posts->create( $cluster_id, $item, $structured, $keyword, [ $item->url ], $invented !== [] ); $this->clusters->set_canonical_post( $cluster_id, $post_id ); $this->items->set_post( $item->id, $post_id ); $item->post_id = $post_id; $item->flags['post_id'] = $post_id; - $invented = $this->facts->invented( $content, (string) ( $structured['rewritten_body'] ?? '' ) ); if ( $invented ) { $item->flags['invented'] = $invented; update_post_meta( $post_id, '_ai_invented', $invented ); - EventLog::warning( sprintf( 'Post #%d: possible made-up numbers: %s', $post_id, implode( ', ', $invented ) ) ); + EventLog::warning( sprintf( 'Post #%d saved as a draft: the rewrite has numbers not in the source (%s). Review it before publishing.', $post_id, implode( ', ', $invented ) ) ); } $this->images->maybe_import( $post_id, (string) ( $item->flags['image'] ?? '' ), (string) ( $structured['seo_title'] ?? '' ) ); @@ -213,22 +239,32 @@ private function update( Item $item ): string { if ( ! $cluster || ! $cluster->canonical_post_id ) { $item->flags['suppressed'] = 'no-canonical-post'; - $this->suppressed( $item, 'no-canonical-post', sprintf( 'Article #%d could not update its story — the canonical post is missing.', $item->id ) ); + $this->suppressed( $item, 'no-canonical-post', sprintf( 'Article #%d could not update its story: the canonical post is missing.', $item->id ) ); return Schema::STATE_ENTITY_LINKED; } if ( ! $novel ) { $item->flags['suppressed'] = 'no-novelty'; - $this->suppressed( $item, 'no-novelty', sprintf( 'Article #%d folded into an existing story with no new facts — nothing added.', $item->id ) ); + $this->suppressed( $item, 'no-novelty', sprintf( 'Article #%d folded into an existing story with no new facts. Nothing added.', $item->id ) ); return Schema::STATE_ENTITY_LINKED; } - $rewrite = $this->rewriter->rewrite( (string) ( $item->flags['title'] ?? '' ), $content, null, $this->length( $item ) ); - $structured = $rewrite['result']; + $rewrite = $this->rewriter->rewrite_update( (string) ( $item->flags['title'] ?? '' ), $content, $novel ); + $structured = $rewrite['result']; + $update_body = (string) ( $structured['rewritten_body'] ?? '' ); $this->cost->record( Schema::STATE_CLUSTERED, $rewrite['tokens'], $rewrite['cost_usd'], $item->id ); - $post_id = (int) $cluster->canonical_post_id; - $this->posts->append_update( $post_id, (string) ( $structured['rewritten_body'] ?? '' ), $item->url ); + $post_id = (int) $cluster->canonical_post_id; + $invented = $this->facts->invented( $content, $update_body ); + if ( $invented ) { + $item->flags['suppressed'] = 'invented-facts'; + $item->flags['invented'] = $invented; + $this->suppressed( $item, 'invented-facts', sprintf( 'Update for post #%d skipped: it has numbers not in the source (%s).', $post_id, implode( ', ', $invented ) ) ); + EventLog::warning( sprintf( 'Post #%d not updated: the generated update has numbers not in the source (%s).', $post_id, implode( ', ', $invented ) ) ); + return Schema::STATE_ENTITY_LINKED; + } + + $this->posts->append_update( $post_id, $update_body, $item->url ); $this->clusters->merge_facts( (int) $item->cluster_id, $novel ); $this->items->set_post( $item->id, $post_id ); diff --git a/src/Pipeline/EntityStage.php b/src/Pipeline/EntityStage.php index c30aee8..cdd75cb 100644 --- a/src/Pipeline/EntityStage.php +++ b/src/Pipeline/EntityStage.php @@ -126,7 +126,7 @@ public function process( Item $item ): string { if ( $skipped ) { ActivityLog::record( 'info', - sprintf( 'Post #%d: skipped %d topic(s) — %s.', $post_id, count( $skipped ), implode( ', ', array_map( static fn ( $s ) => $s['name'] . ' (' . $s['reason'] . ')', $skipped ) ) ), + sprintf( 'Post #%d: skipped %d topic(s): %s.', $post_id, count( $skipped ), implode( ', ', array_map( static fn ( $s ) => $s['name'] . ' (' . $s['reason'] . ')', $skipped ) ) ), [ 'item_id' => $item->id, 'source_id' => $item->source_id, diff --git a/src/Pipeline/ExtractStage.php b/src/Pipeline/ExtractStage.php index b5dc47e..1a9c5e3 100644 --- a/src/Pipeline/ExtractStage.php +++ b/src/Pipeline/ExtractStage.php @@ -49,7 +49,7 @@ public function process( Item $item ): string { $item->flags['content_length'] = mb_strlen( $result['content'] ); $item->flags['thin'] = mb_strlen( $result['content'] ) < self::MIN_CONTENT; - // Prefer the publisher's share image (og:image) over the feed enclosure — the + // Prefer the publisher's share image (og:image) over the feed enclosure, the // enclosure is often a generic or sub-topic image, not the article's hero. A // transient fetch failure is rethrown for the first couple of attempts so the queue // retries (and the rate-limit gap clears) rather than freezing the post image-less. diff --git a/src/Pipeline/PaidStage.php b/src/Pipeline/PaidStage.php index 400e878..b658b2b 100644 --- a/src/Pipeline/PaidStage.php +++ b/src/Pipeline/PaidStage.php @@ -6,7 +6,7 @@ /** * Marker for stages that spend money (AI calls). When the daily spend cap is reached the - * worker defers these instead of running them — free stages keep flowing. + * worker defers these instead of running them; free stages keep flowing. */ interface PaidStage extends Stage { } diff --git a/src/Pipeline/Stage.php b/src/Pipeline/Stage.php index dbfc653..940dcd8 100644 --- a/src/Pipeline/Stage.php +++ b/src/Pipeline/Stage.php @@ -7,7 +7,7 @@ /** * A single pipeline stage. Each stage handles exactly one input state and returns the * next state. Throwing signals a retryable failure (the queue backs off / dead-letters). - * Stages must be idempotent — they may be re-run after a crash mid-claim. + * Stages must be idempotent; they may be re-run after a crash mid-claim. */ interface Stage { diff --git a/src/Plugin.php b/src/Plugin.php index 7076cff..0cd4359 100644 --- a/src/Plugin.php +++ b/src/Plugin.php @@ -102,7 +102,7 @@ public function boot(): void { $this->pipeline->register( new EmbedStage( $this->providers, $vectors, $this->cost ) ); $this->pipeline->register( new ClusterStage( $clusterer, $vectors, $this->items ) ); $this->pipeline->register( - new ComposeStage( $rewriter, $facts, $keywords, $cluster_repo, $post_factory, $this->imageImporter(), $related, $seo, $vectors, $this->items, $this->cost, $this->settings, $categories, $clusterer ) + new ComposeStage( $this->providers, $rewriter, $facts, $keywords, $cluster_repo, $post_factory, $this->imageImporter(), $related, $seo, $vectors, $this->items, $this->cost, $this->settings, $categories, $clusterer ) ); $this->pipeline->register( new EntityStage( @@ -210,7 +210,7 @@ public function seo(): Seo { } public function reprocessor(): Reprocessor { - return new Reprocessor( $this->rewriter(), $this->seo() ); + return new Reprocessor( $this->rewriter(), $this->seo(), $this->providers ); } public function imageImporter(): ImageImporter { @@ -218,7 +218,7 @@ public function imageImporter(): ImageImporter { } public function deduplicator(): Deduplicator { - return new Deduplicator( $this->settings ); + return new Deduplicator( $this->settings, new VectorStore() ); } public function seed_enabled(): bool { diff --git a/src/Publish/CategoryResolver.php b/src/Publish/CategoryResolver.php index 8ad82c3..22fc987 100644 --- a/src/Publish/CategoryResolver.php +++ b/src/Publish/CategoryResolver.php @@ -8,7 +8,7 @@ /** * Maps an AI-suggested category name to a real category term, creating it on first use. - * Matching is exact by name/slug — substring matching sends unrelated posts to the wrong + * Matching is exact by name/slug; substring matching sends unrelated posts to the wrong * category. Generic buckets are hidden from the AI's options so it classifies by topic. */ final class CategoryResolver { diff --git a/src/Publish/FieldMapper.php b/src/Publish/FieldMapper.php index 3a90a65..2455cd4 100644 --- a/src/Publish/FieldMapper.php +++ b/src/Publish/FieldMapper.php @@ -6,8 +6,8 @@ /** * Routes extracted field values to WordPress post destinations. Each field maps to a - * destination string — post_title / post_content / post_excerpt / post_date / - * featured_image / taxonomy: / meta: — so any scraped shape (events, jobs, + * destination string: post_title / post_content / post_excerpt / post_date / + * featured_image / taxonomy: / meta:, so any scraped shape (events, jobs, * listings) lands on any post type without site-specific code. Unmapped fields fall back * to a sensible default (standard fields to their obvious slot, everything else to meta). */ diff --git a/src/Publish/ImageImporter.php b/src/Publish/ImageImporter.php index c3d316a..e9df5bf 100644 --- a/src/Publish/ImageImporter.php +++ b/src/Publish/ImageImporter.php @@ -9,7 +9,7 @@ /** * Imports a feed/article image into the media library (never hotlinked) and sets it as - * the featured image with generated alt text. Configurable off/import — default + * the featured image with generated alt text. Configurable off/import; default * conservative. Source images carry copyright risk, so this is opt-out-able. */ final class ImageImporter { @@ -60,7 +60,7 @@ public function maybe_import( int $post_id, string $image_url, string $alt, bool /** * Download first, then name the file from its real MIME. WordPress's media_sideload_image() * derives the extension from the URL and rejects extensionless URLs ("Invalid image URL.") - * before downloading — but most publishers serve og:image via extensionless CDN/proxy URLs + * before downloading, but most publishers serve og:image via extensionless CDN/proxy URLs * (imgix, Cloudinary, thumbor, signed URLs), which is why featured images go missing. */ private function sideload( string $url, int $post_id, string $alt ): int { diff --git a/src/Publish/PostFactory.php b/src/Publish/PostFactory.php index e7ecaea..07e7296 100644 --- a/src/Publish/PostFactory.php +++ b/src/Publish/PostFactory.php @@ -12,6 +12,8 @@ final class PostFactory { + private const UPDATE_MAX_WORDS = 120; + public function __construct( private Settings $settings, private SlugGenerator $slugs, @@ -22,8 +24,9 @@ public function __construct( /** * @param array $structured * @param string[] $source_urls + * @param bool $force_draft hold the post for review whatever the publish setting says */ - public function create( int $cluster_id, Item $item, array $structured, string $keyword, array $source_urls ): int { + public function create( int $cluster_id, Item $item, array $structured, string $keyword, array $source_urls, bool $force_draft = false ): int { $title = (string) ( $structured['seo_title'] ?? $item->flags['title'] ?? 'Untitled' ); $slug = (string) ( $structured['slug'] ?? '' ); $slug = $this->slugs->generate( $slug !== '' ? $slug : $keyword, $title ); @@ -31,6 +34,7 @@ public function create( int $cluster_id, Item $item, array $structured, string $ $source = $item->source_id ? $this->sources->get( $item->source_id ) : null; $status = $source ? $source->publish_status( $this->settings->publish_status() ) : $this->settings->publish_status(); $post_type = $source && $source->post_type_connection() !== '' ? $source->post_type_connection() : $this->settings->target_post_type(); + $status = $force_draft ? 'draft' : $status; $post_id = wp_insert_post( [ @@ -186,11 +190,11 @@ public function append_update( int $post_id, string $update_body, string $source $heading = sprintf( /* translators: %s: human-readable date */ - __( 'Update — %s', 'aggregate-it' ), + __( 'Update: %s', 'aggregate-it' ), date_i18n( get_option( 'date_format' ) ) ); - $section = "\n\n\n

" . esc_html( $heading ) . "

\n" . $this->body( $update_body ); + $section = "\n\n\n

" . esc_html( $heading ) . "

\n" . $this->short_body( $update_body ); wp_update_post( [ @@ -205,6 +209,26 @@ public function append_update( int $post_id, string $update_body, string $source update_post_meta( $post_id, '_ai_source_urls', array_values( array_unique( array_filter( $urls ) ) ) ); } + /** Caps an update at UPDATE_MAX_WORDS so a live post grows by a note, not a second article. */ + private function short_body( string $text ): string { + $budget = self::UPDATE_MAX_WORDS; + $html = ''; + foreach ( preg_split( '/\n{2,}/', trim( $text ) ) ?: [] as $p ) { + $words = preg_split( '/\s+/', trim( $p ), -1, PREG_SPLIT_NO_EMPTY ) ?: []; + if ( ! $words ) { + continue; + } + $clipped = count( $words ) > $budget; + $p = implode( ' ', array_slice( $words, 0, $budget ) ) . ( $clipped ? '…' : '' ); + $html .= '

' . esc_html( $p ) . "

\n"; + $budget -= count( $words ); + if ( $budget <= 0 ) { + break; + } + } + return $html; + } + private function body( string $text ): string { $paragraphs = preg_split( '/\n{2,}/', trim( $text ) ) ?: []; $html = ''; diff --git a/src/Publish/RelatedArticles.php b/src/Publish/RelatedArticles.php index 4546484..13ee20a 100644 --- a/src/Publish/RelatedArticles.php +++ b/src/Publish/RelatedArticles.php @@ -10,7 +10,7 @@ /** * Automatic internal linking between articles. On publish, finds the most semantically - * similar other stories and cross-links their posts — stored as `_ai_related` meta + * similar other stories and cross-links their posts, stored as `_ai_related` meta * (both directions, no content churn) and rendered live as a "Related articles" list. */ final class RelatedArticles { diff --git a/src/Publish/Reprocessor.php b/src/Publish/Reprocessor.php index a3b5bfd..a6d62f2 100644 --- a/src/Publish/Reprocessor.php +++ b/src/Publish/Reprocessor.php @@ -2,26 +2,33 @@ namespace AggregateIt\Publish; +use AggregateIt\Ai\ProviderFactory; use AggregateIt\Ai\Rewriter; use AggregateIt\Seo\Seo; defined( 'ABSPATH' ) || exit; /** - * Re-runs an already-published post through the rewriter again — with the current model, - * writing style, and prompt — without re-importing. Uses the stored original article + * Re-runs an already-published post through the rewriter again, with the current model, + * writing style, and prompt, without re-importing. Uses the stored original article * text. The slug is left unchanged so the URL stays stable. */ final class Reprocessor { public function __construct( private Rewriter $rewriter, - private Seo $seo + private Seo $seo, + private ProviderFactory $providers ) {} + /** True when no real AI provider is usable, so a rewrite would echo the source text. */ + public function blocked(): bool { + return $this->providers->blocked_reason() !== null; + } + public function reprocess( int $post_id ): bool { $original = (string) get_post_meta( $post_id, '_ai_original', true ); - if ( $original === '' ) { + if ( $original === '' || $this->blocked() ) { return false; } diff --git a/src/Queue/ItemStore.php b/src/Queue/ItemStore.php index c61eb68..51e56f5 100644 --- a/src/Queue/ItemStore.php +++ b/src/Queue/ItemStore.php @@ -199,7 +199,7 @@ public function purge_old( int $days ): int { /** * Delete not-yet-published items that are stale: those whose stored publish date is * older than $cutoff_ts, plus legacy rows imported before publish-date tracking existed - * (no `published_at` key) — that's the backlog this is meant to clear. Items imported + * (no `published_at` key); that's the backlog this is meant to clear. Items imported * since, with an unknown date (published_at = 0), are kept since freshness can't be told. */ public function purge_stale_queued( int $cutoff_ts ): int { @@ -386,7 +386,7 @@ private function where_for( ?string $status, string $search = '' ): array { } if ( $search !== '' ) { - // Match the published post (its AI title/body) first — that's what users look for — + // Match the published post (its AI title/body) first, since users look for it, // then the source URL and the original feed title (which lives in the flags JSON). $like = '%' . $wpdb->esc_like( $search ) . '%'; $where .= " AND ( post_id IN ( SELECT ID FROM {$wpdb->posts} WHERE post_title LIKE %s OR post_content LIKE %s ) OR url LIKE %s OR flags LIKE %s )"; diff --git a/src/Queue/QueueWorker.php b/src/Queue/QueueWorker.php index 3b4cf92..9141f4f 100644 --- a/src/Queue/QueueWorker.php +++ b/src/Queue/QueueWorker.php @@ -2,6 +2,7 @@ namespace AggregateIt\Queue; +use AggregateIt\Ai\ProviderNotConfigured; use AggregateIt\Cost\CostMeter; use AggregateIt\Cost\SpendCap; use AggregateIt\Pipeline\Item; @@ -48,7 +49,7 @@ public function register(): void { /** * Background tick (every minute). Runs only when automatic processing is on, and at - * most once per the configured interval — the transient gates the cadence without + * most once per the configured interval; the transient gates the cadence without * rescheduling cron, so interval changes take effect immediately. */ public function cron_tick(): void { @@ -143,9 +144,12 @@ private function process( Item $item ): void { $next = $stage->process( $item ); $this->items->advance( $item->id, $next, $item->flags ); } catch ( RateLimited $e ) { - // Politeness gap, not a failure — re-queue (no attempt consumed) with jitter so + // Politeness gap, not a failure: re-queue (no attempt consumed) with jitter so // same-host items spread out past the per-host window instead of re-colliding. $this->items->defer( $item->id, 5 + wp_rand( 0, 10 ) ); + } catch ( ProviderNotConfigured $e ) { + // Not a failure of the item: hold it until the site owner configures a provider. + $this->items->defer( $item->id, 15 * MINUTE_IN_SECONDS ); } catch ( \Throwable $e ) { $this->fail( $item, $attempts, $e ); } diff --git a/src/Research/ResearchProvider.php b/src/Research/ResearchProvider.php index 98b0615..754110d 100644 --- a/src/Research/ResearchProvider.php +++ b/src/Research/ResearchProvider.php @@ -5,7 +5,7 @@ defined( 'ABSPATH' ) || exit; /** - * Optional entity enrichment. Absent by default — entities are then created from + * Optional entity enrichment. Absent by default; entities are then created from * in-article context only. A premium add-on registers an implementation via the * `aggregate_it_research_provider` filter to supply cited external facts + sameAs URLs. */ diff --git a/src/Seo/IndexNow.php b/src/Seo/IndexNow.php index 7fc1829..2bcc084 100644 --- a/src/Seo/IndexNow.php +++ b/src/Seo/IndexNow.php @@ -3,6 +3,7 @@ namespace AggregateIt\Seo; use AggregateIt\Settings; +use AggregateIt\Support\EventLog; defined( 'ABSPATH' ) || exit; @@ -40,14 +41,13 @@ public function ping( int $post_id ): void { } $url = get_permalink( $post_id ); - if ( ! $url ) { + if ( ! $url || get_post_status( $post_id ) !== 'publish' ) { return; } - $key = $this->key(); - $host = wp_parse_url( home_url(), PHP_URL_HOST ); + $key = $this->key(); - wp_remote_get( + $response = wp_remote_get( add_query_arg( [ 'url' => rawurlencode( $url ), @@ -56,8 +56,18 @@ public function ping( int $post_id ): void { ], self::ENDPOINT ), - [ 'blocking' => false, 'timeout' => 2, 'headers' => [ 'Host' => $host ] ] + [ 'timeout' => 5 ] ); + + if ( is_wp_error( $response ) ) { + EventLog::warning( sprintf( 'IndexNow ping for post #%d failed: %s', $post_id, $response->get_error_message() ) ); + return; + } + + $code = (int) wp_remote_retrieve_response_code( $response ); + if ( $code < 200 || $code > 299 ) { + EventLog::warning( sprintf( 'IndexNow rejected post #%d with HTTP %d.', $post_id, $code ) ); + } } public function key(): string { diff --git a/src/Seo/SlugGenerator.php b/src/Seo/SlugGenerator.php index 9b1bccb..ad95e10 100644 --- a/src/Seo/SlugGenerator.php +++ b/src/Seo/SlugGenerator.php @@ -5,7 +5,7 @@ defined( 'ABSPATH' ) || exit; /** - * Flat, keyword-based slugs — no dates (living posts get updated; a date would make them + * Flat, keyword-based slugs, no dates (living posts get updated; a date would make them * read as stale and dilute the keyword). Derived from the target keyword, not the * clickbait source headline. Uniqueness is left to wp_insert_post. */ diff --git a/src/Settings.php b/src/Settings.php index a94e7f4..7d959b1 100644 --- a/src/Settings.php +++ b/src/Settings.php @@ -36,7 +36,7 @@ public function update( array $values ): void { } public function provider_key(): string { - return (string) $this->get( 'provider', 'mock' ); + return (string) $this->get( 'provider', 'none' ); } public function ai_model(): string { @@ -52,6 +52,7 @@ public function voyage_api_key(): string { return $stored === '' ? '' : Crypto::decrypt( $stored ); } + /** @throws \RuntimeException when the key cannot be encrypted on this site */ public function set_voyage_api_key( string $key ): void { $this->set( 'voyage_api_key', $key === '' ? '' : Crypto::encrypt( $key ) ); } @@ -61,10 +62,30 @@ public function api_key(): string { return $stored === '' ? '' : Crypto::decrypt( $stored ); } + /** @throws \RuntimeException when the key cannot be encrypted on this site */ public function set_api_key( string $key ): void { $this->set( 'api_key', $key === '' ? '' : Crypto::encrypt( $key ) ); } + /** True when a key is saved but cannot be decrypted (salts changed, or corrupted value). */ + public function api_key_unreadable(): bool { + return (string) $this->get( 'api_key', '' ) !== '' && $this->api_key() === ''; + } + + /** Re-encrypts keys saved by older releases with the current scheme. */ + public function upgrade_key_encryption(): void { + foreach ( [ 'api_key', 'voyage_api_key' ] as $field ) { + $stored = (string) $this->get( $field, '' ); + if ( $stored === '' || Crypto::is_current( $stored ) || ! Crypto::available() ) { + continue; + } + $plain = Crypto::decrypt( $stored ); + if ( $plain !== '' ) { + $this->set( $field, Crypto::encrypt( $plain ) ); + } + } + } + public function brand_name(): string { $name = trim( (string) $this->get( 'brand_name', '' ) ); return $name !== '' ? $name : __( 'Aggregate It', 'aggregate-it' ); diff --git a/src/Source/ContentExtractor.php b/src/Source/ContentExtractor.php index 67576be..7dac001 100644 --- a/src/Source/ContentExtractor.php +++ b/src/Source/ContentExtractor.php @@ -38,7 +38,7 @@ public function extract( string $url, string $feed_content, int $feed_threshold } /** - * The publisher's chosen share image for this article — the correct hero in almost + * The publisher's chosen share image for this article: the correct hero in almost * all cases (feed enclosures are often generic/sub-topic images). Fetches the page * (cached + polite) and returns the best meta image, or '' on any failure. */ @@ -49,10 +49,10 @@ public function share_image( string $url, bool $rethrow = false ): string { try { $html = $this->fetcher->fetch( $url ); } catch ( RateLimited $e ) { - throw $e; // always retriable — let the queue defer and re-fetch once the host clears + throw $e; // always retriable; let the queue defer and re-fetch once the host clears } catch ( \Throwable $e ) { if ( $rethrow ) { - throw $e; // transient HTTP/network blip — let the queue retry rather than lose the image + throw $e; // transient HTTP/network blip; let the queue retry rather than lose the image } return ''; } @@ -62,7 +62,7 @@ public function share_image( string $url, bool $rethrow = false ): string { private function best_meta_image( string $html ): string { foreach ( [ 'og:image:secure_url', 'og:image', 'twitter:image', 'twitter:image:src' ] as $key ) { $image = $this->meta_content( $html, $key ); - // Trust the publisher's explicit share tag — it's a deliberate per-article choice. + // Trust the publisher's explicit share tag; it's a deliberate per-article choice. // Only the junk heuristic is reserved for the guessed fallbacks below. if ( $image !== '' && strpos( $image, 'data:' ) !== 0 ) { return $image; @@ -133,7 +133,7 @@ private function is_junk_image( string $url ): bool { if ( strpos( $url, 'data:' ) === 0 ) { return true; } - // Match the filename only — matching the whole URL false-positives on hosts/paths + // Match the filename only: matching the whole URL false-positives on hosts/paths // like siliconangle.com ("icon"), dropping every valid image from that source. $name = basename( (string) wp_parse_url( $url, PHP_URL_PATH ) ); return (bool) preg_match( '/(^|[-_.])(logo|icon|sprite|avatar|gravatar|placeholder|default|blank|spacer|favicon)([-_.]|$)/i', $name ); diff --git a/src/Source/HttpFetcher.php b/src/Source/HttpFetcher.php index d95c95a..2863a34 100644 --- a/src/Source/HttpFetcher.php +++ b/src/Source/HttpFetcher.php @@ -61,7 +61,7 @@ public function fetch( string $url, bool $respect_robots = true, bool $throttle $code = (int) wp_remote_retrieve_response_code( $response ); if ( $code >= 400 ) { // Only negative-cache permanently-gone pages. Caching transient blocks (403/ - // 429/5xx) would make every retry — and the manual refresh — read an empty body + // 429/5xx) would make every retry, and the manual refresh, read an empty body // for an hour, freezing the post without the og:image instead of recovering. if ( in_array( $code, [ 404, 410 ], true ) ) { set_transient( $cache_key, '', self::CACHE_TTL ); diff --git a/src/Source/Importer.php b/src/Source/Importer.php index a1e7848..8fd998c 100644 --- a/src/Source/Importer.php +++ b/src/Source/Importer.php @@ -10,7 +10,7 @@ defined( 'ABSPATH' ) || exit; /** - * Pulls feeds and enqueues new items in the `fetched` state. Import stays lightweight — + * Pulls feeds and enqueues new items in the `fetched` state. Import stays lightweight, * it only parses the feed and stores the feed-provided content; the heavier per-article * extraction happens later in ExtractStage, where it gets retry/backoff/rate-limiting. */ @@ -327,7 +327,7 @@ private function usable_image_url( string $url ): bool { return false; } - // Match the filename only, not the whole URL — a host substring like "silicon" + // Match the filename only, not the whole URL: a host substring like "silicon" // (siliconangle.com) would otherwise drop every image from that feed. if ( preg_match( '/(^|[-_.])(logo|icon|sprite|avatar|gravatar|placeholder|default|blank|spacer|favicon)([-_.]|$)/i', basename( $path ) ) ) { return false; diff --git a/src/Source/Parser/ScraperParser.php b/src/Source/Parser/ScraperParser.php index 38a5366..af1616e 100644 --- a/src/Source/Parser/ScraperParser.php +++ b/src/Source/Parser/ScraperParser.php @@ -10,8 +10,8 @@ defined( 'ABSPATH' ) || exit; /** - * Generalized HTML scraper. At import time it does exactly one fetch — the listing or - * sitemap page — and turns repeating rows into normalized entries; per-item detail-page + * Generalized HTML scraper. At import time it does exactly one fetch (the listing or + * sitemap page) and turns repeating rows into normalized entries; per-item detail-page * fetches are deliberately left to the pipeline (ExtractStage), where throttle/defer and * retry already apply. Nothing here is specific to any site: rows, fields, and their * destinations all come from the source's scrape config. diff --git a/src/Source/RateLimited.php b/src/Source/RateLimited.php index f4101e0..11d8cad 100644 --- a/src/Source/RateLimited.php +++ b/src/Source/RateLimited.php @@ -7,6 +7,6 @@ /** * Thrown when a fetch is skipped purely to honor the per-host rate limit. Distinguishable * from a genuine failure so the queue can defer (retry without consuming an attempt) - * instead of giving up — otherwise same-host article bursts lose their og:image. + * instead of giving up; otherwise same-host article bursts lose their og:image. */ final class RateLimited extends \RuntimeException {} diff --git a/src/Source/Scrape/SelectorAssistant.php b/src/Source/Scrape/SelectorAssistant.php index f46214d..6961738 100644 --- a/src/Source/Scrape/SelectorAssistant.php +++ b/src/Source/Scrape/SelectorAssistant.php @@ -9,7 +9,7 @@ /** * Asks the configured AI provider to propose a scrape config from a sample of listing-page * HTML: the repeating-item selector and a field map. The proposal is a starting point the - * user reviews and edits in the form before saving — it is never applied blind. + * user reviews and edits in the form before saving; it is never applied blind. */ final class SelectorAssistant { diff --git a/src/Source/Source.php b/src/Source/Source.php index c17831a..a07f01a 100644 --- a/src/Source/Source.php +++ b/src/Source/Source.php @@ -77,7 +77,7 @@ public function pagination_max_pages(): int { /** * Post-processing rules: each writes a meta value when a condition on an extracted field - * holds. Generic — any field, operator, meta key, and value. + * holds. Generic: any field, operator, meta key, and value. * * @return array */ diff --git a/src/Support/ActivityLog.php b/src/Support/ActivityLog.php index 62defec..0b1ec85 100644 --- a/src/Support/ActivityLog.php +++ b/src/Support/ActivityLog.php @@ -7,7 +7,7 @@ defined( 'ABSPATH' ) || exit; /** - * Durable activity feed backed by the `log` table — every pipeline decision and the + * Durable activity feed backed by the `log` table: every pipeline decision and the * before/after of each transformation, queryable and filterable. Cost rows (written by * CostMeter into the same table) are activity too; they simply carry tokens/cost and an * empty message, so the human feed filters to rows that have a message. diff --git a/src/Support/Crypto.php b/src/Support/Crypto.php index 2201318..26a66e1 100644 --- a/src/Support/Crypto.php +++ b/src/Support/Crypto.php @@ -4,20 +4,61 @@ defined( 'ABSPATH' ) || exit; +/** + * Encrypts stored API keys with libsodium's authenticated secretbox, keyed from the site's + * AUTH_KEY and SECURE_AUTH_SALT. New values carry a "v2:" prefix. Values written by older + * releases (unauthenticated AES-256-CBC, base64 only) still decrypt and are re-encrypted + * the next time settings are saved. + */ final class Crypto { - private const METHOD = 'aes-256-cbc'; + private const PREFIX = 'v2:'; + private const LEGACY_METHOD = 'aes-256-cbc'; + /** True when this site can encrypt: sodium is loaded and a non-empty salt exists. */ + public static function available(): bool { + return function_exists( 'sodium_crypto_secretbox' ) && self::secret() !== ''; + } + + /** True when $payload was written by the current scheme and needs no re-encryption. */ + public static function is_current( string $payload ): bool { + return strncmp( $payload, self::PREFIX, strlen( self::PREFIX ) ) === 0; + } + + /** @throws \RuntimeException when sodium or a site salt is missing; keys are never stored in plaintext. */ public static function encrypt( string $plaintext ): string { - if ( $plaintext === '' || ! function_exists( 'openssl_encrypt' ) ) { - return base64_encode( $plaintext ); + if ( $plaintext === '' ) { + return ''; + } + if ( ! self::available() ) { + throw new \RuntimeException( 'Cannot store the API key: this site has no libsodium or no AUTH_KEY/SECURE_AUTH_SALT to encrypt it with.' ); } - $iv = openssl_random_pseudo_bytes( openssl_cipher_iv_length( self::METHOD ) ); - $cipher = openssl_encrypt( $plaintext, self::METHOD, self::key(), OPENSSL_RAW_DATA, $iv ); - return base64_encode( $iv . $cipher ); + $nonce = random_bytes( SODIUM_CRYPTO_SECRETBOX_NONCEBYTES ); + $box = sodium_crypto_secretbox( $plaintext, $nonce, self::key() ); + return self::PREFIX . base64_encode( $nonce . $box ); } public static function decrypt( string $payload ): string { + if ( self::is_current( $payload ) ) { + return self::decrypt_secretbox( substr( $payload, strlen( self::PREFIX ) ) ); + } + return self::decrypt_legacy( $payload ); + } + + private static function decrypt_secretbox( string $encoded ): string { + if ( ! self::available() ) { + return ''; + } + $raw = base64_decode( $encoded, true ); + if ( $raw === false || strlen( $raw ) <= SODIUM_CRYPTO_SECRETBOX_NONCEBYTES ) { + return ''; + } + $nonce = substr( $raw, 0, SODIUM_CRYPTO_SECRETBOX_NONCEBYTES ); + $plaintext = sodium_crypto_secretbox_open( substr( $raw, SODIUM_CRYPTO_SECRETBOX_NONCEBYTES ), $nonce, self::key() ); + return $plaintext === false ? '' : $plaintext; + } + + private static function decrypt_legacy( string $payload ): string { $raw = base64_decode( $payload, true ); if ( $raw === false ) { return ''; @@ -25,15 +66,15 @@ public static function decrypt( string $payload ): string { if ( ! function_exists( 'openssl_decrypt' ) ) { return $raw; } - $ivlen = openssl_cipher_iv_length( self::METHOD ); + $ivlen = openssl_cipher_iv_length( self::LEGACY_METHOD ); if ( strlen( $raw ) <= $ivlen ) { return ''; } $iv = substr( $raw, 0, $ivlen ); $cipher = substr( $raw, $ivlen ); - foreach ( self::keys() as $key ) { - $plaintext = openssl_decrypt( $cipher, self::METHOD, $key, OPENSSL_RAW_DATA, $iv ); + foreach ( self::legacy_keys() as $key ) { + $plaintext = openssl_decrypt( $cipher, self::LEGACY_METHOD, $key, OPENSSL_RAW_DATA, $iv ); if ( $plaintext !== false ) { return $plaintext; } @@ -41,7 +82,22 @@ public static function decrypt( string $payload ): string { return ''; } + /** Site secret; empty when the install has no usable salt. */ + private static function secret(): string { + $auth = defined( 'AUTH_KEY' ) ? (string) AUTH_KEY : ''; + $salt = defined( 'SECURE_AUTH_SALT' ) ? (string) SECURE_AUTH_SALT : ''; + if ( $auth !== '' && $salt !== '' ) { + return $auth . $salt; + } + return function_exists( 'wp_salt' ) ? (string) wp_salt( 'secure_auth' ) : ''; + } + private static function key(): string { + return hash( 'sha256', 'aggregate-it/secretbox/' . self::secret(), true ); + } + + /** @return string[] keys earlier releases may have used, including their weak fallbacks */ + private static function legacy_keys(): array { if ( defined( 'AUTH_KEY' ) && AUTH_KEY !== '' ) { $salt = AUTH_KEY; } elseif ( function_exists( 'wp_salt' ) ) { @@ -49,13 +105,8 @@ private static function key(): string { } else { $salt = __DIR__; } - return hash( 'sha256', $salt, true ); - } - - /** @return string[] */ - private static function keys(): array { return [ - self::key(), + hash( 'sha256', $salt, true ), hash( 'sha256', __DIR__, true ), hash( 'sha256', '', true ), ]; diff --git a/src/Support/Vector.php b/src/Support/Vector.php index f79e433..5c9a2e4 100644 --- a/src/Support/Vector.php +++ b/src/Support/Vector.php @@ -23,12 +23,15 @@ public static function unpack( string $blob ): array { } /** + * Vectors from different models or fallbacks have different dimensions and share no + * coordinate system, so they score 0 rather than a meaningless partial overlap. + * * @param float[] $a * @param float[] $b */ public static function cosine( array $a, array $b ): float { - $len = min( count( $a ), count( $b ) ); - if ( $len === 0 ) { + $len = count( $a ); + if ( $len === 0 || $len !== count( $b ) ) { return 0.0; } diff --git a/src/Vector/VectorStore.php b/src/Vector/VectorStore.php index 00e929e..7943e9a 100644 --- a/src/Vector/VectorStore.php +++ b/src/Vector/VectorStore.php @@ -10,7 +10,7 @@ /** * Stores packed float32 embeddings and finds nearest neighbours by brute-force cosine. * The live set is thousands of rows at single-site scale, so a full scan inside a queue - * job is cheap — and it keeps the plugin free of an external vector database. + * job is cheap, and it keeps the plugin free of an external vector database. */ final class VectorStore { @@ -43,6 +43,34 @@ public function get( string $owner_type, int $owner_id ): array { return $blob ? Vector::unpack( (string) $blob ) : []; } + /** + * Embeddings for many owners at once, keyed by owner id. + * + * @param int[] $owner_ids + * @return array + */ + public function get_many( string $owner_type, array $owner_ids ): array { + global $wpdb; + $ids = array_map( 'intval', $owner_ids ); + if ( ! $ids ) { + return []; + } + + $placeholders = implode( ',', array_fill( 0, count( $ids ), '%d' ) ); + $rows = $wpdb->get_results( + $wpdb->prepare( + "SELECT owner_id, vector FROM {$this->table()} WHERE owner_type = %s AND owner_id IN ( {$placeholders} )", + array_merge( [ $owner_type ], $ids ) + ) + ); + + $vectors = []; + foreach ( (array) $rows as $row ) { + $vectors[ (int) $row->owner_id ] = Vector::unpack( (string) $row->vector ); + } + return $vectors; + } + /** * Best matches for a query vector among rows of $owner_type, scored by cosine. * @@ -53,19 +81,20 @@ public function get( string $owner_type, int $owner_id ): array { public function similar( string $owner_type, array $query, array $only_ids = [], int $limit = 5 ): array { global $wpdb; $table = $this->table(); + $dims = count( $query ); if ( $only_ids ) { $ids = array_map( 'intval', $only_ids ); $placeholders = implode( ',', array_fill( 0, count( $ids ), '%d' ) ); $rows = $wpdb->get_results( $wpdb->prepare( - "SELECT owner_id, vector FROM {$table} WHERE owner_type = %s AND owner_id IN ( {$placeholders} )", - array_merge( [ $owner_type ], $ids ) + "SELECT owner_id, vector FROM {$table} WHERE owner_type = %s AND dims = %d AND owner_id IN ( {$placeholders} )", + array_merge( [ $owner_type, $dims ], $ids ) ) ); } else { $rows = $wpdb->get_results( - $wpdb->prepare( "SELECT owner_id, vector FROM {$table} WHERE owner_type = %s", $owner_type ) + $wpdb->prepare( "SELECT owner_id, vector FROM {$table} WHERE owner_type = %s AND dims = %d", $owner_type, $dims ) ); } diff --git a/tests/BootSmokeTest.php b/tests/BootSmokeTest.php index 65aebd4..6a5ba2e 100644 --- a/tests/BootSmokeTest.php +++ b/tests/BootSmokeTest.php @@ -10,7 +10,7 @@ /** * Boots the whole container with the WP stubs to catch wiring / constructor-arity bugs - * automatically — the thing only a real-ish boot can find. + * automatically: the thing only a real-ish boot can find. */ final class BootSmokeTest extends TestCase { diff --git a/tests/CryptoTest.php b/tests/CryptoTest.php new file mode 100644 index 0000000..e535f4b --- /dev/null +++ b/tests/CryptoTest.php @@ -0,0 +1,37 @@ +assertTrue( Crypto::is_current( $stored ) ); + $this->assertStringNotContainsString( 'sk-secret', base64_decode( substr( $stored, 3 ) ) ); + $this->assertSame( 'sk-secret', Crypto::decrypt( $stored ) ); + } + + public function test_tampered_value_does_not_decrypt(): void { + $stored = Crypto::encrypt( 'sk-secret' ); + $raw = base64_decode( substr( $stored, 3 ) ); + $raw[ strlen( $raw ) - 1 ] = $raw[ strlen( $raw ) - 1 ] ^ "\x01"; + $this->assertSame( '', Crypto::decrypt( 'v2:' . base64_encode( $raw ) ) ); + } + + public function test_legacy_cbc_values_still_decrypt(): void { + $iv = random_bytes( openssl_cipher_iv_length( 'aes-256-cbc' ) ); + $cipher = openssl_encrypt( 'old-key', 'aes-256-cbc', hash( 'sha256', AUTH_KEY, true ), OPENSSL_RAW_DATA, $iv ); + $legacy = base64_encode( $iv . $cipher ); + + $this->assertFalse( Crypto::is_current( $legacy ) ); + $this->assertSame( 'old-key', Crypto::decrypt( $legacy ) ); + } +} diff --git a/tests/DeduplicatorTest.php b/tests/DeduplicatorTest.php index 01074dd..4c7b5b9 100644 --- a/tests/DeduplicatorTest.php +++ b/tests/DeduplicatorTest.php @@ -4,53 +4,45 @@ use AggregateIt\Cluster\Deduplicator; use AggregateIt\Settings; +use AggregateIt\Vector\VectorStore; use PHPUnit\Framework\TestCase; final class DeduplicatorTest extends TestCase { - private function similarity( string $a_title, string $a_body, string $b_title, string $b_body ): float { - $dedup = new Deduplicator( new Settings() ); - $ref = new \ReflectionClass( $dedup ); - $tokens = $ref->getMethod( 'tokens' ); - $tokens->setAccessible( true ); - $cosine = $ref->getMethod( 'cosine' ); - $cosine->setAccessible( true ); + private function scan( array $ids, array $embeddings, float $threshold, int $cursor = 0, ?int $deadline = null ): array { + $dedup = new Deduplicator( new Settings(), new VectorStore() ); + $assigned = []; + $next = $dedup->scan( $ids, $embeddings, $threshold, $cursor, $deadline ?? PHP_INT_MAX, $assigned ); + return [ $next, $assigned ]; + } - $a = $tokens->invoke( $dedup, $a_title, $a_body ); - $b = $tokens->invoke( $dedup, $b_title, $b_body ); - $na = sqrt( array_sum( array_map( static fn ( $v ) => $v * $v, $a ) ) ); - $nb = sqrt( array_sum( array_map( static fn ( $v ) => $v * $v, $b ) ) ); + public function test_similar_stories_group_under_the_oldest_post(): void { + [ $next, $assigned ] = $this->scan( + [ 1, 2, 3 ], + [ 1 => [ 1.0, 0.0 ], 2 => [ 0.99, 0.1 ], 3 => [ 0.0, 1.0 ] ], + 0.72 + ); + $this->assertSame( 3, $next ); + $this->assertSame( [ 2 => 1 ], $assigned ); + } - return $cosine->invoke( $dedup, $a, $na, $b, $nb ); + public function test_threshold_comes_from_the_caller(): void { + $embeddings = [ 1 => [ 1.0, 0.0 ], 2 => [ 0.8, 0.6 ] ]; + $this->assertSame( [ 2 => 1 ], $this->scan( [ 1, 2 ], $embeddings, 0.75 )[1] ); + $this->assertSame( [], $this->scan( [ 1, 2 ], $embeddings, 0.9 )[1] ); } - public function test_same_story_paraphrases_score_above_threshold(): void { - $sim = $this->similarity( - "Bally's Faces Increased Pressure in Chicago, Las Vegas with New York Project on Horizon", - "Bally's faces mounting pressure in Chicago and Las Vegas as its New York casino project approaches construction. The company must juggle several developments at once.", - "Bally's Faces Challenges as New York Casino Construction Approaches", - "Bally's is under pressure in Chicago and Las Vegas while the New York casino construction project looms, forcing the company to manage multiple developments." - ); - $this->assertGreaterThan( 0.50, $sim ); + public function test_posts_with_different_dimensions_never_match(): void { + $assigned = $this->scan( [ 1, 2 ], [ 1 => [ 1.0, 0.0 ], 2 => [ 1.0, 0.0, 0.0 ] ], 0.5 )[1]; + $this->assertSame( [], $assigned ); } - public function test_different_stories_about_same_entity_score_low(): void { - $sim = $this->similarity( - 'Ballys reports strong quarterly earnings beating estimates', - 'Ballys posted quarterly revenue above analyst estimates driven by online betting growth and interactive segment gains.', - 'Ballys Faces Challenges as New York Casino Construction Approaches', - 'Ballys is under pressure in Chicago and Las Vegas while the New York casino construction project looms.' - ); - $this->assertLessThan( 0.40, $sim ); + public function test_posts_without_an_embedding_are_skipped(): void { + $this->assertSame( [], $this->scan( [ 1, 2 ], [ 1 => [ 1.0, 0.0 ] ], 0.5 )[1] ); } - public function test_unrelated_stories_score_near_zero(): void { - $sim = $this->similarity( - 'UKGC fines Stakelogic over slot speed rule breaches', - 'The UK Gambling Commission fined Stakelogic for breaching slot speed rules designed to slow play.', - 'Entain sells 20% of CEE business launching full exit strategy', - 'Entain announced the sale of a stake in its central and eastern Europe business as part of an exit.' - ); - $this->assertLessThan( 0.40, $sim ); + public function test_scan_stops_at_the_deadline_and_reports_where_to_resume(): void { + [ $next ] = $this->scan( [ 1, 2 ], [ 1 => [ 1.0 ], 2 => [ 1.0 ] ], 0.5, 0, 0 ); + $this->assertSame( 0, $next ); } } diff --git a/tests/FactsGuardTest.php b/tests/FactsGuardTest.php index 0206b9e..07f2478 100644 --- a/tests/FactsGuardTest.php +++ b/tests/FactsGuardTest.php @@ -19,6 +19,15 @@ public function test_salient_catches_numbers_and_proper_nouns(): void { $this->assertContains( 'acme corp', $salient ); } + public function test_sentence_openers_and_stopwords_are_not_entities(): void { + $salient = $this->guard->salient( 'Shares rose on Monday. The company said Acme Corp would expand. However, Tim Cook disagreed.' ); + $this->assertNotContains( 'shares', $salient ); + $this->assertNotContains( 'the', $salient ); + $this->assertNotContains( 'however', $salient ); + $this->assertContains( 'acme corp', $salient ); + $this->assertContains( 'tim cook', $salient ); + } + public function test_invented_flags_fabricated_number(): void { $invented = $this->guard->invented( 'Revenue was $5 million', 'Revenue reached $9 million' ); $this->assertContains( '9', $invented ); diff --git a/tests/KeywordStrategyTest.php b/tests/KeywordStrategyTest.php index bc7fb94..7d1f323 100644 --- a/tests/KeywordStrategyTest.php +++ b/tests/KeywordStrategyTest.php @@ -2,6 +2,7 @@ namespace AggregateIt\Tests; +use AggregateIt\Keyword\KeywordProvider; use AggregateIt\Keyword\KeywordStrategy; use AggregateIt\Settings; use PHPUnit\Framework\TestCase; @@ -10,6 +11,41 @@ final class KeywordStrategyTest extends TestCase { protected function setUp(): void { $GLOBALS['__options'] = []; + $GLOBALS['__filters'] = []; + } + + private function provider_with_volume( int $volume ): KeywordProvider { + return new class( $volume ) implements KeywordProvider { + public function __construct( private int $volume ) {} + + public function key(): string { + return 'fake'; + } + + public function metrics( array $keywords ): array { + return array_fill_keys( $keywords, [ 'volume' => $this->volume, 'difficulty' => 1 ] ); + } + }; + } + + public function test_filter_supplies_the_provider_and_rejects_other_types(): void { + $strategy = new KeywordStrategy( new Settings() ); + $this->assertNull( $strategy->provider() ); + + $GLOBALS['__filters']['aggregate_it_keyword_provider'] = static fn () => new \stdClass(); + $this->assertNull( $strategy->provider() ); + + $provider = $this->provider_with_volume( 10 ); + $GLOBALS['__filters']['aggregate_it_keyword_provider'] = static fn () => $provider; + $this->assertSame( $provider, $strategy->provider() ); + } + + public function test_strategic_mode_skips_zero_volume_keywords_when_a_provider_reports_them(): void { + $GLOBALS['__options']['aggregate_it_settings'] = [ 'strategic_mode' => true ]; + $provider = $this->provider_with_volume( 0 ); + $GLOBALS['__filters']['aggregate_it_keyword_provider'] = static fn () => $provider; + + $this->assertTrue( ( new KeywordStrategy( new Settings() ) )->resolve( 'dead topic', 'text' )['skip'] ); } public function test_no_list_keeps_inferred_keyword(): void { diff --git a/tests/VectorTest.php b/tests/VectorTest.php index 733a498..56c32bb 100644 --- a/tests/VectorTest.php +++ b/tests/VectorTest.php @@ -26,6 +26,10 @@ public function test_orthogonal_vectors_have_zero_similarity(): void { $this->assertEqualsWithDelta( 0.0, Vector::cosine( [ 1.0, 0.0 ], [ 0.0, 1.0 ] ), 1e-6 ); } + public function test_vectors_of_different_dimensions_score_zero(): void { + $this->assertSame( 0.0, Vector::cosine( [ 1.0, 2.0 ], [ 1.0, 2.0, 3.0 ] ) ); + } + public function test_empty_vector_is_safe(): void { $this->assertSame( 0.0, Vector::cosine( [], [ 1.0 ] ) ); } diff --git a/tests/bootstrap.php b/tests/bootstrap.php index f7665aa..8632596 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -13,6 +13,7 @@ defined( 'MINUTE_IN_SECONDS' ) || define( 'MINUTE_IN_SECONDS', 60 ); defined( 'HOUR_IN_SECONDS' ) || define( 'HOUR_IN_SECONDS', 3600 ); defined( 'DAY_IN_SECONDS' ) || define( 'DAY_IN_SECONDS', 86400 ); +defined( 'AGGREGATE_IT_TESTING' ) || define( 'AGGREGATE_IT_TESTING', true ); defined( 'WPINC' ) || define( 'WPINC', 'wp-includes' ); $GLOBALS['__options'] = [];