diff --git a/.rubocop.yml b/.rubocop.yml index 6458bc528..c6985f27d 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -257,6 +257,9 @@ Naming/PredicatePrefix: Metrics/ParameterLists: Exclude: + - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb' + - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb' + - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/configuration.rb' - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/collections/base_collection.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/datasource.rb' @@ -357,6 +360,9 @@ Metrics/BlockLength: Metrics/ClassLength: Exclude: + - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb' + - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb' + - 'packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/collection.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/introspection/introspector.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/introspection/schema_converter.rb' @@ -454,6 +460,12 @@ Layout/LineLength: RSpec/VerifiedDoubles: Exclude: + - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb' + - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb' + - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb' + - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb' + - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb' + - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb' - 'packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/composite_datasource_spec.rb' RSpec/VerifiedDoubleReference: diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md new file mode 100644 index 000000000..ce3799507 --- /dev/null +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -0,0 +1,347 @@ +# Audit trail + +Capture who changed what (before/after) for every change Forest performs through its data layer, and +persist it into a SQL database. Built into the agent: it turns on as soon as an audit-trail **database +is configured**, and stays completely off otherwise. + +Two parts, both internal: + +- **Capture** (`ForestAdminAgent::AuditTrail::Capture`) — datasource-agnostic. It instruments every + collection through the customizer hooks, so it behaves the same whether the audited datasource is + ActiveRecord, Mongoid, etc. +- **Storage** (`ForestAdminAgent::AuditTrail::Store`) — ActiveRecord-backed. It creates the `forest` + schema and creates/evolves the `audit_logs` table through versioned migrations, and reads the + per-record history back for the routes below. + +Storage uses ActiveRecord: outside Rails, add `gem 'activerecord'` (and the adapter gem) to your +Gemfile. Nothing is loaded and no connection is opened until the feature is configured. + +## Turn it on + +### Rails (forest_admin_rails) + +```ruby +# config/initializers/forest_admin_rails.rb +ForestAdminRails.configure do |config| + config.auth_secret = ENV['FOREST_AUTH_SECRET'] + config.env_secret = ENV['FOREST_ENV_SECRET'] + + config.audit_trail = { + database: { # or an ActiveRecord URL: ENV['AUDIT_TRAIL_DATABASE_URL'] + adapter: 'postgresql', host: ENV['AUDIT_DB_HOST'], port: ENV['AUDIT_DB_PORT'], + username: ENV['AUDIT_DB_USER'], password: ENV['AUDIT_DB_PASSWORD'], database: ENV['AUDIT_DB_NAME'] + } + } +end +``` + +### Plain agent (no Rails) + +```ruby +ForestAdminAgent::Builder::AgentFactory.instance.setup( + auth_secret: ENV['FOREST_AUTH_SECRET'], + env_secret: ENV['FOREST_ENV_SECRET'], + # ...usual options... + audit_trail: { database: ENV['AUDIT_TRAIL_DATABASE_URL'] } +) +``` + +| option | description | +| ------------ | ------------------------------------------------------------------------------------ | +| `database` | ActiveRecord URL or config hash. **Setting it activates the audit trail.** | +| `schema` | Postgres schema holding the table (default `forest`; ignored on other adapters) | +| `table_name` | default `audit_logs` | +| `redact` | `{ 'collection_name' => ['field', ...] }` — values masked while recording the change | +| `critical` | default `false`. `true` refuses an operation the audit trail cannot record — see below | + +The store connects and migrates **at boot**, not on the first write: an audit database the agent cannot +reach stops it starting, rather than leaving it looking healthy while recording nothing. Every create / +update / delete performed through Forest then writes one row per record, and the **Historic** tab in the UI +reads from the same table. + +## The write protocol, and `critical` + +Every operation is recorded twice: a `pending` row **before** the write, confirmed `done` **after** it. One +code path either way, so `status` always means the same thing. + +| `critical` | a pending row that cannot be written | +| ---------- | ------------------------------------------------------------------------------------------ | +| `false` | is logged and dropped; the operation goes ahead unaudited (the default, today's behaviour) | +| `true` | **refuses the operation**. Nothing was written, so there is nothing to repair and no compensating write ever happens | + +What this buys is **no unaudited write** — not that every row holds exact after-values. A row left `pending` +means the write may or may not have landed: that residue is evidence, and it is the point. Everything after +the pending insert stays best-effort in both modes, because by then the write has happened and raising would +report a failure for an operation that succeeded. + +> **The guarantee is opt-in.** `critical` defaults to `false`, so on a default configuration a write can +> succeed with no audit row at all — an unreachable audit database costs rows, not writes. Configuring the +> database gets you a best-effort trail; `critical: true` is what makes "no unaudited write" true. The Node +> agent defaults the same way, so the two agree. + +Consequences worth knowing: + +- A write that turns out to change nothing has its pending row **discarded** rather than confirmed, so no-op + updates leave no trace. +- A record the agent cannot read back after the write keeps its row **pending**. Confirming from the patch + would claim values that may never have been written. +- A write nested inside another that fails and is rescued keeps its row **pending** too. Each snapshot is + matched to its own operation by the object the hook decorator hands to both of its hooks, so the outer write + settles its own rows rather than the failed inner one's. Where a customization replaced that object there is + nothing to match on: with one operation in flight it still pairs, with several the rows stay pending rather + than the wrong ones being marked done. +- One operation audits at most **1000** records — the same number as the Node agent's `MAX_SNAPSHOT_RECORDS`, + so a bulk operation is not audited on one agent and truncated on the other. Under `critical: false` a wider + selection is truncated, with `N records audited, M skipped` logged at `Warn`; a smart action over the cap is + recorded as one row attached to no record. Under `critical: true` both are **refused**: auditing a subset + while the operation touches every match is precisely the invariant that mode exists for. +- Pending rows stay visible in the history — they are evidence of an attempt, and `status` says so — but the + state reconstruction ignores them, since undoing a change that may never have happened would invent a state + the record was never in. + +## Routes + +All routes live under `/forest/_audit-trail`, are registered only when `audit_trail[:database]` is +set, and require read permission on the target collection (`can?(:read, collection)`). + +### Record-history route + +`GET /forest/_audit-trail/{collection}/{recordId}` returns the current page of history (newest first +by default) together with the filtered total: + +```json +{ "data": [ /* current page rows */ ], "meta": { "count": 137 } } +``` + +`meta.count` is the number of rows matching the active filters (not the absolute total) and is independent of +the page. `meta.availableUsers` rides along **on the first fetch only** — the front keeps the list it saw — +and holds the distinct authors of the entries the current filters match, as `{ id, firstName, lastName, +email }`, whatever page was asked for. The identity comes from the rows themselves, so someone since renamed +or removed still reads as they were when they acted. Optional filters (all combine with `AND`; omit them for the full history): + +| query param | format | effect | +| ----------- | -------------------------------- | ----------------------------------------------- | +| `userIds` | comma-separated integers `12,45` | keep only entries whose `user_id` is in the list | +| `startDate` | `YYYY-MM-DD` or datetime (incl.) | keep entries from this lower bound onward | +| `endDate` | `YYYY-MM-DD` or datetime (incl.) | keep entries up to this upper bound | +| `fields` | comma-separated field names | keep only entries whose diff touched one of them | +| `search` | free text, trimmed | keep only entries the term matches | + +`search` is matched case-insensitively, as a substring, against the action's name, the actor's first name, +last name and email, and the **keys and values of both value objects at any depth** — searching `Lyon` finds +`{"address": {"city": "Lyon"}}`, which is what only the agent can answer, since it is the only side holding +the recorded values. It is matched in SQL, not in memory, so it composes with pagination and `meta.count` +like every other filter. + +It deliberately does **not** match `operation`, `correlationKey`, `recordId`, `collection`, `status` or +`timestamp`: machine identifiers nobody searches for, whose matches read as noise. + +A field masked by `redact` never matches — neither by its `[redacted]` mask nor by the value it hid, which +was never recorded. A search must not confirm a value the trail refused to keep. + +`fields` matches whole keys, never paths, so a name holding a dot (`address.city`) is quoted before it +reaches SQL. Both sides of the diff are searched, since a field the change added exists in `newValues` +only and one it removed in `previousValues` only. The JSON test is per adapter (Postgres, SQLite, MySQL / +MariaDB); on any other adapter the filter raises rather than silently returning everything. + +`startDate` / `endDate` are read as **local wall-clock time** in the request `timezone` query param +(e.g. `Europe/Paris`, default `UTC`) and converted to a UTC instant before querying, so filtering +happens in SQL. Two shapes are accepted: + +- **Bare day** `YYYY-MM-DD` — `startDate` snaps to `00:00:00.000`, `endDate` to `23:59:59.999`. +- **Datetime** `YYYY-MM-DD[T| ]HH:mm[:ss]` — `T` or space separator, seconds optional; when seconds + are omitted `endDate` is completed to `:59.999` and `startDate` stays at `:00.000`. + +Both bounds are **inclusive**. Defensive parsing: non-numeric `userIds` tokens are dropped +(`12,abc,45` → `12,45`), and a `startDate` / `endDate` matching no accepted format returns **HTTP +400** (`ValidationError`); an invalid `timezone` likewise returns **400**. + +Pagination follows JSON:API: `page[number]` is 1-based (default `1`), `page[size]` defaults to `20` +and is capped at `100`; out-of-bound or non-numeric values fall back to the defaults rather than +erroring. Sorting follows JSON:API `sort` on `timestamp`: `sort=-timestamp` (or absent/unrecognized) +is newest first, `sort=timestamp` is oldest first. Ties on equal timestamps fall back to insertion +order (the auto-increment `id`), so paging is deterministic in either direction. + +All routes serialize audit records the same way: top-level keys are camelCased — `id`, `recordId`, `userId`, +`userFirstName`, `userLastName`, `userEmail`, `actionName`, `status`, `correlationKey`, `previousValues`, +`newValues`. The row `id` is exposed because both agents order by `(timestamp, id)` and the front uses it as +the merge tiebreaker. + +Inside the value objects: a record's column names pass through untouched, while an action answer's keys are +Forest's own and so are camelCase (`mimeType`, not `mime_type`) — the agent transforms them on write. + +**A record that was renamed keeps one timeline** — on this agent. The Node agent files an update under the +record's new id too, but has no `previous_record_id` and so cannot walk back past the rename: the same record +shows a complete chain here and a history beginning at the rename there, until that column is ported. + An update that moves a writable primary key files its row +under the record's new id — the id later lookups use — and remembers the one it left. Both the history and the +state routes walk that back, so asking for the current id returns everything the record has ever been filed +under, rather than starting the story at the rename. + +Each earlier id counts only **up to the moment it was left**, because a primary key a record abandons can be +taken by another record afterwards, and those rows are none of this record's business. What the trail cannot +separate is the opposite case: rows written under an id *before* the record that holds it now arrived — a +reused key, or a delete followed by a recreate — since the packed id is the only identity the trail has. That +is deliberate for delete/recreate (the state reconstruction walks into an earlier life on purpose) and the +same limitation for a reused key. Telling those apart needs a lineage of its own on every row, which is a +bigger change than this one. + +A record that no longer exists keeps its history: only a record that still exists *outside* the +caller's permission scope is refused (404). Inspecting what was deleted is much of the point of an +audit trail, and the delete event itself is the last thing recorded. + +### State route + +`GET /forest/_audit-trail/{collection}/{recordId}/state?timestamp=…` returns the record as it stood at +that instant, rebuilt by taking the record as it stands now and undoing every entry recorded **strictly +after** the timestamp — an entry stamped exactly at it counts as part of that state: + +```json +{ "data": { "status": "paid", "address": { "city": "Paris" } } } +``` + +`timestamp` accepts an ISO-8601 instant, or the same wall-clock forms as the filters above read in the +request `timezone`; it is required (**400** otherwise). `data` is `null` when the record did not exist at +that instant — either created later, or deleted and never recreated. + +Walking back stops being able to help where the trail stops: only audited (writable) columns are +reconstructed, and a `create` means the record did not exist before it, while a `delete` restores the whole +row it recorded and the walk carries on into any earlier life of the same id. + +### Correlation route + +`GET /forest/_audit-trail/correlation/{correlationKey}` returns `{ "data": [...] }` — the +operation(s) recorded under one `correlation_key` for a single record (usually one), oldest first, or +an empty array if none. Scoped through query params; same auth and gating as above. + +| query param | required | effect | +| ------------ | -------- | ------------------------------------------------------------ | +| `collection` | yes | collection the record belongs to (also the permission scope) | +| `recordId` | yes | packed record id to scope the lookup | + +A missing `collection` or `recordId` returns **HTTP 400** (`ValidationError`). + +### Batch correlation route + +`GET /forest/_audit-trail/correlations` returns `{ "data": [...] }` — a **flat** list of every record +whose `correlation_key` is in `correlationKeys`, scoped to one record (the client groups by +`correlation_key`). Same auth and gating; empty array when nothing matches. + +| query param | required | effect | +| ----------------- | -------- | ------------------------------------------------------------ | +| `correlationKeys` | yes\* | comma-separated keys; blank tokens are dropped | +| `collection` | yes | collection the record belongs to (also the permission scope) | +| `recordId` | yes | packed record id to scope the lookup | + +\* To dodge any URL length limit, the same path also accepts **`POST`** with a JSON body +`{ "correlationKeys": [...], "collection": "...", "recordId": "..." }` (the body array takes +precedence over the query param). An empty/absent key list returns `{ "data": [] }` without hitting +the store. A missing `collection` or `recordId` returns **HTTP 400** (`ValidationError`). + +## Smart actions + +Running a smart action writes one row per selected record, in the same table: + +| column | value | +| ---------------- | -------------------------------------------------------------------------- | +| `operation` | `action` when it went through, `action_failed` when it raised or answered with an `Error` result | +| `previousValues` | the submitted form values (redacted with the same `redact` config) | +| `newValues` | what the action answered — an allowlist of the result: `type`, `message`, `name`, `mimeType`, `method`, `url`, `path`. Empty when it raised | +| `recordId` | each selected record — empty for a global action or a select-all selection | + +The two value columns carry what went in and what came back, rather than a record's before and after. A +result also holds the file's contents, a webhook's body and headers, and arbitrary response headers: file +bytes have no business in an audit table and the other two routinely hold credentials, so the stored answer +is an allowlist — a field added to a result later is not recorded until someone decides it should be. `html` +is left out too, being operator-facing markup the message already summarises. + +Because these are the same columns the field filter searches, filtering by a field named like a result key +(`message`, `type`) also matches action rows. + +**Which** action ran is not stored: the Forest activity logs already record it, and +`correlationKey` is the join between the two. + +A **global** action targets no record, and a **select-all** selection only tells the agent which ids were +*excluded*, so naming the targets would mean querying the whole selection: those runs are recorded once, +attached to no record. + +Recording follows the same policy as a write: the row goes in before the action runs, so under +`critical: false` a failing audit database logs an error and the action goes ahead, while under +`critical: true` it refuses the run — nothing has happened yet, so there is nothing to repair. Everything +after that point, the answer included, is best-effort either way. + +**A row attached to no record is not readable through any route today.** Every history route is scoped to a +record id, and the correlation routes reject an empty one, so global and over-cap action runs are recorded but +cannot be fetched. They are evidence in the table rather than something the UI can show — reaching them needs +a collection-level endpoint that does not exist yet. + +The targeted records are read back through the caller's own filter rather than taken from the request: the ids +a client sends are a claim, and in a compliance record asserting that an operator acted on a record their +scope excludes is worse than a missing row. + +`url` and `path` are sanitised before storage — credentials in the userinfo and anything in a query string or +fragment come off, since either can carry a signed one-time token that would otherwise sit permanently in the +one table nobody deletes from. + +> **What an action changes is only audited when it goes through Forest.** `context.collection.update(...)` +> passes through the same hooks as any other write, so it produces the usual field-level rows sharing the +> action's `correlationKey`. A direct ORM write (`Customer.find(id).update!(...)`) is invisible to the +> agent, so nothing is recorded for it beyond the invocation row above. + +## What gets stored + +`forest.audit_logs`, one row per audited change: + +| column | description | +| ----------------- | ----------------------------------------------------------- | +| `id` | auto-increment primary key, exposed in the payload | +| `status` | `pending` before the write, `done` once confirmed | +| `timestamp` | when the change happened | +| `operation` | `create` / `update` / `delete` | +| `collection` | audited collection name | +| `record_id` | packed record id (primary keys joined by `\|`), TEXT and nullable — a create's pending row has none yet, and a composite id outgrows a varchar | +| `previous_record_id` | set only on an update that moved a writable primary key: the id the row was filed under before | +| `user_id` | the Forest user who made the change | +| `user_first_name`, `user_last_name`, `user_email` | denormalised from the caller at write time: who acted then, not whoever holds that id today | +| `action_name` | smart-action rows only | +| `correlation_key` | per-request id; groups every change made within one request. Empty for a write outside any request — inventing one would make the row look like a single-row request of its own | +| `previous_values` | values before the change (JSON) | +| `new_values` | values after the change (JSON) | + +`previous_values` / `new_values` store **only the parts that actually changed**: nested hashes and +arrays of hashes are diffed structurally, so a single sub-field change records just that leaf. Only +writable columns are audited — read-only, computed and DB-managed fields are never written by Forest. + +The `correlation_key` is the agent's per-request id (`caller.request_id`), generated by the agent and echoed +back to the client in the `X-Forest-Correlation-Id` response header — so every change made in one request +shares a key, and the caller can tie it to its own activity log. + +Outside Rails, mount `ForestAdminAgent::Http::CorrelationIdMiddleware` yourself: it resets the id at the start +of each request, and without it a pooled thread would hand its previous request's key to the next one. + +The capture layer registers its **after** hooks ahead of any other customization's (`prepend: true`, +since `execute_after` stops at the first exception, and by then the write has already happened) and its +**before** hooks after them, so the snapshot sees the filter and patch everyone else has had their say on. + +## Concurrent writes to one record + +The before/after values are captured around the write, not inside it: the customizer hooks bracket the +write as separate calls, and the data layer deliberately exposes no lock or transaction primitive since +it spans ActiveRecord, Mongoid, HTTP APIs and more. + +So when two writes race on the same record, both snapshot the same state and the one that lands second +records a `previousValues` that had already been overwritten. `newValues` is always exact — it is the +patch that was written — and no row is ever lost; only the prior state of an overlapping write can be +stale. Exact before-images under concurrency need the database itself (triggers, or CDC), not an agent +hook. + +## Schema migrations & concurrency + +The table is created and evolved through an ordered, append-only migration list, tracked in a companion table +named after the audited one — `forest.audit_logs_migration` beside `forest.audit_logs`. One tracker per audited +table, so two stores configured with different `table_name`s each keep their own schema history rather than +reading the other's as done. + +On Postgres the migrations run inside a transaction-scoped advisory lock, so several agents booting at once +apply them one after another; the schema is created (and committed, idempotently) first, since the lock cannot +cover a schema that does not exist yet. diff --git a/packages/forest_admin_agent/Gemfile b/packages/forest_admin_agent/Gemfile index 0e0b74330..d22fe37a2 100644 --- a/packages/forest_admin_agent/Gemfile +++ b/packages/forest_admin_agent/Gemfile @@ -3,6 +3,7 @@ source "https://rubygems.org" gemspec group :development, :test do + gem 'activerecord', '>= 6.1' gem 'forest_admin_datasource_customizer', path: '../forest_admin_datasource_customizer' gem 'forest_admin_datasource_toolkit', path: '../forest_admin_datasource_toolkit' gem 'forest_admin_test_toolkit', path: '../forest_admin_test_toolkit' @@ -13,4 +14,5 @@ group :development, :test do gem 'simplecov', '~> 0.22', require: false gem 'simplecov-html', '~> 0.12.3' gem 'simplecov_json_formatter', '~> 0.1.4' + gem 'sqlite3', '>= 2.1' end diff --git a/packages/forest_admin_agent/Gemfile-test b/packages/forest_admin_agent/Gemfile-test index 0e0b74330..d22fe37a2 100644 --- a/packages/forest_admin_agent/Gemfile-test +++ b/packages/forest_admin_agent/Gemfile-test @@ -3,6 +3,7 @@ source "https://rubygems.org" gemspec group :development, :test do + gem 'activerecord', '>= 6.1' gem 'forest_admin_datasource_customizer', path: '../forest_admin_datasource_customizer' gem 'forest_admin_datasource_toolkit', path: '../forest_admin_datasource_toolkit' gem 'forest_admin_test_toolkit', path: '../forest_admin_test_toolkit' @@ -13,4 +14,5 @@ group :development, :test do gem 'simplecov', '~> 0.22', require: false gem 'simplecov-html', '~> 0.12.3' gem 'simplecov_json_formatter', '~> 0.1.4' + gem 'sqlite3', '>= 2.1' end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent.rb b/packages/forest_admin_agent/lib/forest_admin_agent.rb index be8270393..22d65f2b9 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent.rb @@ -4,7 +4,11 @@ loader = Zeitwerk::Loader.for_gem loader.inflector.inflect('oauth2' => 'OAuth2') +loader.inflector.inflect('sql' => 'Sql') loader.inflector.inflect('sse_cache_invalidation' => 'SSECacheInvalidation') +# ActiveRecord is only needed by agents configuring an audit-trail database, and Rails eager loads +# every gem loader (Zeitwerk::Loader.eager_load_all), so these files must stay strictly autoloaded. +loader.do_not_eager_load("#{__dir__}/forest_admin_agent/audit_trail/sql") loader.setup module ForestAdminAgent diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb new file mode 100644 index 000000000..eebf51aec --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb @@ -0,0 +1,68 @@ +module ForestAdminAgent + # The audit trail is inert unless `config.audit_trail[:database]` was set: the agent factory builds the + # store during setup — connecting and migrating there rather than on first write — and everything (capture + # layers and routes) resolves it from here. + module AuditTrail + # One operation must not materialise an unbounded number of records: a "delete all" would otherwise read + # every matched row and, with the pending/confirm protocol, write each of them twice. Truncation is logged, + # never silent. + # + # Matches the Node agent's `MAX_SNAPSHOT_RECORDS`: the same feature behind the same config key, so a bulk + # operation must not be audited on one agent and truncated on the other. Change it in both or neither. + MAX_RECORDS_PER_OPERATION = 1000 + + # Auditing a subset while the write touches every match is the one thing `critical` exists to prevent, so + # over the cap the operation is refused instead — before anything is written, in both the write and the + # action path, which is why the message lives here rather than in either of them. + def self.refuse_over_cap! + raise ForestAdminDatasourceToolkit::Exceptions::ForestException, + 'The audit trail is configured as critical and cannot record an operation touching more than ' \ + "#{MAX_RECORDS_PER_OPERATION} records at once. Narrow the selection." + end + + def self.log_truncation(kept, total) + skipped = total ? total - kept : 'further' + + Facades::Container.logger.log( + 'Warn', + "[ForestAdmin] Audit trail: #{kept} records audited, #{skipped} skipped " \ + "(cap #{MAX_RECORDS_PER_OPERATION} per operation)" + ) + end + + def self.options + config = Facades::Container.config_from_cache + + (config && config[:audit_trail]) || {} + end + + def self.store + options[:store] + end + + # `critical: true` makes the pending insert a precondition of the write: if the audit trail cannot record + # that an operation is about to happen, the operation is refused. Nothing was written, so there is nothing + # to repair and no compensating write ever happens. Default false keeps today's behaviour, where a broken + # audit database costs rows rather than writes. + def self.critical? + options[:critical] == true + end + + def self.log_failure(error) + Facades::Container.logger.log('Error', "[ForestAdmin] Audit trail unavailable, skipping: #{error.message}") + end + + # Runs the pending insert under the configured policy: refusing the operation when critical, logging and + # carrying on otherwise. + def self.gate + return yield if critical? + + begin + yield + rescue StandardError => e + log_failure(e) + nil + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb new file mode 100644 index 000000000..0adaef2cd --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb @@ -0,0 +1,99 @@ +require 'uri' + +module ForestAdminAgent + module AuditTrail + # Records smart-action runs into the same table as the field-level history: the submitted form on the + # `previous_values` side, what the action answered on the `new_values` side. + # + # {Capture} cannot see them — the customizer has no `Execute` hook — and an action's writes are only + # audited when they go through the Forest data layer, so a direct ORM write stays invisible. What lands + # here is the run itself: who ran which action, on which records, with which form, and how it ended. + # + # Same protocol as a write: {#pending} before the action, {#confirm} after. The route owns the gate, so a + # pending insert that fails refuses the run under `critical: true`. + class ActionCapture + include Recording + + EXECUTED = 'action'.freeze + FAILED = 'action_failed'.freeze + # A global action, and a bulk run over a selection wider than the cap, name no single target: they get + # one row attached to no record rather than none at all. + NO_RECORD = ''.freeze + # What of the action's answer is worth keeping — an allowlist, not a denylist: a result also carries the + # file's contents, a webhook's body and headers and arbitrary response headers. File bytes have no + # business in an audit table and the other two routinely hold credentials, and an allowlist means a field + # added to a result later is not stored until someone decides it should be. `html` is left out too: + # operator-facing markup, sometimes large, and the message already says what happened. + RESULT_FIELDS = %i[type message name mime_type method url path].freeze + # Either can carry userinfo credentials or a signed one-time token, which would then sit permanently in + # the one table nobody deletes from. + URL_FIELDS = %i[url path].freeze + + def initialize(store, redact = {}) + @store = store + @redact = redact || {} + end + + # One row per targeted record, provisionally an `action` — {#confirm} settles which it really was. Returns + # the row ids to confirm. + def pending(caller:, collection:, action_name:, form_values:, record_ids:) + return [] unless @store + + timestamp = now + correlation_key = correlation_key_for(caller) + identity = identity_of(caller) + submitted = redact(form_values || {}, @redact[collection] || []) + ids = record_ids.empty? ? [NO_RECORD] : record_ids + + @store.append_all( + ids.map do |record_id| + AuditRecord.new( + timestamp: timestamp, operation: EXECUTED, collection: collection, record_id: record_id, + status: PENDING, action_name: action_name, correlation_key: correlation_key, + previous_values: submitted, new_values: {}, **identity + ) + end + ) + end + + # Best-effort: the action has already run, so a failure here loses the answer, never the run. + def confirm(ids, result: nil, failed: false) + return if ids.nil? || ids.empty? + + audit_safely do + answer = summarize(result) + + ids.each { |id| @store.confirm(id, operation: failed ? FAILED : EXECUTED, new_values: answer) } + end + end + + private + + # Keys of an action's answer are Forest's own, so they are camelCase on the wire — unlike a record's + # column names, which pass through untouched. + def summarize(result) + return {} unless result.is_a?(Hash) + + result.slice(*RESULT_FIELDS).compact.to_h do |field, value| + [field.to_s.camelize(:lower), URL_FIELDS.include?(field) ? sanitize_url(value) : value] + end + end + + # `userinfo = nil` is a no-op on URI, so the credentials come off textually; the parser then takes care + # of the query and fragment. + def sanitize_url(value) + # Both `https://user:pass@host` and the scheme-relative `//user:pass@host`, which parses fine and + # would otherwise keep its credentials. + bare = value.to_s.sub(%r{\A([a-z][a-z0-9+.-]*:)?//[^/@]*@}i, '\1//') + uri = URI.parse(bare) + uri.query = nil + uri.fragment = nil + + uri.to_s + rescue StandardError + # Not something the parser accepts: keep the shape, drop everything that can carry a secret. + bare.to_s.split(/[?#]/).first.to_s + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb new file mode 100644 index 000000000..1fa3e8d7f --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb @@ -0,0 +1,16 @@ +module ForestAdminAgent + module AuditTrail + # One audited change. Mirrors the columns of `forest.audit_logs`. + # + # The actor's name and email are denormalised from the caller at write time: the row says who acted then, + # not whoever holds that user id today. `action_name` is set on smart-action rows only, and `status` + # follows the write protocol — inserted as {Recording::PENDING} before the write and confirmed + # {Recording::DONE} after, so a row left pending means the write may or may not have landed. + AuditRecord = Struct.new( + :id, :timestamp, :operation, :collection, :record_id, :previous_record_id, :status, + :user_id, :user_first_name, :user_last_name, :user_email, :action_name, + :correlation_key, :previous_values, :new_values, + keyword_init: true + ) + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb new file mode 100644 index 000000000..b8ce59496 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb @@ -0,0 +1,229 @@ +module ForestAdminAgent + module AuditTrail + # Datasource-agnostic capture layer, installed by the agent factory as soon as an audit-trail database is + # configured. It instruments every collection through the Forest customizer hooks, so it behaves the same + # whatever the audited datasource is (ActiveRecord, Mongoid, ...). + # + # Every operation is recorded twice: a PENDING row before the write, confirmed DONE after it. One code + # path in both `critical` modes, so `status` always means the same thing — and what the protocol buys is + # that no write goes unaudited, not that every row holds exact after-values. A row left pending says the + # write may or may not have landed, which is evidence rather than a defect. + class Capture + include Recording + + # Signature imposed by DatasourceCustomizer#use; the agent always instruments the whole datasource. + def run(datasource_customizer, _collection_customizer = nil, options = {}) + @store = options[:store] + @redact = options[:redact] || {} + @snapshots = Snapshots.new + + datasource_customizer.collections.each_value { |collection| instrument(collection) } + end + + private + + def instrument(collection_customizer) + schema = collection_customizer.collection.schema + # Writable columns only: Forest audits what it writes. Read-only fields cover computed/virtual + # fields and DB-managed columns, none of which Forest mutates. + columns = schema[:fields].select do |_name, field| + field.type == 'Column' && !field.is_read_only + end.keys + primary_keys = ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(collection_customizer.collection) + # Reads must carry the primary keys (even read-only ones) so the record id can be built; the + # diff itself stays restricted to the writable columns. + projection = ForestAdminDatasourceToolkit::Components::Query::Projection.new( + (primary_keys + columns).uniq + ) + target = { columns: columns, primary_keys: primary_keys, projection: projection, + name: collection_customizer.name } + + add_create_hooks(collection_customizer, target) + add_update_hooks(collection_customizer, target) + add_delete_hooks(collection_customizer, target) + end + + # The "after" hooks are prepended: `execute_after` stops at the first exception, so a customization + # raising in its own after hook would otherwise drop the record of a write that already happened. + # The "before" hooks stay appended, so they read the data, filter and patch every other customization + # has had its say on. + def add_create_hooks(collection_customizer, target) + collection_customizer.add_hook('Before', 'Create') do |context| + # No record id yet — that is what the column being nullable is for. + rows = [{ record_id: nil, new_values: pick(context.data, target[:columns]) }] + + @snapshots.push(context.data, ids: pending_rows(context.caller, 'create', target[:name], rows)) + end + + collection_customizer.add_hook('After', 'Create', prepend: true) do |context| + pending = @snapshots.pop_for(context.data) + next unless pending + + confirm(pending[:ids].first, + record_id: record_id(context.record, target[:primary_keys]), + new_values: redacted(target[:name], pick(context.record, target[:columns]))) + end + end + + def add_update_hooks(collection_customizer, target) + collection_customizer.add_hook('Before', 'Update') do |context| + records = @snapshots.take(context, target[:projection]) + @snapshots.push( + context.filter, + records: records, + patch: context.patch, + ids: pending_rows( + context.caller, 'update', target[:name], + records.map do |record| + { record_id: record_id(record, target[:primary_keys]), + previous_values: pick(record, context.patch.keys & target[:columns]), + new_values: pick(context.patch, target[:columns]) } + end + ) + ) + end + + collection_customizer.add_hook('After', 'Update', prepend: true) do |context| + pending = @snapshots.pop_for(context.filter) + next unless pending + + confirm_updates(context, pending, target) + end + end + + def add_delete_hooks(collection_customizer, target) + collection_customizer.add_hook('Before', 'Delete') do |context| + records = @snapshots.take(context, target[:projection]) + @snapshots.push( + context.filter, + records: records, + ids: pending_rows( + context.caller, 'delete', target[:name], + records.map do |record| + { record_id: record_id(record, target[:primary_keys]), + previous_values: pick(record, target[:columns]) } + end + ) + ) + end + + collection_customizer.add_hook('After', 'Delete', prepend: true) do |context| + pending = @snapshots.pop_for(context.filter) + next unless pending + + pending[:ids].each { |id| confirm(id) } + end + end + + # The diff is taken against the record as persisted, not the patch that was requested, so a value the + # datasource normalised or a decorator rewrote is recorded as what actually landed. The id is packed from + # those same values, so an update that changed a primary key files the row under the id the history will + # be queried by. + def confirm_updates(context, pending, target) + pks = target[:primary_keys] + persisted = reread(context, pending[:records], pending[:patch], target) + empty = [] + + pending[:records].each_with_index do |record, index| + # No row read back: the write may or may not have landed, and inventing after-values from the patch + # would confirm — or worse, discard — a row for something that may never have happened. Left pending, + # which is exactly what that state means. + after = persisted[record_id(record.merge(pending[:patch]), pks)] + next if after.nil? + + delta = Diff.changed_values(record, after, target[:columns]) + + if delta[:new_values].empty? + empty << pending[:ids][index] + else + before_id = record_id(record, pks) + after_id = record_id(after, pks) + + confirm(pending[:ids][index], + record_id: after_id, + # Only when the key actually moved, so a history query can walk back to the rows filed + # under the id this record used to have. + previous_record_id: after_id == before_id ? nil : before_id, + previous_values: redacted(target[:name], delta[:previous_values]), + new_values: redacted(target[:name], delta[:new_values])) + end + end + + audit_safely { @store.discard(empty.compact) } if empty.any? + end + + # Reads the updated records back in one query, keyed by their own id, so each snapshot can find what + # actually landed — including when the patch moved a primary key. + def reread(context, records, patch, target) + pks = target[:primary_keys] + return {} if records.empty? + + audit_safely do + condition = ids_condition(pks, records.map { |record| record.merge(patch).slice(*pks) }) + filter = ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition) + + context.collection.list(filter, target[:projection]).to_h { |row| [record_id(row, pks), row] } + end || {} + end + + # Built from the primary keys the instrumentation already resolved, rather than through + # ConditionTreeFactory, which would need the underlying collection a hook context does not hand out. + def ids_condition(primary_keys, ids) + leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + factory = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeFactory + + operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + + if primary_keys.size == 1 + key = primary_keys.first + + leaf.new(key, operators::IN, ids.map { |id| id[key] }.uniq) + else + factory.union( + ids.map { |id| factory.intersect(id.map { |key, value| leaf.new(key, operators::EQUAL, value) }) } + ) + end + end + + # The one place the audit trail may refuse an operation, and only under `critical: true`: if we cannot + # record that a write is about to happen, the write does not happen. + def pending_rows(caller, operation, collection, rows) + timestamp = now + correlation_key = correlation_key_for(caller) + identity = identity_of(caller) + + AuditTrail.gate do + @store.append_all( + rows.map do |row| + AuditRecord.new( + timestamp: timestamp, operation: operation, collection: collection, status: PENDING, + correlation_key: correlation_key, record_id: row[:record_id], + previous_values: redacted(collection, row[:previous_values] || {}), + new_values: redacted(collection, row[:new_values] || {}), + **identity + ) + end + ) + end || [] + end + + def confirm(id, attributes = {}) + return unless id + + audit_safely { @store.confirm(id, attributes) } + end + + def redacted(collection, values) + redact(values, @redact[collection] || []) + end + + def record_id(record, primary_keys) + primary_keys.map { |pk| record[pk].to_s }.join('|') + end + + def pick(record, columns) + columns.to_h { |column| [column, record[column]] } + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb new file mode 100644 index 000000000..7ec8fd0c2 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb @@ -0,0 +1,156 @@ +module ForestAdminAgent + module AuditTrail + # Minimal structural diff. Nested hashes and arrays of hashes are recursed into, so only the + # keys/indexes whose leaf value actually changed are kept — a single sub-field change does not + # store the whole object/array. Scalars, primitive arrays, dates and other values are compared and + # kept as a whole. + # + # Ruby's `==` already performs deep, key-order-independent equality on Hash and (ordered) equality + # on Array, so it is used directly as the equality primitive. + module Diff + # A key that does not exist on one side of the diff. It is never stored: the key is simply left out + # of that side's hash, so `{"flag" => nil}` (a key holding nil) and `{}` (no key) stay tellable + # apart in the database — which a revert needs and a string sentinel would only fake. + ABSENT = Object.new.freeze + + module_function + + # @return [Hash{Symbol=>Object}, nil] { previous:, next: } of the changed leaves, or nil when equal. + def diff(before, after) + return nil if before == after + + return diff_hashes(before, after) if before.is_a?(Hash) && after.is_a?(Hash) + + return diff_object_arrays(before, after) if object_array?(before) && object_array?(after) + + { previous: before.nil? ? nil : before, next: after.nil? ? nil : after } + end + + # Build the previous/new value hashes for the writable columns that actually changed. + # + # @param before [Hash] snapshot of the record before the change (string keys) + # @param patch [Hash] the values being written (string keys); only present keys are considered + # @param columns [Array] writable column names to inspect + # @return [Hash{Symbol=>Hash}] { previous_values:, new_values: } + def changed_values(before, patch, columns) + previous_values = {} + new_values = {} + + columns.each do |column| + delta = patch.key?(column) ? diff(before[column], patch[column]) : nil + next unless delta + + previous_values[column] = delta[:previous] + new_values[column] = delta[:next] + end + + { previous_values: previous_values, new_values: new_values } + end + + # Arrays whose every element is a hash (record-like collections, e.g. a workflow history). + def object_array?(value) + value.is_a?(Array) && !value.empty? && value.all?(Hash) + end + + def diff_hashes(before, after) + previous = {} + next_values = {} + + (before.keys | after.keys).each do |key| + sub = diff_at(before, after, key) + next unless sub + + previous[key] = sub[:previous] unless sub[:previous].equal?(ABSENT) + next_values[key] = sub[:next] unless sub[:next].equal?(ABSENT) + end + + { previous: previous, next: next_values } + end + + # A key held with a nil value is not the same thing as a missing key: recursing on the values alone + # reads both as nil and reports no change at all. + def diff_at(before, after, key) + return diff(before[key], after[key]) if before.key?(key) == after.key?(key) + + { + previous: before.key?(key) ? before[key] : ABSENT, + next: after.key?(key) ? after[key] : ABSENT + } + end + + def diff_object_arrays(before, after) + previous = {} + next_values = {} + + [before.length, after.length].max.times do |index| + sub = diff(before[index], after[index]) + next unless sub + + # Same rule as for hash keys: an index one side does not reach is left out of that side, so a + # revert can tell an appended element (drop it) from one whose value became nil (keep it). + previous[index] = sub[:previous] if index < before.length + next_values[index] = sub[:next] if index < after.length + end + + { previous: previous, next: next_values } + end + + # Undo one recorded change: given the value as it stands now and the two sides of the diff that + # produced it, return the value as it was before. Nested hashes are walked so untouched keys keep + # their current value; a key missing from `previous` was added by the change, so it goes away. + # + # @param current [Object] the value as it stands now (may be nil when the record is gone) + # @param previous [Object] the `previous_values` side of the recorded diff + # @param changed [Object] the `new_values` side of the recorded diff + def revert(current, previous, changed) + return revert_array(current, previous, changed) if current.is_a?(Array) && partial?(previous, changed) + return revert_hash(current, previous, changed) if current.is_a?(Hash) && partial?(previous, changed) + + previous + end + + # Both sides being hashes is what `diff` emits for a structural diff; anything else replaced the + # value as a whole. + def partial?(previous, changed) + previous.is_a?(Hash) && changed.is_a?(Hash) + end + + def revert_hash(current, previous, changed) + (previous.keys | changed.keys).each_with_object(current.dup) do |key, result| + if previous.key?(key) + result[key] = revert(current[key], previous[key], changed[key]) + else + # Only the change introduced this key, so before the change there was none. + result.delete(key) + end + end + end + + # Arrays of objects are diffed index by index, and JSON turns those indexes into strings on the way + # back out. Highest index first, so dropping an element the change appended leaves the lower ones + # where the diff expects them. + def revert_array(current, previous, changed) + indexes = (previous.keys | changed.keys).map(&:to_i).sort.reverse + + indexes.each_with_object(current.dup) do |index, result| + if index?(previous, index) + result[index] = revert(current[index], at(previous, index), at(changed, index)) + else + result.delete_at(index) + end + end + end + + def index?(hash, index) + hash.key?(index) || hash.key?(index.to_s) + end + + def at(hash, index) + hash.key?(index) ? hash[index] : hash[index.to_s] + end + + private_class_method :diff_hashes, :diff_at, :diff_object_arrays, :partial?, :revert_hash, + :revert_array, :index?, :at + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/record_state.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/record_state.rb new file mode 100644 index 000000000..6af292c2d --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/record_state.rb @@ -0,0 +1,43 @@ +module ForestAdminAgent + module AuditTrail + # Rebuilds a record as it stood at a given instant by walking its history backwards from the record as + # it stands now, undoing every entry recorded after that instant (newest first). + # + # Only audited columns are reconstructed — read-only, computed and DB-managed fields are never recorded, + # so they cannot be restored. + module RecordState + module_function + + # @param current [Hash, nil] the record as it stands now, nil when it no longer exists + # @param entries [Array] entries recorded after the instant, newest first + # @return [Hash, nil] the record at that instant, nil when it did not exist then + def at(current, entries) + entries.reduce(current) { |state, entry| undo(state, entry) } + end + + def undo(state, entry) + case entry.operation + when 'create' + # Created after the instant, so it did not exist then. An older entry can still bring a previous + # life of the same id back — the walk carries on. + nil + when 'delete' + # The delete recorded the whole record, which is exactly the state it was deleted from. + entry.previous_values + when 'update' + revert_columns(state, entry) + else + # Action rows carry no field change: their two value columns hold what was submitted to the action + # and what it answered, so applying either as a column change would corrupt the rebuild. + state + end + end + + def revert_columns(state, entry) + entry.previous_values.each_with_object((state || {}).dup) do |(column, previous), result| + result[column] = Diff.revert(result[column], previous, entry.new_values[column]) + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb new file mode 100644 index 000000000..39092d28e --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb @@ -0,0 +1,57 @@ +require 'time' + +module ForestAdminAgent + module AuditTrail + # Shared by the two capture layers: {Capture} for the changes Forest writes, {ActionCapture} for the + # smart actions it runs. Holds the write protocol's vocabulary and its failure policy. + module Recording + REDACTED = '[redacted]'.freeze + # A row is inserted before the write and confirmed after it. One left PENDING means the write may or + # may not have landed — that residue is evidence, and it is the point. + PENDING = 'pending'.freeze + DONE = 'done'.freeze + + IDENTITY = { user_id: :id, user_first_name: :first_name, + user_last_name: :last_name, user_email: :email }.freeze + + # Denormalised at write time, so the row says who acted then rather than whoever holds that id today. + # Read defensively: a caller built by another code path need not carry a full identity, and a + # NoMethodError here would refuse the write outright under `critical: true`. + def identity_of(caller) + IDENTITY.transform_values { |reader| caller.respond_to?(reader) ? caller.public_send(reader) : nil } + end + + # Same id for every change made within one request — set on the caller by the agent (see + # CallerParser), mirroring the Node agent's caller.requestId. + # + # nil when the caller carries none, which is what a write outside any request looks like. Inventing one + # would group the row into a request of its own, indistinguishable from a genuine single-row request. + def correlation_key_for(caller) + caller.respond_to?(:request_id) ? caller.request_id : nil + end + + def redact(values, redacted_fields) + return values if redacted_fields.empty? + + values.each_with_object({}) do |(field, value), result| + result[field] = redacted_fields.include?(field) ? REDACTED : value + end + end + + def now + Time.now.utc.iso8601(3) + end + + # Everything after the pending insert is best-effort: by then the write has happened, so raising would + # report a failure for an operation that succeeded (and invite a retry that duplicates it). Losing the + # row is the lesser evil, so it is logged and dropped. Only the pending insert itself can refuse an + # operation, and only under `critical: true` — see {AuditTrail.critical?}. + def audit_safely + yield + rescue StandardError => e + AuditTrail.log_failure(e) + nil + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb new file mode 100644 index 000000000..c72903432 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb @@ -0,0 +1,85 @@ +module ForestAdminAgent + module AuditTrail + # What a "before" hook leaves for the matching "after" hook: the records as they stood, the patch, and the + # ids of the pending rows to confirm. + # + # Entries are keyed by the object the hook decorator hands to both contexts — the filter, or the data on a + # create — because taking the newest entry is wrong as soon as writes nest: an inner write that fails skips + # its after hook and stays on the stack, and the outer hook would then confirm the failed operation's rows + # as done and leave its own stranded. Both directions of that are lies. + # + # An operation raising between the two hooks strands its entry, hence the cap; its rows stay `pending` in + # the table, which is the truthful state for a write that may not have landed. + class Snapshots + include Recording + + # ponytail: 16 deep is far past any legitimate nesting; raise it if one ever gets that far. + MAX_PENDING = 16 + + def push(key, snapshot) + stack = pending + stack.shift while stack.size >= MAX_PENDING + stack.push(snapshot.merge(key: key)) + end + + # The entry this operation left, or nothing rather than someone else's. + # + # A customization that replaced the filter or the data leaves no identity to match on: our before hook saw + # the replacement and the after context carries the original. With a single operation in flight that is + # unambiguous, so it still pairs; with several it does not guess, and the rows stay pending. + def pop_for(key) + stack = pending + index = stack.rindex { |entry| entry[:key].equal?(key) } + index = 0 if index.nil? && stack.size == 1 + + index.nil? ? nil : stack.delete_at(index) + end + + # The records an operation is about to touch, capped. Reading "delete all" unbounded would materialise + # every matched row — and the pending/confirm protocol writes each of them twice. Truncation is logged + # rather than silent: an incomplete audit somebody knows about beats an OOM. + # + # Read outside the write's transaction — hooks bracket the write as separate calls and the data layer + # exposes no lock, on purpose, since it spans ActiveRecord, Mongoid, HTTP APIs. So two updates racing on + # one record both snapshot the same state, and the one that lands second records a `previous_values` that + # was already overwritten. + # + # An empty list on failure rather than no snapshot at all: the after hook pops unconditionally, so + # skipping the push would pair it with an unrelated entry. Reading it goes through the gate, not + # `audit_safely`: knowing what an operation is about to touch is part of being able to record it, so + # under `critical: true` a snapshot that cannot be read refuses the operation. + def take(context, projection) + cap = AuditTrail::MAX_RECORDS_PER_OPERATION + page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 0, limit: cap + 1) + records = AuditTrail.gate { context.collection.list(context.filter.override(page: page), projection) } || [] + return records if records.size <= cap + + refuse_or_truncate(context, records, cap) + end + + private + + # Truncating means the operation writes more records than it audits. Tolerable when the audit trail is + # advisory; under `critical: true` it breaks the one invariant the mode exists for, so the operation is + # refused instead — before the write, so there is nothing to repair. + def refuse_or_truncate(context, records, cap) + AuditTrail.refuse_over_cap! if AuditTrail.critical? + + kept = records.first(cap) + AuditTrail.log_truncation(kept.size, audit_safely { count_matching(context) }) + + kept + end + + def count_matching(context) + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + context.collection.aggregate(context.filter, aggregation).first&.fetch('value', nil) + end + + def pending + Thread.current[:forest_audit_trail_snapshots] ||= [] + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb new file mode 100644 index 000000000..446a4d47d --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb @@ -0,0 +1,53 @@ +begin + require 'active_record' +rescue LoadError + raise LoadError, 'config.audit_trail needs the activerecord gem: add `gem "activerecord"` to your Gemfile.' +end + +module ForestAdminAgent + module AuditTrail + module Sql + # Dedicated abstract base so the audit storage keeps its own connection pool, isolated from the + # host application's ActiveRecord::Base connection. Also the level the `attribute` overrides in + # AuditLog need: declaring them straight on an ActiveRecord::Base child resolves the type + # eagerly and blows up before any connection is established. + class AuditConnectionBase < ActiveRecord::Base + self.abstract_class = true + + class << self + # `establish_connection` is class-level, so two stores pointed at different databases would silently + # clobber each other's pool and both end up writing to whichever connected last. One audit database + # per agent is the supported shape; a second, different one is a configuration mistake worth hearing + # about at boot. + # + # Under one mutex for all stores, not one each: the check, the connect and the assignment have to be + # one step, or two stores connecting at once both pass the check and the loser writes to the winner's + # database. + def connect_to(database) + connection_mutex.synchronize do + if @database && @database != database + raise ForestAdminDatasourceToolkit::Exceptions::ForestException, + 'The audit trail is already connected to another database. One agent, one audit database.' + end + next if @database + + establish_connection(database) + @database = database + end + end + + def disconnect! + connection_mutex.synchronize do + @database = nil + remove_connection + end + end + + def connection_mutex + @connection_mutex ||= Mutex.new + end + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_log.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_log.rb new file mode 100644 index 000000000..a0eb7c442 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_log.rb @@ -0,0 +1,16 @@ +module ForestAdminAgent + module AuditTrail + module Sql + # Abstract template for the audit model. Each Store builds its own concrete subclass bound to its + # own (schema-qualified) table, so stores with different `table_name`/`schema` can't clobber a + # shared one. The JSON attribute overrides force Hash <-> JSON casting on every adapter (Postgres + # json, or text on SQLite) and are inherited by every subclass. + class AuditLog < AuditConnectionBase + self.abstract_class = true + + attribute :previous_values, :json + attribute :new_values, :json + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/field_filter.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/field_filter.rb new file mode 100644 index 000000000..a29737e9f --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/field_filter.rb @@ -0,0 +1,58 @@ +module ForestAdminAgent + module AuditTrail + module Sql + # SQL keeping only the audit entries whose diff touched one of the given fields. Both JSON columns are + # searched: a field the change added exists in `new_values` only, one it removed in `previous_values` + # only. + # + # The test is per adapter, and a field name is always a whole key — never a path — so a name holding a + # dot (`address.city`) has to be quoted or the database reads it as a traversal. + class FieldFilter + COLUMNS = %w[previous_values new_values].freeze + + def initialize(connection) + @connection = connection + end + + def condition(fields) + adapter = @connection.adapter_name.downcase + + case adapter + when /postgres/ then COLUMNS.map { |column| postgres_has_key(column, fields) }.join(' OR ') + when /sqlite/ then json_paths(fields) { |column, path| "json_type(#{column}, #{path}) IS NOT NULL" } + when /mysql|maria/ then mysql_has_keys(fields) + else + raise ForestAdminDatasourceToolkit::Exceptions::ForestException, + "Filtering the audit trail by field is not supported on #{adapter}" + end + end + + private + + # `jsonb_object_keys` rather than the `?|` operator: `?` is a bind placeholder for ActiveRecord and + # the function form needs no escaping. The column is `json`, hence the cast. + def postgres_has_key(column, fields) + list = fields.map { |field| @connection.quote(field) }.join(', ') + + "EXISTS (SELECT 1 FROM jsonb_object_keys(#{column}::jsonb) AS key WHERE key IN (#{list}))" + end + + # `json_type` and not `json_extract`: a key holding a JSON null extracts as SQL NULL, which would + # read as "no such key". + def json_paths(fields) + COLUMNS.flat_map { |column| fields.map { |field| yield(column, json_path(field)) } }.join(' OR ') + end + + def mysql_has_keys(fields) + paths = fields.map { |field| json_path(field) }.join(', ') + + COLUMNS.map { |column| "JSON_CONTAINS_PATH(#{column}, 'one', #{paths})" }.join(' OR ') + end + + def json_path(field) + @connection.quote(%($."#{field.to_s.gsub(/["\\]/) { |char| "\\#{char}" }}")) + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb new file mode 100644 index 000000000..9cada5f0d --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb @@ -0,0 +1,53 @@ +module ForestAdminAgent + module AuditTrail + module Sql + # The audit table's schema, as an ordered, append-only list. Nothing has shipped yet, so this is still a + # single entry; once a release is out, never edit one — add another, since a database out there has + # already recorded the earlier ones as applied. Every statement tolerates being replayed + # (`if_not_exists`), which is what makes a lost race harmless. + module Migrations + ALL = [ + { + name: '001-create-audit-logs', + up: lambda do |connection, table| + # if_not_exists: a non-PG race (no advisory lock) can let two instances both reach here. + connection.create_table(table, if_not_exists: true) do |t| + t.datetime :timestamp, null: false + t.string :operation, null: false + t.string :collection, null: false + # Nullable, because a create's pending row has no id yet; text, because a packed composite id + # outgrows a varchar. + t.text :record_id + # No default on purpose: every write sets it, so a row arriving without one is a bug worth + # hearing about rather than a row that quietly claims to be done. + t.string :status, null: false + t.integer :user_id + # Denormalised from the caller at write time: who acted then, not whoever holds that id today. + t.text :user_first_name + t.text :user_last_name + t.text :user_email + # Set only on an update that moved a writable primary key: the id the row was filed under + # before. What lets a history query follow a record across a rename. + t.text :previous_record_id + # Smart-action rows only. + t.text :action_name + t.string :correlation_key + t.json :previous_values + t.json :new_values + end + + base = table.split('.').last + # MySQL cannot index unbounded TEXT, so that one index needs a length prefix. + record_id_index = { name: "#{base}_record_id", if_not_exists: true } + record_id_index[:length] = 255 if connection.adapter_name.downcase.match?(/mysql|maria/) + + connection.add_index(table, :record_id, **record_id_index) + connection.add_index(table, :correlation_key, name: "#{base}_correlation_key", if_not_exists: true) + connection.add_index(table, :user_id, name: "#{base}_user_id", if_not_exists: true) + end + } + ].freeze + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb new file mode 100644 index 000000000..d396ad432 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb @@ -0,0 +1,117 @@ +module ForestAdminAgent + module AuditTrail + module Sql + # Applies {Migrations::ALL} to the audit table, tracking what has run in a companion table named after + # it — `audit_logs_migration` beside `audit_logs` (both namespaced in the `forest` schema on Postgres). + # One tracker per audited table, so two stores configured with different `table_name`s each get their + # own schema history instead of reading each other's as done. + # + # On Postgres the migrations run inside a transaction-scoped advisory lock, so several agent + # instances booting at once apply them one after another instead of racing on the same DDL. The + # schema is created (and committed) first, made idempotent (CREATE SCHEMA IF NOT EXISTS + + # tolerating a concurrent create), because the lock cannot cover a not-yet-existing schema. + class Migrator + # Arbitrary but stable key pair identifying the audit-trail migration critical section. + ADVISORY_LOCK = [0x464f, 0x5254].freeze # "FO", "RT" + # duplicate_schema, and the unique violation on pg_namespace the same race can raise instead. + DUPLICATE_SCHEMA_STATES = %w[42P06 23505].freeze + + def initialize(connection, schema:, table_name:) + @connection = connection + @schema = schema # nil on adapters without schema support + @table_name = table_name + end + + def run + ensure_schema + + if postgres? + @connection.transaction do + @connection.execute("SELECT pg_advisory_xact_lock(#{ADVISORY_LOCK[0]}, #{ADVISORY_LOCK[1]})") + apply_pending + end + else + apply_pending + end + end + + private + + def postgres? + @connection.adapter_name.downcase.include?('postgres') + end + + def schema? + postgres? && @schema.present? + end + + def qualified(name) + schema? ? "#{@schema}.#{name}" : name + end + + # Create the schema first and commit it: the migrations open DDL on the same connection, and a + # CREATE SCHEMA still pending in the lock transaction would not be visible to them. + def ensure_schema + return unless schema? + + @connection.execute("CREATE SCHEMA IF NOT EXISTS #{@connection.quote_schema_name(@schema)}") + rescue ActiveRecord::RecordNotUnique + # 23505 on pg_namespace, already mapped to its own class by ActiveRecord: another instance + # created the schema between our IF NOT EXISTS check and the create itself. + nil + rescue ActiveRecord::StatementInvalid => e + raise unless duplicate_schema?(e) + end + + # By SQLSTATE where the adapter exposes one, so an unrelated failure (no permission to create a + # schema, say) is not read as a lost race just because its message says "exists". + def duplicate_schema?(error) + state = sql_state(error) + return DUPLICATE_SCHEMA_STATES.include?(state) if state + + /already exists|duplicate/i.match?(error.message) + end + + def sql_state(error) + cause = error.cause + return nil unless cause.respond_to?(:result) && defined?(PG::Result) + + cause.result.error_field(PG::Result::PG_DIAG_SQLSTATE) + rescue StandardError + nil + end + + def apply_pending + done = applied_migrations + table = qualified(@table_name) + + Migrations::ALL.each do |migration| + next if done.include?(migration[:name]) + + migration[:up].call(@connection, table) + @connection.execute( + "INSERT INTO #{@connection.quote_table_name(migrations_table)} (name) " \ + "VALUES (#{@connection.quote(migration[:name])})" + ) + end + end + + def applied_migrations + ensure_migrations_table + + @connection.select_values("SELECT name FROM #{@connection.quote_table_name(migrations_table)}") + end + + def ensure_migrations_table + @connection.create_table(migrations_table, id: false, if_not_exists: true) do |t| + t.string :name, null: false + end + end + + def migrations_table + qualified("#{@table_name}_migration") + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/text_search.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/text_search.rb new file mode 100644 index 000000000..d8404e9d1 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/text_search.rb @@ -0,0 +1,78 @@ +require 'json' + +module ForestAdminAgent + module AuditTrail + module Sql + # SQL keeping only the audit entries a free-text term matches, case-insensitively and as a substring: + # the action's name, who acted, and the keys and values recorded on both sides of the change — at any + # depth, since only the changed leaves of a JSON column are stored and the term has to reach them. + # + # Deliberately not searched: operation, correlation_key, record_id, collection, status and timestamp. + # Machine identifiers nobody searches for, and matching them turns one term into a pile of confusing + # hits. + # + # The values are matched against the JSON document as text, which is what lets one condition reach any + # depth and compose with pagination and the count. It cannot use an index, which is affordable here + # because a history query is already narrowed to one record. + class TextSearch + TEXT_COLUMNS = %w[action_name user_first_name user_last_name user_email].freeze + JSON_COLUMNS = %w[previous_values new_values].freeze + # `!` rather than a backslash: MySQL treats a backslash as an escape inside string literals too, so + # `ESCAPE '\'` needs doubling there and nowhere else. + ESCAPE = '!'.freeze + # A masked value is stored as this. It is removed before matching, so a search for "redacted" cannot + # hit it — and since the real value was never recorded, searching that finds nothing either. A search + # must never confirm a value the trail refused to keep. + REDACTED = Recording::REDACTED + + def initialize(connection) + @connection = connection + end + + def condition(term) + text = term.to_s.downcase + # The value objects are matched as serialized JSON, where a quote, a backslash or a newline is + # escaped — so `15" monitor` sits in the document as `15\" monitor` and the raw term would never + # find it. Escaping the term the same way makes it match, and stops a bare quote from matching the + # document's own structure. + clauses = TEXT_COLUMNS.map { |column| like(column, text) } + clauses += JSON_COLUMNS.map { |column| like(searchable_json(column), json_escaped(text)) } + + clauses.join(' OR ') + end + + private + + def like(expression, text) + "LOWER(#{expression}) LIKE #{@connection.quote("%#{escape(text)}%")} ESCAPE '#{ESCAPE}'" + end + + # What JSON generation would have done to the term: `to_json` on the string, minus its own quotes. + def json_escaped(text) + text.to_json[1..-2] + end + + def searchable_json(column) + "REPLACE(#{as_text(column)}, #{@connection.quote(REDACTED)}, '')" + end + + def as_text(column) + adapter = @connection.adapter_name.downcase + + case adapter + when /postgres/ then "#{column}::text" + when /sqlite/ then column + when /mysql|maria/ then "CAST(#{column} AS CHAR)" + else + raise ForestAdminDatasourceToolkit::Exceptions::ForestException, + "Searching the audit trail is not supported on #{adapter}" + end + end + + def escape(term) + term.gsub(/[!%_]/) { |char| "#{ESCAPE}#{char}" } + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb new file mode 100644 index 000000000..dd5d1272b --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb @@ -0,0 +1,253 @@ +require 'time' + +module ForestAdminAgent + module AuditTrail + # SQL-backed storage that both writes every audited change and reads the per-record history back. + # + # {#connect!} opens the connection and migrates; the agent factory calls it at boot rather than leaving it + # to the first write, so a database the agent cannot reach is a startup failure instead of an agent that + # looks healthy while recording nothing — and, under `critical: true`, instead of one that refuses every + # write the moment somebody first tries to save something. + class Store + DEFAULT_SCHEMA = 'forest'.freeze + DEFAULT_TABLE = 'audit_logs'.freeze + COLUMNS = %i[timestamp operation collection record_id previous_record_id status user_id user_first_name + user_last_name user_email action_name correlation_key previous_values new_values].freeze + AUTHOR_COLUMNS = %i[user_id user_first_name user_last_name user_email].freeze + + def initialize(database:, schema: DEFAULT_SCHEMA, table_name: DEFAULT_TABLE) + @database = database + @schema = schema + @table_name = table_name + @mutex = Mutex.new + @ready = false + end + + def connect! + ensure_ready + + self + end + + def append(record) + append_all([record]).first + end + + # Inserts rows and returns their ids, in the order given. Batched, because a "delete all" snapshot can + # be thousands of records and the pending/confirm protocol writes each of them twice. + # + # The ids are matched to their rows by `record_id` rather than by the order RETURNING happens to come + # back in, which Postgres does not promise: pairing them positionally would confirm each pending row with + # another record's diff. One row per record per operation, so that key is unique within a batch — bar a + # pending create, which has no id yet and is always a batch of one. + def append_all(records) + return [] if records.empty? + + rows = records.map { |record| to_row(record) } + return rows.map { |row| model.create!(row).id } unless batch_returning?(rows) + + returned = model.insert_all(rows, returning: %i[id record_id]).rows.to_h { |id, key| [key, id] } + + rows.map { |row| returned[row[:record_id]] } + end + + # One insert per row when the ids cannot be matched back: no RETURNING on this adapter (MySQL), or a + # batch whose record ids are not distinct enough to pair on. + def batch_returning?(rows) + return false unless model.connection.supports_insert_returning? + + keys = rows.map { |row| row[:record_id] } + + keys.none?(&:nil?) && keys.uniq.size == keys.size + end + + def confirm(id, attributes) + row = model.find_by(id: id) + + row&.update!(**attributes, status: Recording::DONE) + end + + # A write that turned out to change nothing leaves no trace: the pending row goes rather than sitting + # there implying the write is unaccounted for. + def discard(ids) + model.where(id: ids).delete_all unless ids.empty? + end + + def list_by_record(collection:, record_id:, skip: 0, limit: nil, user_ids: nil, start_timestamp: nil, + end_timestamp: nil, fields: nil, search: nil, order: 'asc') + # `id` (insertion order) breaks ties on equal timestamps in both directions, keeping pages + # deterministic and stable. + relation = scope(collection, record_id, user_ids: user_ids, start_timestamp: start_timestamp, + end_timestamp: end_timestamp, fields: fields, search: search) + .order(timestamp: order.to_s == 'desc' ? :desc : :asc, id: :asc) + .offset(skip || 0) + relation = relation.limit(limit) unless limit.nil? + + relation.map { |row| from_row(row) } + end + + def count_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil, + end_timestamp: nil, fields: nil, search: nil) + scope(collection, record_id, user_ids: user_ids, start_timestamp: start_timestamp, + end_timestamp: end_timestamp, fields: fields, search: search).count + end + + # The distinct authors of the entries the current filters match, whatever page is being asked for. The + # identity comes from the rows themselves, so a user who has since been renamed or removed still reads + # as they were when they acted. + def authors_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil, + end_timestamp: nil, fields: nil, search: nil) + scope(collection, record_id, user_ids: user_ids, start_timestamp: start_timestamp, + end_timestamp: end_timestamp, fields: fields, search: search) + .where.not(user_id: nil) + .distinct + .pluck(*AUTHOR_COLUMNS) + .map { |values| AUTHOR_COLUMNS.zip(values).to_h } + .uniq { |author| author[:user_id] } + end + + # The ids this record was renamed from, each with the moment it stopped being that id. Walking those back + # is what lets a history query reach rows written before a rename — they stay under the id they were + # written with, since that is the id they were true of — and the moment bounds how far: the id it left may + # have been taken by another record afterwards, whose rows are none of this record's business. + def renamed_from(collection:, record_id:) + model.where(collection: collection, record_id: record_id) + .where.not(previous_record_id: nil) + .pluck(:previous_record_id, :timestamp, :id) + .group_by(&:first) + .map do |id, rows| + # The row id comes along as the tie-breaker: the trail orders itself by (timestamp, id), so a + # bound that knew only the timestamp would mean something slightly different from "before". + _, at, row = rows.max_by { |(_, timestamp, row_id)| [timestamp, row_id] } + + { id: id, until: as_iso(at), until_row: row } + end + end + + # Entries recorded strictly after `timestamp`, newest first: what a state reconstruction has to undo. + # Strictly after, so an entry stamped exactly at the requested instant counts as part of that state + # instead of being reverted out of it. + # Confirmed rows only: a pending one records an attempt whose outcome is unknown, and undoing a change + # that may never have happened would invent a state the record was never in. The history reads keep + # pending rows — they are evidence, and `status` tells the reader what they are — but a reconstruction + # cannot act on them. + def list_since(collection:, record_id:, timestamp:) + model.where(collection: collection, status: Recording::DONE) + .where(*segments_condition(record_id)) + .where('timestamp > ?', as_time(timestamp)) + .order(timestamp: :desc, id: :desc) + .map { |row| from_row(row) } + end + + def list_by_correlation(collection:, record_id:, correlation_key:) + list_by_correlations(collection: collection, record_id: record_id, correlation_keys: [correlation_key]) + end + + def list_by_correlations(collection:, record_id:, correlation_keys:) + return [] if correlation_keys.empty? + + model.where(collection: collection, record_id: record_id, correlation_key: correlation_keys) + .order(:timestamp, :id) + .map { |row| from_row(row) } + end + + private + + # Every filter is an AND, so the count matches exactly what a page of this history holds. + def scope(collection, record_id, user_ids: nil, start_timestamp: nil, end_timestamp: nil, + fields: nil, search: nil) + relation = model.where(collection: collection).where(*segments_condition(record_id)) + relation = relation.where(user_id: user_ids) if user_ids + relation = relation.where(Sql::FieldFilter.new(model.connection).condition(fields)) if fields&.any? + relation = relation.where(Sql::TextSearch.new(model.connection).condition(search)) if search + # Compare as Time so ActiveRecord casts the bound to the datetime column's storage format + # (raw ISO strings with a `Z` would compare lexically against the cast rows and never match). + relation = relation.where('timestamp >= ?', as_time(start_timestamp)) if start_timestamp + relation = relation.where('timestamp <= ?', as_time(end_timestamp)) if end_timestamp + relation + end + + def as_time(value) + value.is_a?(::Time) ? value : ::Time.iso8601(value.to_s) + end + + def as_iso(value) + value.respond_to?(:iso8601) ? value.iso8601(3) : value.to_s + end + + # One record's history is its current id plus every id it was renamed from, each earlier one only up to + # the rename: `record_id = '7' OR (record_id = '1' AND timestamp <= …)`. A plain `IN` would hand over the + # rows of whichever record holds that id now. + # + # Takes an id, several, or segments — `{ id:, until: }` — so a caller that has no rename to care about + # simply passes the id. + def segments_condition(record_id) + binds = [] + sql = Array(record_id).map { |value| value.is_a?(Hash) ? value : { id: value, until: nil } }.map do |segment| + binds << segment[:id] + next 'record_id = ?' unless segment[:until] + + binds << as_time(segment[:until]) + # Same millisecond as the rename, and the id says which side of it a row falls on: another record + # taking the abandoned key that fast would otherwise land in this record's history. + next '(record_id = ? AND timestamp <= ?)' unless segment[:until_row] + + binds << as_time(segment[:until]) << segment[:until_row] + '(record_id = ? AND (timestamp < ? OR (timestamp = ? AND id <= ?)))' + end + + [sql.join(' OR '), *binds] + end + + def model + ensure_ready + @model + end + + def ensure_ready + return if @ready + + @mutex.synchronize do + return if @ready + + Sql::AuditConnectionBase.connect_to(@database) + connection = Sql::AuditConnectionBase.connection + Sql::Migrator.new(connection, schema: schema_for(connection), table_name: @table_name).run + @model = build_model(qualified(connection)) + @ready = true + end + end + + # A per-instance concrete subclass bound to this store's own table, so distinct stores can't + # clobber each other's table name. reset_column_information drops stale metadata for the table + # the migration just created/evolved. + def build_model(table) + Class.new(Sql::AuditLog) { self.table_name = table }.tap(&:reset_column_information) + end + + def schema_for(connection) + connection.adapter_name.downcase.include?('postgres') ? @schema : nil + end + + def qualified(connection) + schema = schema_for(connection) + schema ? "#{schema}.#{@table_name}" : @table_name + end + + # An AuditRecord is a Struct and a row answers to `[]` too, so the mapping is the column list itself. + def to_row(record) + COLUMNS.to_h { |column| [column, record[column]] } + end + + def from_row(row) + values = COLUMNS.to_h { |column| [column, row[column]] } + values[:id] = row.id + values[:timestamp] = row.timestamp.respond_to?(:iso8601) ? row.timestamp.iso8601(3) : row.timestamp.to_s + values[:previous_values] ||= {} + values[:new_values] ||= {} + + AuditRecord.new(**values) + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/builder/agent_factory.rb b/packages/forest_admin_agent/lib/forest_admin_agent/builder/agent_factory.rb index 1974d6baa..4fca40d36 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/builder/agent_factory.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/builder/agent_factory.rb @@ -58,6 +58,7 @@ def use(plugin, options = {}) end def build + install_audit_trail @container.register(:datasource, @customizer.datasource(@logger)) # Reset route cache to ensure routes are computed with all customizations @@ -289,12 +290,31 @@ def build_cache @options[:customize_error_message] = clean_option_value(@options[:customize_error_message], 'config.customize_error_message =') @options[:logger] = clean_option_value(@options[:logger], 'config.logger =') + build_audit_trail_store @container.register(:config, @options.to_h) configure_rpc_polling_pool if @options[:rpc_max_polling_threads] end + # The audit trail switches on as soon as a database is configured. The store connects and migrates here, + # at boot, rather than on the first write: an audit database the agent cannot reach should stop it + # starting, not leave it looking healthy while recording nothing — and under `critical: true` it would + # otherwise refuse every write from the moment somebody first tried to save something. + def build_audit_trail_store + options = @options[:audit_trail] + return unless options && options[:database] + + options[:store] = AuditTrail::Store.new(**options.slice(:database, :schema, :table_name).compact).connect! + end + + def install_audit_trail + options = @options[:audit_trail] + return if options.nil? || options[:store].nil? + + @customizer.use(AuditTrail::Capture, { store: options[:store], redact: options[:redact] }) + end + def configure_rpc_polling_pool max_threads = @options[:rpc_max_polling_threads].to_i return unless max_threads.positive? diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/http/correlation_id.rb b/packages/forest_admin_agent/lib/forest_admin_agent/http/correlation_id.rb new file mode 100644 index 000000000..ded3da334 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/http/correlation_id.rb @@ -0,0 +1,37 @@ +require 'securerandom' + +module ForestAdminAgent + module Http + # Per-request correlation id, generated by the agent. Shared between the caller (so the audit + # trail can group every change made within one request) and the response header echoed back to + # the client. Mirrors the Node agent's `context.state.requestId` + `x-forest-correlation-id` + # header: the agent generates the id, never reads it from the incoming request. + # + # Stored thread-locally and generated lazily on first read (e.g. when the caller is parsed). The + # host resets it at the start of each request so a pooled thread never reuses a previous id. + module CorrelationId + HEADER = 'x-forest-correlation-id'.freeze + KEY = :forest_admin_correlation_id + + module_function + + # Lazily generate and memoize the id for the current request/thread. + def current + Thread.current[KEY] ||= SecureRandom.uuid + end + + # The id if one was generated during this request, otherwise nil (does not generate one). + def current? + Thread.current[KEY] + end + + def current=(value) + Thread.current[KEY] = value + end + + def reset! + Thread.current[KEY] = nil + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/http/correlation_id_middleware.rb b/packages/forest_admin_agent/lib/forest_admin_agent/http/correlation_id_middleware.rb new file mode 100644 index 000000000..fbd6c6e99 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/http/correlation_id_middleware.rb @@ -0,0 +1,29 @@ +module ForestAdminAgent + module Http + # Rack middleware echoing the agent-generated correlation id back to the client, mirroring the + # Node agent's `correlationIdMiddleware` (`router.use(...)`). Hosts mount it in their middleware + # stack; CORS exposure of the header is handled by the host's CORS config (see the Rails engine). + # + # The id itself is generated lazily by the agent during the request (see CorrelationId, called + # from CallerParser). This middleware only resets the thread-local around the request — so a + # pooled thread never reuses a previous id — and sets the response header when one was generated. + class CorrelationIdMiddleware + def initialize(app) + @app = app + end + + def call(env) + CorrelationId.reset! + + status, headers, body = @app.call(env) + + id = CorrelationId.current? + headers[CorrelationId::HEADER] = id if id + + [status, headers, body] + ensure + CorrelationId.reset! + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb b/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb index 4bd0b896f..2c7063a2a 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb @@ -50,6 +50,12 @@ def self.routes { name: 'charts', handler: -> { Charts::Charts.new.routes } }, { name: 'collections', handler: -> { Capabilities::Collections.new.routes } }, { name: 'native_query', handler: -> { Resources::NativeQuery.new.routes } }, + # Both must come before the routes matching on `:collection_name`: Rails matches in + # definition order, so `/_audit-trail/correlations` would otherwise be read as + # `/:collection_name/:id` and 404 on a collection named `_audit-trail`. Correlation first, so + # `/_audit-trail/correlations` wins over the per-record `/_audit-trail/:collection_name/:id`. + { name: 'audit_trail_correlation', handler: -> { Resources::AuditTrailCorrelation.new.routes } }, + { name: 'audit_trail', handler: -> { Resources::AuditTrail.new.routes } }, { name: 'count', handler: -> { Resources::Count.new.routes } }, { name: 'delete', handler: -> { Resources::Delete.new.routes } }, { name: 'csv', handler: -> { Resources::Csv.new.routes } }, diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb index 265f56b35..e10e71605 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb @@ -75,7 +75,7 @@ def handle_request(args = {}) fields.reject { |field| field.type == 'Layout' } ) - result = context.collection.execute(context.caller, @action_name, data, filter_for_caller) + result = execute_and_audit(context, args, data, filter_for_caller) { content: ForestAdminAgent::Utils::ActionResult.parse(result) } end @@ -117,6 +117,81 @@ def handle_hook_request(args = {}) private + # Recorded as pending before the action runs and confirmed after, so an action that takes the process + # down with it still leaves evidence that it started. A failed run is worth recording too — "who tried + # to run this" is usually the interesting part — and an action answering with an Error result failed + # just as much as one that raised, it simply said so through `result_builder.error`. + def execute_and_audit(context, _args, data, filter) + pending = audit_pending(context, data, filter) + + begin + result = context.collection.execute(context.caller, @action_name, data, filter) + rescue StandardError + audit_confirm(pending, failed: true) + raise + end + + audit_confirm(pending, result: result, failed: error_result?(result)) + + result + end + + def error_result?(result) + result.is_a?(Hash) && result[:type] == 'Error' + end + + # Everything audit-related sits inside the gate, the record selection included: without an audit + # database none of it runs at all, and a failure refuses the action only under `critical: true` — which + # is safe here, since the action has not run yet. + def audit_pending(context, data, filter) + store = ForestAdminAgent::AuditTrail.store + return [] unless store + + ForestAdminAgent::AuditTrail.gate do + action_capture(store).pending( + caller: context.caller, + collection: context.collection.name, + action_name: @action_name, + form_values: data, + record_ids: audited_record_ids(context, filter) + ) + end || [] + end + + def audit_confirm(pending, result: nil, failed: false) + store = ForestAdminAgent::AuditTrail.store + + action_capture(store).confirm(pending, result: result, failed: failed) if store && pending.any? + end + + def action_capture(store) + ForestAdminAgent::AuditTrail::ActionCapture.new(store, ForestAdminAgent::AuditTrail.options[:redact]) + end + + # Packed ids, the form the audit store keys on — read back through the caller's own filter rather than + # taken from the request. The ids a client sends are a claim: in a compliance record, asserting that an + # operator acted on a record their scope excludes is worse than a missing row. A global action targets + # no record, and a selection wider than the cap is recorded as one row attached to none. + def audited_record_ids(context, filter) + return [] if context.collection.schema[:actions][@action_name].scope == Types::ActionScope::GLOBAL + + cap = ForestAdminAgent::AuditTrail::MAX_RECORDS_PER_OPERATION + primary_keys = ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(context.collection) + records = context.collection.list( + context.caller, filter.override(page: Page.new(offset: 0, limit: cap + 1)), Projection.new(primary_keys) + ) + + return records.map { |record| Utils::Id.pack_id(context.collection, record) } if records.size <= cap + + # Same rule as a bulk write: recording one unattached row for a run that touched more records than + # the cap is a partial audit, which `critical` exists to refuse. Nothing has run yet — the gate is + # ahead of `execute` — so refusing costs nothing to repair. + ForestAdminAgent::AuditTrail.refuse_over_cap! if ForestAdminAgent::AuditTrail.critical? + + ForestAdminAgent::AuditTrail.log_truncation(0, nil) + [] + end + def middleware_custom_action_approval_request_data(args) raise Http::Exceptions::UnprocessableError if args.dig(:params, :data, :attributes, :requester_id) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/capabilities/collections.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/capabilities/collections.rb index b25d63ff3..869d9cd3f 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/capabilities/collections.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/capabilities/collections.rb @@ -67,12 +67,22 @@ def handle_request(args = {}) canUseProjectionOnGetOne: true, canUseProjectionViaHeader: true, canUseProjectionViaHeaderOnList: true, - canUseMultipleFieldsProjectionOnRelation: true + canUseMultipleFieldsProjectionOnRelation: true, + canUseAuditTrail: audit_trail_enabled? } }, status: 200 } end + + private + + # True only where the store the record-history route reads from exists — the same lookup that route + # mounts itself on, so the capability cannot drift from what the routes actually serve. The front gates + # its History tab on this. + def audit_trail_enabled? + !::ForestAdminAgent::AuditTrail.store.nil? + end end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb new file mode 100644 index 000000000..b34109550 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb @@ -0,0 +1,237 @@ +require 'active_support/time' + +module ForestAdminAgent + module Routes + module Resources + # Record-history route, mirroring the Node agent's `/_audit-trail/{collection}/:id`. + # + # Registered only when `config.audit_trail[:database]` is set, in which case the agent factory + # built the store the capture layer writes to. + class AuditTrail < AbstractAuthenticatedRoute + include ForestAdminAgent::Utils + include AuditTrailRoute + + DEFAULT_PAGE_SIZE = 20 + MAX_PAGE_SIZE = 100 + DATE_ONLY = /\A\d{4}-\d{2}-\d{2}\z/ + # Wall-clock datetime, `T` or space separator, seconds optional: `YYYY-MM-DD[T ]HH:mm[:ss]`. + DATE_TIME = /\A(\d{4}-\d{2}-\d{2})[T ](\d{2}):(\d{2})(?::(\d{2}))?\z/ + + def setup_routes + return self unless store + + add_route( + 'forest_audit_trail', + 'get', + '/_audit-trail/:collection_name/:id', + ->(args) { handle_request(args) } + ) + add_route( + 'forest_audit_trail_state', + 'get', + '/_audit-trail/:collection_name/:id/state', + ->(args) { handle_state(args) } + ) + + self + end + + def handle_request(args = {}) + context = build(args) + context.permissions.can?(:read, context.collection) + assert_record_in_scope(context, context.collection, args[:params]['id']) + + skip, limit = parse_pagination(args) + filters = { + collection: context.collection.name, + # args[:params]['id'] is already Forest's packed id, the form the audit store keys on — plus any id + # this record was filed under before a rename, each bounded by when it stopped being that id. + record_id: record_segments(context.collection, args[:params]['id']), + **parse_filters(args) + } + + history = store.list_by_record(**filters, skip: skip, limit: limit, order: parse_sort(args)) + # `count` reflects the active filters (not the absolute total) and is independent of the page. + count = store.count_by_record(**filters) + + { + name: args[:params]['collection_name'], + content: { data: history.map { |record| serialize_record(record) }, meta: meta(args, filters, count) } + } + end + + # Record as it stood at `timestamp`: the current record with every later entry undone. `data` is + # null when the record did not exist yet (or not any more) at that instant. Shape matches the Node + # agent's handleStateAt — `data` and nothing else. + def handle_state(args = {}) + context = build(args) + context.permissions.can?(:read, context.collection) + # Authorizes and reads in one query: the record it hands back is the one the scope covered. + current = scoped_record( + context, context.collection, args[:params]['id'], audited_projection(context.collection) + ) + + timestamp = parse_state_timestamp(args) + entries = store.list_since( + collection: context.collection.name, + record_id: record_segments(context.collection, args[:params]['id']), + timestamp: timestamp + ) + # Fully qualified: inside this class, `AuditTrail` is the route itself. + state = ::ForestAdminAgent::AuditTrail::RecordState.at(current, entries) + + { name: args[:params]['collection_name'], content: { data: state } } + end + + private + + # `availableUsers` rides along on the first fetch only — the front keeps the list it saw — and lists the + # distinct authors of the entries the current filters match, whatever page was asked for. The identity + # comes from the rows, so someone since renamed or removed still reads as they were when they acted. + def meta(args, filters, count) + return { count: count } unless first_fetch?(args) + + { count: count, availableUsers: available_users(filters) } + end + + def first_fetch?(args) + page = args.dig(:params, 'page') + + (page.is_a?(Hash) ? page['number'].to_i : 0) <= 1 + end + + def available_users(filters) + store.authors_by_record(**filters).map do |author| + { id: author[:user_id], firstName: author[:user_first_name], + lastName: author[:user_last_name], email: author[:user_email] } + end + end + + # An ISO-8601 instant, or the same wall-clock forms the history filters accept, read in the request + # timezone. + def parse_state_timestamp(args) + raw = args.dig(:params, 'timestamp').to_s + raise Http::Exceptions::ValidationError, 'Missing timestamp' if raw.empty? + # A wall-clock value carries no offset, so it belongs to the request timezone. Handing it to + # Time.iso8601 would read it in the server's instead — silently, since it parses just fine. + return parse_date_boundary(raw, request_timezone(args), :start) if wall_clock?(raw) + + begin + Time.iso8601(raw).utc.iso8601(3) + rescue ArgumentError + parse_date_boundary(raw, request_timezone(args), :start) + end + end + + def wall_clock?(raw) + DATE_ONLY.match?(raw) || DATE_TIME.match?(raw) + end + + # JSON:API `sort`: `timestamp` → oldest first, anything else (absent/unsupported) → newest first. + def parse_sort(args) + args.dig(:params, 'sort').to_s == 'timestamp' ? 'asc' : 'desc' + end + + # JSON:API pagination: 1-based page[number] (default 1) and page[size] (default 20, capped at + # 100). Out-of-bound or non-numeric values fall back to the defaults rather than erroring. + def parse_pagination(args) + # `?page=foo` reaches us as a bare String, which `dig` refuses to walk into. + page = args.dig(:params, 'page') + page = {} unless page.is_a?(Hash) + + size = page['size'].to_i + size = DEFAULT_PAGE_SIZE if size < 1 + size = MAX_PAGE_SIZE if size > MAX_PAGE_SIZE + + number = page['number'].to_i + number = 1 if number < 1 + + [(number - 1) * size, size] + end + + def request_timezone(args) + timezone = args.dig(:params, 'timezone').to_s + + timezone.empty? ? 'UTC' : timezone + end + + def parse_filters(args) + timezone = request_timezone(args) + + { + user_ids: parse_user_ids(args.dig(:params, 'userIds')), + fields: parse_fields(args.dig(:params, 'fields')), + search: parse_search(args.dig(:params, 'search')), + start_timestamp: parse_date_boundary(args.dig(:params, 'startDate'), timezone, :start), + end_timestamp: parse_date_boundary(args.dig(:params, 'endDate'), timezone, :end) + }.compact + end + + # Free text, trimmed; blank means no filter rather than a term that matches everything. + def parse_search(raw) + term = raw.to_s.strip + + term.empty? ? nil : term + end + + # Comma-separated field names, kept verbatim (a name may hold a dot). Empty after parsing → no filter. + def parse_fields(raw) + return nil if raw.nil? + + names = (raw.is_a?(Array) ? raw : raw.to_s.split(',')).map { |name| name.to_s.strip }.reject(&:empty?) + names.empty? ? nil : names + end + + # Comma-separated integer ids; non-numeric tokens are dropped. Empty after parsing → no filter. + def parse_user_ids(raw) + return nil if raw.nil? || raw.to_s.empty? + + ids = raw.to_s.split(',').map(&:strip).grep(/\A\d+\z/).map(&:to_i) + ids.empty? ? nil : ids + end + + # `startDate`/`endDate` accept a bare day (`YYYY-MM-DD`) or a wall-clock datetime + # (`YYYY-MM-DD[T ]HH:mm[:ss]`), read as local time in the request timezone and returned as a UTC + # ISO instant the store can compare against stored timestamps. + def parse_date_boundary(raw, timezone, boundary) + return nil if raw.nil? || raw.to_s.empty? + + zone = Time.find_zone(timezone) + raise Http::Exceptions::ValidationError, "Invalid timezone: \"#{timezone}\"" if zone.nil? + + instant = begin + local_instant(zone, raw.to_s, boundary) + rescue ArgumentError + nil + end + + if instant.nil? + raise Http::Exceptions::ValidationError, + "Invalid date: \"#{raw}\" (expected YYYY-MM-DD or YYYY-MM-DDTHH:mm)" + end + + instant.utc.iso8601(3) + end + + def local_instant(zone, raw, boundary) + if DATE_ONLY.match?(raw) + day = zone.parse(raw) + # Bare day → start (00:00:00.000) or end (23:59:59.999) of that local day. + boundary == :end ? day.end_of_day : day.beginning_of_day + elsif (match = DATE_TIME.match(raw)) + date, hours, minutes, seconds = match.captures + base = zone.parse("#{date}T#{hours}:#{minutes}") + if seconds + base.change(sec: seconds.to_i, usec: 0) + elsif boundary == :end + # Minutes-only end boundary stays inclusive to :59.999; start stays at :00.000. + base.change(sec: 59, usec: 999_000) + else + base + end + end + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb new file mode 100644 index 000000000..a2767d7cd --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb @@ -0,0 +1,101 @@ +module ForestAdminAgent + module Routes + module Resources + # Correlation-scoped record-history routes, mirroring the Node agent's + # `/_audit-trail/correlation/:key` and `/_audit-trail/correlations`. Registered only when + # `config.audit_trail[:database]` is set. All three routes are scoped to a single record through the + # `collection`/`recordId` query (GET) or body (POST) params and share the per-record auth. + class AuditTrailCorrelation < AbstractAuthenticatedRoute + include AuditTrailRoute + + def setup_routes + return self unless store + + add_route( + 'forest_audit_trail_correlation', + 'get', + '/_audit-trail/correlation/:correlation_key', + ->(args) { handle_history(args) } + ) + # GET carries the keys in `correlationKeys`; POST accepts a body list to dodge URL limits. + add_route( + 'forest_audit_trail_correlations', + 'get', + '/_audit-trail/correlations', + ->(args) { handle_batch(args) } + ) + add_route( + 'forest_audit_trail_correlations_batch', + 'post', + '/_audit-trail/correlations', + ->(args) { handle_batch(args) } + ) + + self + end + + def handle_history(args = {}) + collection, record_id = assert_scope(args) + + history = store.list_by_correlation( + collection: collection.name, + record_id: record_id, + correlation_key: args[:params]['correlation_key'] + ) + + { name: collection.name, content: { data: history.map { |record| serialize_record(record) } } } + end + + def handle_batch(args = {}) + collection, record_id = assert_scope(args) + correlation_keys = parse_correlation_keys(args) + + history = if correlation_keys.empty? + [] + else + store.list_by_correlations( + collection: collection.name, + record_id: record_id, + correlation_keys: correlation_keys + ) + end + + { name: collection.name, content: { data: history.map { |record| serialize_record(record) } } } + end + + private + + def assert_scope(args) + context = build(args) + name = args.dig(:params, 'collection').to_s + record_id = args.dig(:params, 'recordId').to_s + + raise Http::Exceptions::ValidationError, 'Missing collection' if name.empty? + raise Http::Exceptions::ValidationError, 'Missing recordId' if record_id.empty? + + collection = get_collection(context, name) + context.permissions.can?(:read, collection) + assert_record_in_scope(context, collection, record_id) + + [collection, record_id] + end + + def get_collection(context, name) + context.datasource.get_collection(name) + rescue ForestAdminDatasourceToolkit::Exceptions::ForestException => e + raise Http::Exceptions::NotFoundError, e.message if e.message.include?('not found') + + raise + end + + # Body array (POST) takes precedence, otherwise the comma-separated query param (GET). + def parse_correlation_keys(args) + raw = args.dig(:params, 'correlationKeys') + keys = raw.is_a?(Array) ? raw : raw.to_s.split(',') + + keys.map { |key| key.to_s.strip }.reject(&:empty?) + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb new file mode 100644 index 000000000..550e23fee --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb @@ -0,0 +1,115 @@ +module ForestAdminAgent + module Routes + module Resources + # Behaviour shared by every audit-trail route: they all take a packed record id straight from the + # request, so the caller's permission scope has to be checked against that record before any + # history is returned (`can?(:read, collection)` alone only proves access to the collection — a + # role restricted to a subset of the records would otherwise read the history of any of them), + # and they all serialize audit records the same way. + module AuditTrailRoute + include ForestAdminDatasourceToolkit::Components::Query + + def assert_record_in_scope(context, collection, packed_id) + scoped_record(context, collection, packed_id) + + nil + end + + # The record as it stands, read through the caller's permission scope. nil when it no longer exists + # — a deleted record keeps its history readable, which is much of the point of an audit trail — and + # a 404 when it does exist outside that scope. + # + # Authorizing and reading are the same query on purpose: a scoped check followed by an unscoped read + # would hand back a row the check never covered, the moment the two drifted apart. + def scoped_record(context, collection, packed_id, projection = nil) + condition = ConditionTree::ConditionTreeFactory.match_records( + collection, [Utils::Id.unpack_id(collection, packed_id, with_key: true)] + ) + scope = context.permissions.get_scope(collection) + in_scope = ConditionTree::ConditionTreeFactory.intersect([condition, scope]) + record = first_record(context, collection, in_scope, projection || key_projection(collection)) + + return record if record + # Nothing in scope: either gone for good, or someone else's record. Without a scope the query + # above already answered the question. + return nil if scope.nil? || first_record(context, collection, condition, key_projection(collection)).nil? + + raise Http::Exceptions::NotFoundError, 'Record does not exists' + end + + # Camelize only the top-level keys — the row `id` included, which the front uses as the tiebreaker when + # merging pages ordered by (timestamp, id). Value hashes keep the keys they were stored with: a record's + # own column names, or an action answer's camelCase Forest names. + def serialize_record(record) + # `previous_record_id` stays out: it is how the agent follows a record across a rename, not something + # the payload contract carries. + record.to_h.except(:previous_record_id).transform_keys { |key| key.to_s.camelize(:lower) } + end + + def store + ::ForestAdminAgent::AuditTrail.store + end + + # Every id this record has been filed under, each with the moment it stopped being that id. Rows + # written before an update moved a writable primary key stay under the id they were true of, so a + # history query that asked for the current id alone would start at the rename and call that the whole + # story — and one that asked for the bare ids would sweep up whatever record holds an abandoned id now. + # + # Breadth-first, and no depth limit: every hop adds an id not already seen and there are finitely many + # of those, so skipping what we hold is both the cycle guard and the terminator. A cap would have + # truncated a record renamed often enough, which reads exactly like missing history. + def record_segments(collection, packed_id) + segments = [{ id: packed_id, until: nil, until_row: nil }] + queue = segments.dup + + until queue.empty? + segment = queue.shift + + store.renamed_from(collection: collection.name, record_id: segment[:id]).each do |previous| + next if segments.any? { |seen| seen[:id] == previous[:id] } + + # Bounded by its own rename and by everything walked through to reach it: an id abandoned twice + # only belongs to this record up to the earlier of them. + found = earlier_bound(previous, segment).merge(id: previous[:id]) + segments << found + queue << found + end + end + + segments + end + + # Bounds compare as the trail orders, (timestamp, row id); nil is "no bound yet", which is later than + # any of them. Through `<=>`, since Array is not Comparable and `<=` on one raises. + def earlier_bound(one, other) + return other.slice(:until, :until_row) if one[:until].nil? + return one.slice(:until, :until_row) if other[:until].nil? + + pair = ->(bound) { [bound[:until], bound[:until_row].to_i] } + earlier = (pair[one] <=> pair[other]) <= 0 ? one : other + + earlier.slice(:until, :until_row) + end + + # What the audit trail actually records: primary keys, so a state can be identified, plus the writable + # columns. Reading read-only ones would hand them back at their present value inside an answer that + # claims to describe a past instant. + def audited_projection(collection) + writable = collection.schema[:fields].select do |_name, field| + field.type == 'Column' && !field.is_read_only + end.keys + + Projection.new((ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(collection) + writable).uniq) + end + + def key_projection(collection) + Projection.new(ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(collection)) + end + + def first_record(context, collection, condition_tree, projection) + collection.list(context.caller, Filter.new(condition_tree: condition_tree), projection).first + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/utils/caller_parser.rb b/packages/forest_admin_agent/lib/forest_admin_agent/utils/caller_parser.rb index 1fea51906..cf373e77e 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/utils/caller_parser.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/utils/caller_parser.rb @@ -18,6 +18,10 @@ def parse @token_data = decode_token @token_data[:timezone] = extract_timezone @token_data[:request] = { ip: @args[:headers]['action_dispatch.remote_ip'].to_s } + # One id per request, generated by the agent, shared by every operation it triggers (used to + # correlate the audit trail) and echoed back to the client in the response header. + # See ForestAdminAgent::Http::CorrelationId. + @token_data[:request_id] = Http::CorrelationId.current project, environment = extract_forest_context @token_data[:project] = project @token_data[:environment] = environment diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb new file mode 100644 index 000000000..54e1ab821 --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb @@ -0,0 +1,175 @@ +require 'spec_helper' + +module ForestAdminAgent + module AuditTrail + describe ActionCapture do + # Rows keyed by position, which stands in for the row id the store hands back. + let(:store) do + Class.new do + attr_reader :records + + def initialize + @records = [] + end + + def append_all(rows) + first = @records.size + @records.concat(rows) + + (first...@records.size).to_a + end + + def confirm(id, attributes) + attributes.each { |key, value| @records[id][key] = value } + @records[id][:status] = ForestAdminAgent::AuditTrail::Recording::DONE + end + end.new + end + + let(:caller_double) { double('caller', id: 42, first_name: 'Ada', last_name: 'L', email: 'ada@test') } + let(:capture) { described_class.new(store) } + + def invocation(**over) + { caller: caller_double, collection: 'orders', action_name: 'Refund', + form_values: { 'amount' => 30, 'reason' => 'damaged' }, record_ids: ['4'] }.merge(over) + end + + def run(capture: described_class.new(store), result: { type: 'Success', message: 'Refunded' }, + failed: false, **over) + ids = capture.pending(**invocation(**over)) + capture.confirm(ids, result: result, failed: failed) + + store.records + end + + describe '#pending' do + it 'records the run before it happens, with the form on the previous side' do + capture.pending(**invocation) + + row = store.records.last + expect(row.status).to eq(Recording::PENDING) + expect(row.operation).to eq(described_class::EXECUTED) + expect(row.action_name).to eq('Refund') + expect(row.collection).to eq('orders') + expect(row.record_id).to eq('4') + expect(row.previous_values).to eq({ 'amount' => 30, 'reason' => 'damaged' }) + expect(row.new_values).to eq({}) + end + + it 'denormalises who acted, so a later rename does not rewrite history' do + capture.pending(**invocation) + + row = store.records.last + expect(row.user_id).to eq(42) + expect(row.user_first_name).to eq('Ada') + expect(row.user_last_name).to eq('L') + expect(row.user_email).to eq('ada@test') + end + + it 'copes with a caller carrying no identity at all' do + capture.pending(**invocation(caller: double('caller'))) + + expect(store.records.last.user_email).to be_nil + expect(store.records.last.correlation_key).to be_nil + end + + it 'writes one row per targeted record, sharing timestamp and correlation key' do + ids = capture.pending(**invocation(record_ids: %w[4 7 9])) + + expect(ids.size).to eq(3) + expect(store.records.map(&:record_id)).to eq(%w[4 7 9]) + expect(store.records.map(&:timestamp).uniq.size).to eq(1) + end + + it 'records a run attached to no record when no target can be named' do + capture.pending(**invocation(record_ids: [])) + + expect(store.records.map(&:record_id)).to eq([described_class::NO_RECORD]) + end + + it 'masks redacted form fields, keeping the rest' do + described_class.new(store, { 'orders' => ['reason'] }).pending(**invocation) + + expect(store.records.last.previous_values).to eq({ 'amount' => 30, 'reason' => Recording::REDACTED }) + end + + it 'does nothing without a configured store' do + expect(described_class.new(nil).pending(**invocation)).to eq([]) + end + end + + describe '#confirm' do + it 'settles the row and keeps what the action answered' do + row = run.last + + expect(row.status).to eq(Recording::DONE) + expect(row.operation).to eq(described_class::EXECUTED) + expect(row.new_values).to eq({ 'type' => 'Success', 'message' => 'Refunded' }) + end + + it 'marks a failed run with its own operation' do + row = run(result: { type: 'Error', message: 'not allowed' }, failed: true).last + + expect(row.operation).to eq(described_class::FAILED) + expect(row.new_values).to eq({ 'type' => 'Error', 'message' => 'not allowed' }) + end + + it 'leaves the answer empty when the action raised' do + expect(run(result: nil, failed: true).last.new_values).to eq({}) + end + + # The row already says an action started; losing its answer beats reporting a failure for a run that + # went through. + it 'logs and swallows a store failure' do + logger = instance_spy(Services::LoggerService) + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) + ids = capture.pending(**invocation) + allow(store).to receive(:confirm).and_raise(StandardError, 'audit db is down') + + expect { capture.confirm(ids, result: {}) }.not_to raise_error + expect(logger).to have_received(:log).with('Error', /audit db is down/) + end + end + + describe 'the answer it keeps' do + # Forest's own keys, so camelCase on the wire — unlike a record's column names, which pass through. + it 'camelCases the keys it keeps' do + row = run(result: { type: 'File', name: 'refunds.csv', mime_type: 'text/csv' }).last + + expect(row.new_values).to eq({ 'type' => 'File', 'name' => 'refunds.csv', 'mimeType' => 'text/csv' }) + end + + it 'never keeps a file stream, a webhook body or headers, nor response headers' do + file = run(result: { type: 'File', name: 'r.csv', stream: 'id,amount' }).last + webhook = run(result: { type: 'Webhook', url: 'https://pay.test/refund', method: 'POST', + body: { 'secret' => 'x' }, headers: { 'Authorization' => 'Bearer t' }, + response_headers: { 'Set-Cookie' => 'session=x' } }).last + + expect(file.new_values).to eq({ 'type' => 'File', 'name' => 'r.csv' }) + expect(webhook.new_values).to eq({ 'type' => 'Webhook', 'url' => 'https://pay.test/refund', + 'method' => 'POST' }) + end + + # A signed one-time token or a password in the userinfo would otherwise sit in the one table nobody + # deletes from. + it 'strips credentials and query tokens off a webhook url' do + row = run(result: { type: 'Webhook', url: 'https://user:pass@pay.test/refund?token=abc#frag' }).last + + expect(row.new_values['url']).to eq('https://pay.test/refund') + end + + it 'strips a query token off a redirect path' do + row = run(result: { type: 'Redirect', path: '/orders/4?signature=abc' }).last + + expect(row.new_values['path']).to eq('/orders/4') + end + + it 'still sanitises a url the parser refuses' do + row = run(result: { type: 'Webhook', url: 'https://user:pass@pay test/refund?token=abc' }).last + + expect(row.new_values['url']).to eq('https://pay test/refund') + end + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb new file mode 100644 index 000000000..4210bf112 --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb @@ -0,0 +1,359 @@ +require 'spec_helper' + +module ForestAdminAgent + module AuditTrail + describe Capture do + let(:column_schema) { ForestAdminDatasourceToolkit::Schema::ColumnSchema } + # The hook decorator hands the same filter (or data) object to both contexts, which is what pairs a + # snapshot with its own after hook — so the helpers share one per operation, as production does. + let(:filter) { new_filter } + + # Rows keyed by position, which stands in for the row id the store hands back. + let(:store) do + Class.new do + attr_reader :records, :discarded + + def initialize + @records = [] + @discarded = [] + end + + def append_all(rows) + first = @records.size + @records.concat(rows) + + (first...@records.size).to_a + end + + def confirm(id, attributes) + attributes.each { |key, value| @records[id][key] = value } + @records[id][:status] = ForestAdminAgent::AuditTrail::Recording::DONE + end + + def discard(ids) + @discarded.concat(ids) + end + end.new + end + + let(:fields) do + { + 'id' => column_schema.new(column_type: 'Number', is_primary_key: true, is_read_only: true), + 'name' => column_schema.new(column_type: 'String'), + 'address' => column_schema.new(column_type: 'Json') + } + end + + let(:hooks) { {} } + let(:registrations) { [] } + let(:collection) { double('collection') } + let(:caller_double) do + double('caller', id: 42, first_name: 'Ada', last_name: 'L', email: 'ada@test', request_id: 'req-xyz') + end + + # What a hook context really hands out: the caller is already bound, so `list` takes + # (filter, projection). A verifying double keeps that contract honest. + let(:relaxed_collection) do + instance_double(ForestAdminDatasourceCustomizer::Context::RelaxedWrappers::RelaxedCollection) + end + + let(:collection_customizer) do + customizer = double('CollectionCustomizer', name: 'companies', collection: collection) + allow(customizer).to receive(:add_hook) do |position, type, prepend: false, &block| + registrations << [position, type, prepend] + hooks["#{position}_#{type}"] = block + end + customizer + end + + let(:datasource_customizer) do + double('DatasourceCustomizer', collections: { 'companies' => collection_customizer }) + end + + before do + Thread.current[:forest_audit_trail_snapshots] = nil + allow(collection).to receive(:schema).and_return({ fields: fields }) + described_class.new.run(datasource_customizer, nil, store: store) + end + + def new_filter + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: nil) + end + + def before_hook(type, patch: nil, data: nil, on: filter) + hooks["Before_#{type}"].call( + double('before', caller: caller_double, filter: on, collection: relaxed_collection, + patch: patch, data: data) + ) + end + + def after_hook(type, record: nil, data: nil, on: filter) + hooks["After_#{type}"].call( + double('after', caller: caller_double, filter: on, collection: relaxed_collection, + record: record, data: data) + ) + end + + # execute_after stops at the first exception, so being last would mean losing the record of a write that + # already happened whenever another customization raises in its own after hook. + it 'registers its after hooks ahead of the ones already there, and its before hooks after them' do + expect(registrations).to contain_exactly( + ['Before', 'Create', false], ['After', 'Create', true], + ['Before', 'Update', false], ['After', 'Update', true], + ['Before', 'Delete', false], ['After', 'Delete', true] + ) + end + + describe 'creating a record' do + it 'records the attempt before the write, with no id yet' do + before_hook('Create', data: { 'name' => 'Acme' }) + + row = store.records.last + expect(row.status).to eq(Recording::PENDING) + expect(row.operation).to eq('create') + expect(row.record_id).to be_nil + expect(row.new_values).to eq({ 'name' => 'Acme', 'address' => nil }) + end + + it 'confirms it with the id and the record that landed' do + data = { 'name' => 'Acme' } + before_hook('Create', data: data) + after_hook('Create', record: { 'id' => 1, 'name' => 'Acme', 'address' => { 'city' => 'Paris' } }, + data: data) + + row = store.records.last + expect(row.status).to eq(Recording::DONE) + expect(row.record_id).to eq('1') + expect(row.new_values).to eq({ 'name' => 'Acme', 'address' => { 'city' => 'Paris' } }) + end + + it 'denormalises who acted' do + before_hook('Create', data: { 'name' => 'Acme' }) + + expect(store.records.last.user_email).to eq('ada@test') + expect(store.records.last.correlation_key).to eq('req-xyz') + end + + # A write outside any request belongs to no request: inventing a key would make the row look like a + # single-row request of its own. + it 'leaves the correlation key empty when the caller carries none' do + allow(caller_double).to receive(:request_id).and_return(nil) + + before_hook('Create', data: { 'name' => 'Acme' }) + + expect(store.records.last.correlation_key).to be_nil + end + end + + describe 'updating records' do + def update(before:, persisted:, patch:) + allow(relaxed_collection).to receive(:list).and_return(before, persisted) + before_hook('Update', patch: patch) + after_hook('Update') + + store.records + end + + it 'records one pending row per matched record before the write' do + allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'Acme' }, + { 'id' => 2, 'name' => 'Other' }]) + before_hook('Update', patch: { 'name' => 'Z' }) + + expect(store.records.map(&:record_id)).to eq(%w[1 2]) + expect(store.records.map(&:status).uniq).to eq([Recording::PENDING]) + end + + # The diff is taken against the record as persisted, so normalisation and decorator side effects are + # what gets recorded — not merely what was asked for. + it 'diffs against the record as persisted, not the requested patch' do + rows = update(before: [{ 'id' => 1, 'name' => 'acme' }], + persisted: [{ 'id' => 1, 'name' => 'ACME NORMALISED' }], + patch: { 'name' => 'Acme' }) + + # No rename, so nothing to remember. + expect(rows.last.previous_record_id).to be_nil + expect(rows.last.status).to eq(Recording::DONE) + expect(rows.last.previous_values).to eq({ 'name' => 'acme' }) + expect(rows.last.new_values).to eq({ 'name' => 'ACME NORMALISED' }) + end + + # Otherwise the row would be filed under an id History never queries. + context 'when the primary key itself is writable' do + let(:fields) do + { + 'id' => column_schema.new(column_type: 'Number', is_primary_key: true), + 'name' => column_schema.new(column_type: 'String') + } + end + + it 'files the row under the id the record ended up with, remembering the one it left' do + rows = update(before: [{ 'id' => 1, 'name' => 'Acme' }], + persisted: [{ 'id' => 7, 'name' => 'Acme' }], + patch: { 'id' => 7 }) + + expect(rows.last.record_id).to eq('7') + expect(rows.last.new_values).to eq({ 'id' => 7 }) + # How a history query reaches the rows written while it was still 1. + expect(rows.last.previous_record_id).to eq('1') + end + end + + # Confirming from the patch would claim values that may never have been written; discarding would erase + # the evidence that something was attempted. + it 'leaves the row pending when the record cannot be read back' do + rows = update(before: [{ 'id' => 1, 'name' => 'Acme' }], persisted: [], patch: { 'name' => 'Z' }) + + expect(rows.last.status).to eq(Recording::PENDING) + expect(store.discarded).to be_empty + end + + # Nothing changed, so nothing is audited: the pending row goes rather than sitting there implying the + # write is unaccounted for. + it 'discards the pending row when the write changed nothing' do + update(before: [{ 'id' => 1, 'name' => 'Acme' }], + persisted: [{ 'id' => 1, 'name' => 'Acme' }], + patch: { 'name' => 'Acme' }) + + expect(store.discarded).to eq([0]) + end + end + + describe 'deleting records' do + it 'records the rows before the write and settles them after' do + allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 7, 'name' => 'Gone', + 'address' => nil }]) + + before_hook('Delete') + expect(store.records.last.status).to eq(Recording::PENDING) + + after_hook('Delete') + row = store.records.last + expect(row.status).to eq(Recording::DONE) + expect(row.operation).to eq('delete') + expect(row.previous_values).to eq({ 'name' => 'Gone', 'address' => nil }) + end + end + + describe 'when the audit database is unreachable' do + before { allow(store).to receive(:append_all).and_raise(StandardError, 'audit db is down') } + + # Knowing what an operation is about to touch is part of being able to record it. + it 'refuses the operation when the snapshot cannot even be read and critical is on' do + allow(ForestAdminAgent::AuditTrail).to receive(:critical?).and_return(true) + allow(relaxed_collection).to receive(:list).and_raise(StandardError, 'datasource down') + + expect { before_hook('Delete') }.to raise_error(StandardError, 'datasource down') + end + + it 'lets the write through, logging the failure, when critical is off' do + logger = instance_spy(Services::LoggerService) + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) + + expect { before_hook('Create', data: { 'name' => 'Acme' }) }.not_to raise_error + expect(logger).to have_received(:log).with('Error', /audit db is down/) + end + + # Nothing has been written yet, so refusing costs nothing to repair. + it 'refuses the operation when critical is on' do + allow(ForestAdminAgent::AuditTrail).to receive(:critical?).and_return(true) + + expect { before_hook('Create', data: { 'name' => 'Acme' }) } + .to raise_error(StandardError, 'audit db is down') + end + end + + describe 'a selection wider than the cap' do + let(:cap) { ForestAdminAgent::AuditTrail::MAX_RECORDS_PER_OPERATION } + + it 'audits up to the cap and says how many it left out' do + logger = instance_spy(Services::LoggerService) + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) + matched = (1..(cap + 1)).map { |id| { 'id' => id, 'name' => "n#{id}" } } + allow(relaxed_collection).to receive_messages(list: matched, aggregate: [{ 'value' => cap + 25 }]) + + before_hook('Delete') + + expect(store.records.size).to eq(cap) + expect(logger).to have_received(:log).with('Warn', /#{cap} records audited, 25 skipped/) + end + + # Auditing 500 of them while the write touches every match breaks the one invariant critical exists + # for, so the operation is refused instead — before the write, with nothing to repair. + it 'refuses the operation when critical is on' do + allow(ForestAdminAgent::AuditTrail).to receive(:critical?).and_return(true) + allow(relaxed_collection).to receive(:list).and_return((1..(cap + 1)).map { |id| { 'id' => id } }) + + expect { before_hook('Delete') }.to raise_error( + ForestAdminDatasourceToolkit::Exceptions::ForestException, /cannot record an operation touching more/ + ) + end + + it 'asks for one more than the cap, so it can tell it was truncated' do + allow(relaxed_collection).to receive(:list).and_return([]) + + before_hook('Delete') + + expect(relaxed_collection).to have_received(:list) do |filter, _projection| + expect(filter.page.limit).to eq(cap + 1) + end + end + end + + it 'masks redacted fields while still recording the change' do + described_class.new.run(datasource_customizer, nil, store: store, redact: { 'companies' => ['name'] }) + before_hook('Create', data: { 'name' => 'Secret' }) + + expect(store.records.last.new_values['name']).to eq(Recording::REDACTED) + end + + # A write nested inside another, rescued after it failed, leaves its snapshot behind. Taking the newest + # entry would confirm that failed operation's rows as done and strand the outer operation's own — both + # of them lies. + it 'settles its own operation, not a failed inner one left on the stack' do + outer = new_filter + inner = new_filter + allow(relaxed_collection).to receive(:list).and_return( + [{ 'id' => 1, 'name' => 'outer' }], + [{ 'id' => 2, 'name' => 'inner' }], + [{ 'id' => 1, 'name' => 'outer written' }] + ) + + before_hook('Update', patch: { 'name' => 'outer written' }, on: outer) + before_hook('Update', patch: { 'name' => 'never written' }, on: inner) + after_hook('Update', on: outer) + + settled = store.records.select { |row| row.status == Recording::DONE } + expect(settled.map(&:record_id)).to eq(['1']) + expect(settled.last.new_values).to eq({ 'name' => 'outer written' }) + # The inner write may or may not have landed, which is what pending says. + expect(store.records.map(&:record_id).zip(store.records.map(&:status))).to include(['2', Recording::PENDING]) + end + + # A customization replacing the filter leaves nothing to match on: our before hook saw the replacement, + # the after context carries the original. One operation in flight is unambiguous, so it still pairs. + it 'still settles a single operation whose filter was replaced' do + allow(relaxed_collection).to receive(:list).and_return( + [{ 'id' => 1, 'name' => 'Acme' }], [{ 'id' => 1, 'name' => 'Z' }] + ) + + before_hook('Update', patch: { 'name' => 'Z' }, on: new_filter) + after_hook('Update', on: new_filter) + + expect(store.records.last.status).to eq(Recording::DONE) + end + + # Replaced *and* nested: nothing identifies which entry is ours, so neither is confirmed rather than the + # wrong one being marked done. + it 'leaves both pending when it cannot tell which operation is which' do + allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'a' }], + [{ 'id' => 2, 'name' => 'b' }]) + + before_hook('Update', patch: { 'name' => 'x' }, on: new_filter) + before_hook('Update', patch: { 'name' => 'y' }, on: new_filter) + after_hook('Update', on: new_filter) + + expect(store.records.map(&:status).uniq).to eq([Recording::PENDING]) + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb new file mode 100644 index 000000000..caad7989d --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb @@ -0,0 +1,164 @@ +require 'spec_helper' + +module ForestAdminAgent + module AuditTrail + describe Diff do + describe '.diff' do + it 'returns nil when values are deeply equal regardless of hash key order' do + expect(described_class.diff({ 'a' => 1, 'b' => 2 }, { 'b' => 2, 'a' => 1 })).to be_nil + end + + it 'keeps only the changed leaf of a nested object' do + before = { 'city' => 'Paris', 'zip' => '75001' } + after = { 'city' => 'Lyon', 'zip' => '75001' } + + expect(described_class.diff(before, after)).to eq( + previous: { 'city' => 'Paris' }, + next: { 'city' => 'Lyon' } + ) + end + + # A key holding nil and a missing key have to stay tellable apart, so the side where the key does + # not exist simply does not carry it. Nothing sentinel-shaped reaches the database. + it 'reports a key that disappeared by leaving it out of the new values' do + expect(described_class.diff({ 'flag' => nil }, {})).to eq( + previous: { 'flag' => nil }, + next: {} + ) + end + + it 'reports a key that appeared by leaving it out of the previous values' do + expect(described_class.diff({}, { 'flag' => nil })).to eq( + previous: {}, + next: { 'flag' => nil } + ) + end + + it 'leaves an appended index out of the previous side' do + expect(described_class.diff([{ 'x' => 1 }], [{ 'x' => 1 }, { 'y' => 2 }])).to eq( + previous: {}, + next: { 1 => { 'y' => 2 } } + ) + end + + it 'leaves a dropped index out of the new side' do + expect(described_class.diff([{ 'x' => 1 }, { 'y' => 2 }], [{ 'x' => 1 }])).to eq( + previous: { 1 => { 'y' => 2 } }, + next: {} + ) + end + + it 'diffs an array of objects index by index' do + before = [{ 'name' => 'a' }, { 'name' => 'b' }] + after = [{ 'name' => 'a' }, { 'name' => 'c' }] + + expect(described_class.diff(before, after)).to eq( + previous: { 1 => { 'name' => 'b' } }, + next: { 1 => { 'name' => 'c' } } + ) + end + + it 'keeps scalars and primitive arrays whole' do + expect(described_class.diff(%w[a b], %w[a c])).to eq(previous: %w[a b], next: %w[a c]) + end + + it 'reports nil for a newly set or cleared value' do + expect(described_class.diff(nil, 'x')).to eq(previous: nil, next: 'x') + expect(described_class.diff('x', nil)).to eq(previous: 'x', next: nil) + end + end + + describe '.revert' do + it 'restores a scalar' do + expect(described_class.revert('Lyon', 'Paris', 'Lyon')).to eq('Paris') + end + + it 'restores only the touched keys, leaving the rest of the object alone' do + current = { 'city' => 'Lyon', 'zip' => '69001', 'country' => 'FR' } + + expect(described_class.revert(current, { 'city' => 'Paris' }, { 'city' => 'Lyon' })).to eq( + { 'city' => 'Paris', 'zip' => '69001', 'country' => 'FR' } + ) + end + + it 'removes a key the change had added' do + expect(described_class.revert({ 'city' => 'Lyon', 'zip' => '69001' }, {}, { 'zip' => '69001' })).to eq( + { 'city' => 'Lyon' } + ) + end + + it 'puts back a key the change had removed, nil value included' do + expect(described_class.revert({ 'city' => 'Lyon' }, { 'zip' => nil }, {})).to eq( + { 'city' => 'Lyon', 'zip' => nil } + ) + end + + it 'walks nested objects' do + current = { 'address' => { 'city' => 'Lyon', 'zip' => '69001' }, 'name' => 'Acme' } + previous = { 'address' => { 'city' => 'Paris' } } + changed = { 'address' => { 'city' => 'Lyon' } } + + expect(described_class.revert(current, previous, changed)).to eq( + { 'address' => { 'city' => 'Paris', 'zip' => '69001' }, 'name' => 'Acme' } + ) + end + + it 'restores an element of an array of objects, JSON string indexes included' do + current = [{ 'name' => 'a' }, { 'name' => 'c' }] + + expect(described_class.revert(current, { '1' => { 'name' => 'b' } }, { '1' => { 'name' => 'c' } })).to eq( + [{ 'name' => 'a' }, { 'name' => 'b' }] + ) + end + + it 'drops an element the change had appended, rather than leaving a nil behind' do + current = [{ 'name' => 'a' }, { 'name' => 'b' }] + + expect(described_class.revert(current, {}, { '1' => { 'name' => 'b' } })).to eq([{ 'name' => 'a' }]) + end + + it 'brings back an element the change had dropped' do + expect(described_class.revert([{ 'name' => 'a' }], { '1' => { 'name' => 'b' } }, {})).to eq( + [{ 'name' => 'a' }, { 'name' => 'b' }] + ) + end + + it 'replaces the value as a whole when the change was not structural' do + expect(described_class.revert({ 'city' => 'Lyon' }, nil, { 'city' => 'Lyon' })).to be_nil + end + + # A round trip is the property that matters: diff then revert gives the original back. The array + # cases carry a length change on purpose — an appended element has to disappear again rather than + # revert to nil, and a dropped one has to come back. + it 'undoes any diff it is given' do + [ + [{ 'a' => 1, 'b' => { 'c' => 2, 'd' => nil } }, { 'a' => 1, 'b' => { 'c' => 3 } }], + [{ 'a' => nil }, { 'a' => 'set' }], + [{ 'list' => [{ 'x' => 1 }, { 'x' => 2 }] }, { 'list' => [{ 'x' => 1 }, { 'x' => 9 }] }], + [[{ 'x' => 1 }], [{ 'x' => 1 }, { 'y' => 2 }]], + [[{ 'x' => 1 }, { 'y' => 2 }], [{ 'x' => 1 }]], + [[{ 'x' => 1 }, { 'y' => 2 }, { 'z' => 3 }], [{ 'x' => 9 }]], + [{ 'list' => [{ 'x' => 1 }] }, { 'list' => [{ 'x' => 1 }, { 'y' => 2 }] }], + ['plain', 'changed'] + ].each do |before, after| + delta = described_class.diff(before, after) + + expect(described_class.revert(after, delta[:previous], delta[:next])).to eq(before) + end + end + end + + describe '.changed_values' do + it 'only records writable columns present in the patch that actually changed' do + before = { 'status' => 'open', 'name' => 'Acme', 'ignored' => 1 } + patch = { 'status' => 'closed', 'name' => 'Acme' } + + expect(described_class.changed_values(before, patch, %w[status name])).to eq( + previous_values: { 'status' => 'open' }, + new_values: { 'status' => 'closed' } + ) + end + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/record_state_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/record_state_spec.rb new file mode 100644 index 000000000..d5b93aa81 --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/record_state_spec.rb @@ -0,0 +1,106 @@ +require 'spec_helper' + +module ForestAdminAgent + module AuditTrail + describe RecordState do + def entry(operation, previous_values = {}, new_values = {}) + AuditRecord.new(operation: operation, collection: 'orders', record_id: '1', + previous_values: previous_values, new_values: new_values) + end + + it 'gives the record back untouched when nothing happened after the instant' do + current = { 'id' => 1, 'status' => 'shipped' } + + expect(described_class.at(current, [])).to eq(current) + end + + it 'undoes updates from newest to oldest' do + current = { 'id' => 1, 'status' => 'shipped' } + entries = [ + entry('update', { 'status' => 'paid' }, { 'status' => 'shipped' }), + entry('update', { 'status' => 'draft' }, { 'status' => 'paid' }) + ] + + expect(described_class.at(current, entries)).to eq({ 'id' => 1, 'status' => 'draft' }) + end + + it 'leaves columns the entries never touched alone' do + current = { 'id' => 1, 'status' => 'shipped', 'note' => 'keep me' } + entries = [entry('update', { 'status' => 'paid' }, { 'status' => 'shipped' })] + + expect(described_class.at(current, entries)['note']).to eq('keep me') + end + + it 'reports no record at all when it was created after the instant' do + current = { 'id' => 1, 'status' => 'draft' } + entries = [entry('create', {}, { 'status' => 'draft' })] + + expect(described_class.at(current, entries)).to be_nil + end + + # The record is gone now, so the walk starts from nothing and the delete brings the whole row back. + it 'restores a deleted record from the snapshot the delete recorded' do + entries = [entry('delete', { 'status' => 'shipped', 'note' => 'bye' }, {})] + + expect(described_class.at(nil, entries)).to eq({ 'status' => 'shipped', 'note' => 'bye' }) + end + + it 'keeps undoing older entries after restoring a delete' do + entries = [ + entry('delete', { 'status' => 'shipped' }, {}), + entry('update', { 'status' => 'paid' }, { 'status' => 'shipped' }) + ] + + expect(described_class.at(nil, entries)).to eq({ 'status' => 'paid' }) + end + + # A create can sit on top of an older life of the same id: nil for the create, then the older + # delete brings that life back. + it 'walks past a re-created id into its previous life' do + entries = [ + entry('create', {}, { 'status' => 'draft' }), + entry('delete', { 'status' => 'archived' }, {}) + ] + + expect(described_class.at({ 'status' => 'draft' }, entries)).to eq({ 'status' => 'archived' }) + end + + # An action row's two value columns hold the submitted form and the action's answer, not a record's + # before and after: applying either would corrupt the rebuild. + it 'ignores smart-action rows, whichever way they went' do + current = { 'status' => 'shipped' } + entries = [ + entry('action', { 'status' => 'submitted value' }, { 'type' => 'Success' }), + entry('action_failed', { 'status' => 'submitted value' }, {}) + ] + + expect(described_class.at(current, entries)).to eq(current) + end + + it 'undoes a nested change without disturbing the rest of the object' do + current = { 'address' => { 'city' => 'Lyon', 'zip' => '69001' } } + entries = [entry('update', { 'address' => { 'city' => 'Paris' } }, { 'address' => { 'city' => 'Lyon' } })] + + expect(described_class.at(current, entries)).to eq( + { 'address' => { 'city' => 'Paris', 'zip' => '69001' } } + ) + end + + # The whole point of the absent-vs-nil encoding: a key the change added has to disappear again, + # rather than come back holding nil. + it 'removes a nested key that the change had introduced' do + current = { 'address' => { 'city' => 'Lyon', 'zip' => '69001' } } + entries = [entry('update', { 'address' => {} }, { 'address' => { 'zip' => '69001' } })] + + expect(described_class.at(current, entries)).to eq({ 'address' => { 'city' => 'Lyon' } }) + end + + it 'puts back a nested key holding nil that the change had removed' do + current = { 'address' => { 'city' => 'Lyon' } } + entries = [entry('update', { 'address' => { 'zip' => nil } }, { 'address' => {} })] + + expect(described_class.at(current, entries)).to eq({ 'address' => { 'city' => 'Lyon', 'zip' => nil } }) + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/field_filter_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/field_filter_spec.rb new file mode 100644 index 000000000..07f92b6d6 --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/field_filter_spec.rb @@ -0,0 +1,79 @@ +require 'spec_helper' + +module ForestAdminAgent + module AuditTrail + module Sql + # The store specs run on SQLite, so the SQL for the other adapters is pinned here — a fake connection + # is enough, the class only builds a condition string. + describe FieldFilter do + def filter_for(adapter) + # rubocop:disable RSpec/VerifiedDoubles + connection = double(adapter.to_s, adapter_name: adapter) + # rubocop:enable RSpec/VerifiedDoubles + allow(connection).to receive(:quote) { |value| "'#{value}'" } + + described_class.new(connection) + end + + it 'asks Postgres for the JSON keys of both sides' do + condition = filter_for('PostgreSQL').condition(%w[status note]) + + expect(condition).to eq( + 'EXISTS (SELECT 1 FROM jsonb_object_keys(previous_values::jsonb) AS key ' \ + "WHERE key IN ('status', 'note')) OR " \ + "EXISTS (SELECT 1 FROM jsonb_object_keys(new_values::jsonb) AS key WHERE key IN ('status', 'note'))" + ) + end + + # `?|` would read as a bind placeholder, and `json_extract` cannot tell a key holding null from a + # missing one. + it 'never emits a bind placeholder or json_extract' do + %w[PostgreSQL SQLite Mysql2].each do |adapter| + condition = filter_for(adapter).condition(%w[status]) + + expect(condition).not_to include('?') + expect(condition).not_to include('json_extract') + end + end + + it 'asks SQLite for the type at a quoted path on both sides' do + condition = filter_for('SQLite').condition(%w[status]) + + expect(condition).to eq( + %(json_type(previous_values, '$."status"') IS NOT NULL OR ) + + %(json_type(new_values, '$."status"') IS NOT NULL) + ) + end + + it 'asks MySQL for any of the paths on both sides' do + condition = filter_for('Mysql2').condition(%w[status note]) + + expect(condition).to eq( + %(JSON_CONTAINS_PATH(previous_values, 'one', '$."status"', '$."note"') OR ) + + %(JSON_CONTAINS_PATH(new_values, 'one', '$."status"', '$."note"')) + ) + end + + it 'treats MariaDB like MySQL' do + expect(filter_for('Mariadb').condition(%w[status])).to include('JSON_CONTAINS_PATH') + end + + # A dot in a field name is part of the name, not a traversal into the object. + it 'quotes a field name holding a dot as a single key' do + expect(filter_for('SQLite').condition(['address.city'])).to include(%('$."address.city"')) + expect(filter_for('Mysql2').condition(['address.city'])).to include(%('$."address.city"')) + end + + it 'escapes a quote inside a field name' do + expect(filter_for('SQLite').condition(['we"ird'])).to include(%q($."we\"ird")) + end + + it 'refuses an adapter it has no JSON test for' do + expect { filter_for('Informix').condition(%w[status]) }.to raise_error( + ForestAdminDatasourceToolkit::Exceptions::ForestException, /not supported on informix/ + ) + end + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb new file mode 100644 index 000000000..76e82c31a --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb @@ -0,0 +1,130 @@ +require 'spec_helper' + +module ForestAdminAgent + module AuditTrail + module Sql + # The store specs cover the SQLite path end to end; these pin the Postgres-only behaviour (schema + # creation and the advisory lock) without needing a Postgres server. + describe Migrator do + subject(:migrator) { described_class.new(connection, schema: 'forest', table_name: 'audit_logs') } + + let(:executed) { [] } + # A plain double on purpose: this fakes the Postgres adapter, which cannot be loaded here (no `pg` + # gem), and `quote_schema_name` only exists on it. + let(:connection) do + double( + 'PostgreSQL connection', + adapter_name: 'PostgreSQL', + create_table: nil, + add_index: nil, + add_column: nil, + remove_index: nil, + change_column: nil, + select_values: ['001-create-audit-logs'], + quote_schema_name: '"forest"', + quote_table_name: '"forest.audit_logs_migration"', + quote: "'x'" + ) + end + + before do + allow(connection).to receive(:execute) { |sql| executed << sql } + allow(connection).to receive(:transaction).and_yield + end + + it 'creates the schema before taking the lock, so the migrations can see it' do + migrator.run + + expect(executed.first).to eq('CREATE SCHEMA IF NOT EXISTS "forest"') + end + + it 'runs the migrations under a transaction-scoped advisory lock' do + migrator.run + + expect(connection).to have_received(:transaction) + expect(executed).to include('SELECT pg_advisory_xact_lock(17999, 21076)') + end + + def raise_on_create_schema(message) + allow(connection).to receive(:execute) do |sql| + raise ActiveRecord::StatementInvalid, message if sql.include?('CREATE SCHEMA') + + executed << sql + end + end + + it 'tolerates another instance having created the schema concurrently' do + raise_on_create_schema('ERROR: schema "forest" already exists') + + expect { migrator.run }.not_to raise_error + expect(executed).to include('SELECT pg_advisory_xact_lock(17999, 21076)') + end + + it 'still reports a schema creation that failed for another reason' do + raise_on_create_schema('ERROR: permission denied for database') + + expect { migrator.run }.to raise_error(ActiveRecord::StatementInvalid, /permission denied/) + end + + it 'treats the unique violation on pg_namespace as a lost race' do + allow(connection).to receive(:execute) do |sql| + raise ActiveRecord::RecordNotUnique, 'duplicate key value' if sql.include?('CREATE SCHEMA') + + executed << sql + end + + expect { migrator.run }.not_to raise_error + end + + # A message match alone would read "role ... already exists" as a lost race; the SQLSTATE does not. + context 'when the adapter exposes a SQLSTATE' do + def raise_with_sql_state(state, message) + stub_const('PG::Result', Class.new { const_set(:PG_DIAG_SQLSTATE, 67) }) unless defined?(PG::Result) + cause = double('PG::Error', result: double('result', error_field: state)) + error = ActiveRecord::StatementInvalid.new(message) + allow(error).to receive(:cause).and_return(cause) + + allow(connection).to receive(:execute) do |sql| + raise error if sql.include?('CREATE SCHEMA') + + executed << sql + end + end + + it 'accepts duplicate_schema' do + raise_with_sql_state('42P06', 'ERROR: schema "forest" already exists') + + expect { migrator.run }.not_to raise_error + end + + it 'falls back to the message when reading the SQLSTATE itself blows up' do + stub_const('PG::Result', Class.new { const_set(:PG_DIAG_SQLSTATE, 67) }) unless defined?(PG::Result) + cause = double('PG::Error') + allow(cause).to receive(:result).and_raise(StandardError, 'connection already closed') + error = ActiveRecord::StatementInvalid.new('ERROR: schema "forest" already exists') + allow(error).to receive(:cause).and_return(cause) + allow(connection).to receive(:execute) do |sql| + raise error if sql.include?('CREATE SCHEMA') + + executed << sql + end + + expect { migrator.run }.not_to raise_error + end + + it 'reports anything else, however its message reads' do + raise_with_sql_state('42501', 'ERROR: permission denied, object already exists elsewhere') + + expect { migrator.run }.to raise_error(ActiveRecord::StatementInvalid) + end + end + + it 'skips migrations already applied to this table' do + migrator.run + + expect(connection).not_to have_received(:create_table).with('forest.audit_logs', any_args) + end + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/text_search_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/text_search_spec.rb new file mode 100644 index 000000000..9eb01564d --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/text_search_spec.rb @@ -0,0 +1,82 @@ +require 'spec_helper' + +module ForestAdminAgent + module AuditTrail + module Sql + # The store specs exercise this against SQLite; these pin the SQL for the adapters no local database + # covers, and the two properties that matter whatever the dialect. + describe TextSearch do + let(:adapters) { %w[PostgreSQL SQLite Mysql2] } + let(:searched_columns) { %w[action_name user_first_name user_last_name user_email] } + + def filter_for(adapter) + # rubocop:disable RSpec/VerifiedDoubles + connection = double(adapter.to_s, adapter_name: adapter) + # rubocop:enable RSpec/VerifiedDoubles + allow(connection).to receive(:quote) { |value| "'#{value}'" } + + described_class.new(connection) + end + + it 'casts the JSON columns to text on Postgres' do + condition = filter_for('PostgreSQL').condition('lyon') + + expect(condition).to include("LOWER(REPLACE(previous_values::text, '[redacted]', '')) LIKE '%lyon%'") + expect(condition).to include("LOWER(REPLACE(new_values::text, '[redacted]', '')) LIKE '%lyon%'") + end + + it 'reads the JSON columns as they are stored on SQLite' do + expect(filter_for('SQLite').condition('lyon')).to include( + "LOWER(REPLACE(previous_values, '[redacted]', '')) LIKE '%lyon%'" + ) + end + + it 'casts the JSON columns to CHAR on MySQL' do + expect(filter_for('Mysql2').condition('lyon')).to include( + "LOWER(REPLACE(CAST(previous_values AS CHAR), '[redacted]', '')) LIKE '%lyon%'" + ) + end + + it 'searches the action name and the actor, on every adapter' do + adapters.each do |adapter| + condition = filter_for(adapter).condition('ada') + + searched_columns.each do |column| + expect(condition).to include("LOWER(#{column}) LIKE '%ada%'") + end + end + end + + # Machine identifiers nobody searches for. + it 'never searches an identifier column' do + condition = filter_for('PostgreSQL').condition('x') + + %w[operation correlation_key record_id collection status timestamp].each do |column| + expect(condition).not_to include(column) + end + end + + # `!` and not a backslash: MySQL treats a backslash as an escape inside string literals too. + it 'escapes LIKE wildcards in the term rather than letting them match anything' do + condition = filter_for('SQLite').condition('50%_off') + + expect(condition).to include("LIKE '%50!%!_off%' ESCAPE '!'") + end + + it 'escapes the escape character itself' do + expect(filter_for('SQLite').condition('a!b')).to include("LIKE '%a!!b%'") + end + + it 'lowercases the term, since the comparison lowercases the column' do + expect(filter_for('SQLite').condition('LyOn')).to include("LIKE '%lyon%'") + end + + it 'refuses an adapter it has no text cast for' do + expect { filter_for('Informix').condition('x') }.to raise_error( + ForestAdminDatasourceToolkit::Exceptions::ForestException, /not supported on informix/ + ) + end + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb new file mode 100644 index 000000000..7f9002a6d --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb @@ -0,0 +1,519 @@ +require 'spec_helper' +require 'tempfile' + +module ForestAdminAgent + module AuditTrail + describe Store do + let(:db) { Tempfile.new(['audit', '.sqlite3']) } + let(:store) { described_class.new(database: { adapter: 'sqlite3', database: db.path }) } + + after do + Sql::AuditConnectionBase.disconnect! + db.close! + end + + def record(over = {}) + AuditRecord.new( + operation: 'update', collection: 'accounts', record_id: '1', status: Recording::DONE, + previous_values: { 'status' => 'open' }, new_values: { 'status' => 'closed' }, + timestamp: '2026-01-02T03:04:05.000Z', user_id: 42, correlation_key: 'req-1', **over + ) + end + + def matching_fields(fields) + store.list_by_record(collection: 'accounts', record_id: '1', fields: fields).map(&:new_values) + end + + def connection + Sql::AuditConnectionBase.connection + end + + it 'creates the audit table with the expected columns on first write' do + store.append(record) + + expect(store.send(:model).column_names.sort).to eq( + %w[action_name collection correlation_key id new_values operation previous_record_id previous_values + record_id status timestamp user_email user_first_name user_id user_last_name] + ) + end + + # An action row uses the two sides for what went in and what came back. + it 'persists a smart-action row, submitted form and answer included' do + store.append(record(operation: 'action', previous_values: { 'amount' => 30 }, + new_values: { 'type' => 'Success', 'message' => 'Refunded' })) + + audit = store.list_by_record(collection: 'accounts', record_id: '1').first + expect(audit.operation).to eq('action') + expect(audit.previous_values).to eq({ 'amount' => 30 }) + expect(audit.new_values).to eq({ 'type' => 'Success', 'message' => 'Refunded' }) + end + + it 'persists and reads back a record, decoding the JSON columns' do + store.append(record) + + audit = store.list_by_record(collection: 'accounts', record_id: '1').first + expect(audit.operation).to eq('update') + expect(audit.user_id).to eq(42) + expect(audit.previous_values).to eq({ 'status' => 'open' }) + expect(audit.new_values).to eq({ 'status' => 'closed' }) + end + + it 'returns a record history oldest-first, scoped to the record, honoring skip/limit' do + store.append(record(timestamp: '2026-01-02T03:04:06.000Z', correlation_key: 'b')) + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'a')) + store.append(record(record_id: '2', correlation_key: 'other')) + + history = store.list_by_record(collection: 'accounts', record_id: '1') + expect(history.map(&:correlation_key)).to eq(%w[a b]) + + page = store.list_by_record(collection: 'accounts', record_id: '1', skip: 1, limit: 1) + expect(page.map(&:correlation_key)).to eq(['b']) + end + + it 'sorts newest first when order is desc, breaking ties by insertion order' do + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'a')) + store.append(record(timestamp: '2026-01-02T03:04:06.000Z', correlation_key: 'b')) + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'a2')) + + history = store.list_by_record(collection: 'accounts', record_id: '1', order: 'desc') + expect(history.map(&:correlation_key)).to eq(%w[b a a2]) + end + + it 'filters by user_ids and inclusive timestamp range' do + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', user_id: 7, correlation_key: 'keep')) + store.append(record(timestamp: '2026-01-02T03:04:09.000Z', user_id: 7, correlation_key: 'late')) + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', user_id: 9, correlation_key: 'other')) + + history = store.list_by_record( + collection: 'accounts', record_id: '1', user_ids: [7], + start_timestamp: '2026-01-02T03:04:04.000Z', end_timestamp: '2026-01-02T03:04:06.000Z' + ) + expect(history.map(&:correlation_key)).to eq(['keep']) + end + + it 'counts matches independently of skip/limit, respecting filters' do + store.append(record(user_id: 7)) + store.append(record(user_id: 7)) + store.append(record(user_id: 9)) + + expect(store.count_by_record(collection: 'accounts', record_id: '1')).to eq(3) + expect(store.count_by_record(collection: 'accounts', record_id: '1', user_ids: [7])).to eq(2) + end + + it 'lists entries under a correlation key for the record, scoped and oldest first' do + store.append(record(record_id: '1', correlation_key: 'req-1', timestamp: '2026-01-01T00:00:02.000Z')) + store.append(record(record_id: '1', correlation_key: 'req-1', timestamp: '2026-01-01T00:00:01.000Z')) + store.append(record(record_id: '1', correlation_key: 'req-2')) + store.append(record(record_id: '2', correlation_key: 'req-1')) + + history = store.list_by_correlation(collection: 'accounts', record_id: '1', correlation_key: 'req-1') + expect(history.map(&:timestamp)).to eq(['2026-01-01T00:00:01.000Z', '2026-01-01T00:00:02.000Z']) + end + + it 'lists a flat history across multiple correlation keys, oldest first' do + store.append(record(correlation_key: 'a', timestamp: '2026-01-03T00:00:00.000Z')) + store.append(record(correlation_key: 'b', timestamp: '2026-01-01T00:00:00.000Z')) + store.append(record(correlation_key: 'a', timestamp: '2026-01-02T00:00:00.000Z')) + store.append(record(correlation_key: 'c', timestamp: '2026-01-04T00:00:00.000Z')) + + history = store.list_by_correlations(collection: 'accounts', record_id: '1', correlation_keys: %w[a b]) + expect(history.map(&:timestamp)).to eq( + ['2026-01-01T00:00:00.000Z', '2026-01-02T00:00:00.000Z', '2026-01-03T00:00:00.000Z'] + ) + end + + it 'returns an empty array for an empty correlation key list' do + store.append(record(correlation_key: 'a')) + + expect(store.list_by_correlations(collection: 'accounts', record_id: '1', correlation_keys: [])).to eq([]) + end + + it 'tracks applied migrations beside the table they built, and is idempotent across stores' do + store.append(record) + described_class.new(database: { adapter: 'sqlite3', database: db.path }).append(record) + + names = connection.select_values('SELECT name FROM audit_logs_migration ORDER BY name') + expect(names).to eq(['001-create-audit-logs']) + end + + # One tracker per audited table, so a second store does not read the first one's history as its own. + it 'migrates a second table in the same database, tracking it separately' do + store.append(record) + other = described_class.new(database: { adapter: 'sqlite3', database: db.path }, table_name: 'other_logs') + + expect { other.append(record) }.not_to raise_error + expect(other.list_by_record(collection: 'accounts', record_id: '1').size).to eq(1) + expect(connection.indexes('other_logs').map(&:name)).to include('other_logs_record_id') + expect(connection.select_values('SELECT name FROM other_logs_migration')).to eq(['001-create-audit-logs']) + end + + # Every write sets it, so a row without one is a bug rather than a row quietly claiming to be done. + it 'refuses a row with no status' do + expect { store.append(record(status: nil)) }.to raise_error(ActiveRecord::NotNullViolation) + end + + it 'binds each store to its own model class instead of mutating a shared one' do + other = described_class.new(database: { adapter: 'sqlite3', database: db.path }) + store.append(record(correlation_key: 'main')) + other.append(record(correlation_key: 'other')) + + # No shared mutable model: AuditLog is an abstract template, each store owns a distinct subclass. + expect(Sql::AuditLog.abstract_class?).to be(true) + expect(store.send(:model)).not_to equal(other.send(:model)) + expect(store.send(:model).table_name).to eq('audit_logs') + expect(store.list_by_record(collection: 'accounts', record_id: '1').map(&:correlation_key)) + .to eq(%w[main other]) + end + + it 'lists only entries whose diff touched one of the given fields' do + store.append(record(previous_values: { 'status' => 'open' }, new_values: { 'status' => 'closed' })) + store.append(record(previous_values: { 'note' => 'a' }, new_values: { 'note' => 'b' })) + # Added key: it exists on the new side only, so both sides have to be searched. + store.append(record(previous_values: {}, new_values: { 'tags' => ['x'] })) + + expect(matching_fields(%w[status])).to eq([{ 'status' => 'closed' }]) + expect(matching_fields(%w[tags])).to eq([{ 'tags' => ['x'] }]) + expect(matching_fields(%w[status note]).size).to eq(2) + expect(matching_fields(%w[missing])).to eq([]) + end + + it 'treats a field name holding a dot as a whole key, not a path' do + store.append(record(previous_values: { 'address.city' => 'Paris' }, + new_values: { 'address.city' => 'Lyon' })) + store.append(record(previous_values: { 'address' => { 'city' => 'Paris' } }, + new_values: { 'address' => { 'city' => 'Lyon' } })) + + expect(matching_fields(['address.city'])).to eq([{ 'address.city' => 'Lyon' }]) + expect(matching_fields(['address'])).to eq([{ 'address' => { 'city' => 'Lyon' } }]) + end + + it 'counts with the field filter applied' do + store.append(record(previous_values: { 'status' => 'open' }, new_values: { 'status' => 'closed' })) + store.append(record(previous_values: { 'note' => 'a' }, new_values: { 'note' => 'b' })) + + expect(store.count_by_record(collection: 'accounts', record_id: '1', fields: %w[status])).to eq(1) + end + + it 'refuses to filter by field on an adapter it has no JSON test for' do + store.append(record) + allow(store.send(:model).connection).to receive(:adapter_name).and_return('Informix') + + expect { store.list_by_record(collection: 'accounts', record_id: '1', fields: %w[status]) } + .to raise_error(ForestAdminDatasourceToolkit::Exceptions::ForestException, /not supported on informix/) + end + + describe '#list_since' do + # A pending row records an attempt whose outcome is unknown; undoing it would invent a state the + # record was never in. + it 'leaves a pending row out, so a reconstruction cannot undo a change that may never have happened' do + store.append(record(timestamp: '2026-01-02T03:04:07.000Z', status: Recording::PENDING, + correlation_key: 'attempt')) + store.append(record(timestamp: '2026-01-02T03:04:07.000Z', correlation_key: 'confirmed')) + + history = store.list_since(collection: 'accounts', record_id: '1', + timestamp: '2026-01-02T03:04:06.000Z') + + expect(history.map(&:correlation_key)).to eq(['confirmed']) + end + + # They are evidence, and `status` tells the reader what they are. + it 'keeps them in the history, where the payload says what they are' do + store.append(record(status: Recording::PENDING)) + + expect(store.list_by_record(collection: 'accounts', record_id: '1').map(&:status)).to eq(['pending']) + end + + it 'returns entries strictly newer than the instant, newest first' do + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'older')) + store.append(record(timestamp: '2026-01-02T03:04:06.000Z', correlation_key: 'at')) + store.append(record(timestamp: '2026-01-02T03:04:07.000Z', correlation_key: 'newer')) + + history = store.list_since(collection: 'accounts', record_id: '1', + timestamp: '2026-01-02T03:04:06.000Z') + + expect(history.map(&:correlation_key)).to eq(['newer']) + end + + it 'breaks ties on equal timestamps by reverse insertion order' do + store.append(record(timestamp: '2026-01-02T03:04:07.000Z', correlation_key: 'first')) + store.append(record(timestamp: '2026-01-02T03:04:07.000Z', correlation_key: 'second')) + + history = store.list_since(collection: 'accounts', record_id: '1', + timestamp: '2026-01-02T03:04:06.000Z') + + expect(history.map(&:correlation_key)).to eq(%w[second first]) + end + end + + describe 'the pending/confirm protocol' do + it 'inserts rows and hands their ids back, in order' do + ids = store.append_all([record(correlation_key: 'a'), record(correlation_key: 'b')]) + + expect(ids.size).to eq(2) + expect(store.list_by_record(collection: 'accounts', record_id: '1').map(&:id)).to eq(ids) + end + + it 'confirms a pending row into a done one, values included' do + id = store.append(record(status: Recording::PENDING, previous_values: {}, new_values: {})) + + store.confirm(id, record_id: '9', new_values: { 'status' => 'closed' }) + + audit = store.list_by_record(collection: 'accounts', record_id: '9').first + expect(audit.status).to eq(Recording::DONE) + expect(audit.new_values).to eq({ 'status' => 'closed' }) + end + + # A write that changed nothing leaves no trace, rather than a row implying it is unaccounted for. + it 'discards rows by id' do + ids = store.append_all([record, record]) + + store.discard(ids) + + expect(store.list_by_record(collection: 'accounts', record_id: '1')).to be_empty + end + + it 'accepts a pending create row with no record id yet' do + id = store.append(record(operation: 'create', record_id: nil, status: Recording::PENDING)) + + expect { store.confirm(id, record_id: '7') }.not_to raise_error + expect(store.list_by_record(collection: 'accounts', record_id: '7').first.operation).to eq('create') + end + + it 'stores a packed composite id far longer than a varchar would hold' do + long_id = (1..40).map { |part| "part-#{part}-#{"x" * 20}" }.join('|') + store.append(record(record_id: long_id)) + + expect(store.list_by_record(collection: 'accounts', record_id: long_id).size).to eq(1) + end + end + + describe '#renamed_from' do + it 'returns each id it was renamed from, with the moment it stopped being that id' do + store.append(record(record_id: '7', previous_record_id: '1', timestamp: '2026-01-02T00:00:01.000Z')) + store.append(record(record_id: '7', previous_record_id: '1', timestamp: '2026-01-02T00:00:09.000Z')) + store.append(record(record_id: '7')) + + renamed = store.renamed_from(collection: 'accounts', record_id: '7') + + expect(renamed.map { |segment| segment.slice(:id, :until) }).to eq( + [{ id: '1', until: '2026-01-02T00:00:09.000Z' }] + ) + # The row id of that rename, so an equal-timestamp row can be placed either side of it. + expect(renamed.first[:until_row]).to be_a(Integer) + end + + it 'returns nothing for a record that was never renamed' do + store.append(record) + + expect(store.renamed_from(collection: 'accounts', record_id: '1')).to be_empty + end + end + + # An id a record left may since have been taken by another record: its rows are none of this one's + # business, so an earlier segment only reaches up to the rename. + describe 'reading a renamed record' do + it 'takes the earlier id only up to the moment it was left' do + store.append(record(record_id: '1', timestamp: '2026-01-02T00:00:01.000Z', correlation_key: 'mine')) + store.append(record(record_id: '7', timestamp: '2026-01-02T00:00:05.000Z', correlation_key: 'renamed', + previous_record_id: '1')) + store.append(record(record_id: '1', timestamp: '2026-01-02T00:00:09.000Z', correlation_key: 'someone-else')) + + history = store.list_by_record( + collection: 'accounts', + record_id: [{ id: '7', until: nil }, { id: '1', until: '2026-01-02T00:00:05.000Z' }] + ) + + expect(history.map(&:correlation_key)).to eq(%w[mine renamed]) + end + + # Same millisecond as the rename: the trail orders by (timestamp, id), so the bound does too. + it 'places a row written in the very millisecond of the rename by insertion order' do + mine = store.append(record(record_id: '1', timestamp: '2026-01-02T00:00:05.000Z', correlation_key: 'mine')) + rename = store.append(record(record_id: '7', timestamp: '2026-01-02T00:00:05.000Z', + previous_record_id: '1', correlation_key: 'renamed')) + store.append(record(record_id: '1', timestamp: '2026-01-02T00:00:05.000Z', correlation_key: 'theirs')) + + history = store.list_by_record( + collection: 'accounts', + record_id: [{ id: '7', until: nil }, { id: '1', until: '2026-01-02T00:00:05.000Z', until_row: rename }] + ) + + expect(history.map(&:correlation_key)).to eq(%w[mine renamed]) + expect(mine).to be < rename + end + + it 'counts the same rows it lists' do + store.append(record(record_id: '1', timestamp: '2026-01-02T00:00:01.000Z')) + store.append(record(record_id: '1', timestamp: '2026-01-02T00:00:09.000Z')) + + segments = [{ id: '1', until: '2026-01-02T00:00:05.000Z' }] + + expect(store.count_by_record(collection: 'accounts', record_id: segments)).to eq(1) + end + end + + describe 'searching free text' do + def matching_search(term) + store.list_by_record(collection: 'accounts', record_id: '1', search: term).map(&:correlation_key) + end + + it 'matches a value, case-insensitively and as a substring' do + store.append(record(correlation_key: 'lyon', new_values: { 'city' => 'Lyon' })) + store.append(record(correlation_key: 'paris', new_values: { 'city' => 'Paris' })) + + expect(matching_search('lyo')).to eq(['lyon']) + expect(matching_search('LYON')).to eq(['lyon']) + end + + # Only the changed leaves of a JSON column are stored, so the match has to reach them. + it 'reaches a value nested at any depth' do + store.append(record(correlation_key: 'nested', + new_values: { 'address' => { 'city' => 'Lyon', 'zip' => '69001' } })) + + expect(matching_search('Lyon')).to eq(['nested']) + expect(matching_search('69001')).to eq(['nested']) + end + + it 'matches a key as well as a value' do + store.append(record(correlation_key: 'keyed', new_values: { 'first_name' => 'Jo' })) + + expect(matching_search('first_name')).to eq(['keyed']) + end + + it 'matches the previous side too, not only the new one' do + store.append(record(correlation_key: 'was', previous_values: { 'city' => 'Lyon' }, new_values: {})) + + expect(matching_search('Lyon')).to eq(['was']) + end + + it 'matches the action name and who acted' do + store.append(record(correlation_key: 'refund', operation: 'action', action_name: 'Refund order', + previous_values: {}, new_values: {})) + store.append(record(correlation_key: 'ada', user_first_name: 'Ada', user_last_name: 'Lovelace', + user_email: 'ada@test', previous_values: {}, new_values: {})) + + expect(matching_search('refund ord')).to eq(['refund']) + expect(matching_search('lovelace')).to eq(['ada']) + expect(matching_search('ada@')).to eq(['ada']) + end + + # Machine identifiers nobody searches for: matching them turns one term into confusing hits. + it 'ignores the operation, the correlation key, the record id and the status' do + store.append(record(correlation_key: 'searchable-key', operation: 'update', record_id: '1', + previous_values: {}, new_values: {})) + + expect(matching_search('searchable-key')).to be_empty + expect(matching_search('update')).to be_empty + expect(matching_search('done')).to be_empty + end + + # A search must never confirm a value the trail refused to record. + it 'never matches a redacted field, by its mask or by the value it hid' do + store.append(record(correlation_key: 'masked', previous_values: { 'email' => '[redacted]' }, + new_values: { 'email' => '[redacted]' })) + + expect(matching_search('redacted')).to be_empty + expect(matching_search('[redacted]')).to be_empty + expect(matching_search('secret@test')).to be_empty + end + + it 'treats LIKE wildcards in the term as ordinary characters' do + store.append(record(correlation_key: 'literal', new_values: { 'code' => '50%_off' })) + store.append(record(correlation_key: 'other', new_values: { 'code' => 'anything' })) + + expect(matching_search('50%_')).to eq(['literal']) + expect(matching_search('%')).to eq(['literal']) + end + + # The values sit in a serialized document, where a quote is escaped: the raw term would never find them. + it 'finds a value holding a quote, a backslash or a newline' do + store.append(record(correlation_key: 'quote', new_values: { 'label' => '15" monitor' })) + store.append(record(correlation_key: 'path', new_values: { 'path' => 'C:\\Users\\ada' })) + store.append(record(correlation_key: 'multiline', new_values: { 'note' => "first\nsecond" })) + + expect(matching_search('15" monitor')).to eq(['quote']) + expect(matching_search('C:\\Users')).to eq(['path']) + expect(matching_search("first\nsecond")).to eq(['multiline']) + end + + # A bare quote would otherwise match the document's own structure, so every row. + it 'does not let a bare quote match every row' do + store.append(record(new_values: { 'city' => 'Lyon' })) + + expect(matching_search('"')).to be_empty + end + + it 'composes with the other filters, and the count agrees with it' do + store.append(record(correlation_key: 'keep', user_id: 7, new_values: { 'city' => 'Lyon' })) + store.append(record(correlation_key: 'wrong-user', user_id: 9, new_values: { 'city' => 'Lyon' })) + store.append(record(correlation_key: 'wrong-term', user_id: 7, new_values: { 'city' => 'Paris' })) + + expect(store.list_by_record(collection: 'accounts', record_id: '1', search: 'lyon', + user_ids: [7]).map(&:correlation_key)).to eq(['keep']) + expect(store.count_by_record(collection: 'accounts', record_id: '1', search: 'lyon', + user_ids: [7])).to eq(1) + end + + it 'narrows the authors it offers to those the term matches' do + store.append(record(user_id: 7, new_values: { 'city' => 'Lyon' })) + store.append(record(user_id: 9, new_values: { 'city' => 'Paris' })) + + authors = store.authors_by_record(collection: 'accounts', record_id: '1', search: 'lyon') + + expect(authors.map { |author| author[:user_id] }).to eq([7]) + end + end + + describe '#authors_by_record' do + it 'lists the distinct authors of the matching entries, as they were when they acted' do + store.append(record(user_id: 7, user_first_name: 'Ada', user_last_name: 'L', user_email: 'ada@test')) + store.append(record(user_id: 7, user_first_name: 'Ada', user_last_name: 'L', user_email: 'ada@test')) + store.append(record(user_id: 9, user_first_name: 'Bob', user_last_name: 'K', user_email: 'bob@test')) + store.append(record(record_id: '2', user_id: 11, user_first_name: 'Other')) + + authors = store.authors_by_record(collection: 'accounts', record_id: '1') + + expect(authors).to contain_exactly( + { user_id: 7, user_first_name: 'Ada', user_last_name: 'L', user_email: 'ada@test' }, + { user_id: 9, user_first_name: 'Bob', user_last_name: 'K', user_email: 'bob@test' } + ) + end + + it 'stays inside the active filters but ignores paging' do + store.append(record(user_id: 7, timestamp: '2026-01-02T03:04:05.000Z')) + store.append(record(user_id: 9, timestamp: '2026-01-09T03:04:05.000Z')) + + authors = store.authors_by_record(collection: 'accounts', record_id: '1', + end_timestamp: '2026-01-03T00:00:00.000Z') + + expect(authors.map { |author| author[:user_id] }).to eq([7]) + end + + it 'leaves out entries with no author' do + store.append(record(user_id: nil)) + + expect(store.authors_by_record(collection: 'accounts', record_id: '1')).to be_empty + end + end + + # establish_connection is class-level: two stores on different databases would silently share one pool. + it 'refuses a second audit database rather than clobbering the first' do + store.append(record) + other = described_class.new(database: { adapter: 'sqlite3', database: ':memory:' }) + + expect { other.append(record) }.to raise_error( + ForestAdminDatasourceToolkit::Exceptions::ForestException, /One agent, one audit database/ + ) + end + + it 'indexes record_id, correlation_key and user_id' do + store.append(record) + + index_names = connection.indexes('audit_logs').map(&:name) + expect(index_names).to include( + 'audit_logs_record_id', 'audit_logs_correlation_key', 'audit_logs_user_id' + ) + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/builder/agent_factory_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/builder/agent_factory_spec.rb index 0382c327f..2f2559969 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/builder/agent_factory_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/builder/agent_factory_spec.rb @@ -254,6 +254,53 @@ module Builder end end + describe 'audit trail' do + let(:instance) { described_class.instance } + let(:options) do + { + auth_secret: 'cba803d01a4d43b55010cab41fa1ea1f1f51a95e', + env_secret: '89719c6d8e2e2de2694c2f220fe2dbf02d5289487364daf1e4c6b13733ed0cdb', + is_production: false, + schema_path: File.join('tmp', '.forestadmin-schema.json') + } + end + let(:audit_options) { { database: { adapter: 'sqlite3', database: ':memory:' } } } + + before { allow(instance).to receive(:send_schema) } + + it 'stays off when no audit trail database is configured' do + instance.setup(options) + allow(instance.customizer).to receive(:use) + + instance.build + + expect(instance.container.resolve(:config)[:audit_trail]).to be_nil + expect(instance.customizer).not_to have_received(:use) + end + + it 'stays off when the audit trail option carries no database' do + instance.setup(options.merge(audit_trail: { redact: { 'users' => ['email'] } })) + allow(instance.customizer).to receive(:use) + + instance.build + + expect(instance.container.resolve(:config)[:audit_trail][:store]).to be_nil + expect(instance.customizer).not_to have_received(:use) + end + + it 'builds the store from the configured database and installs the capture layer' do + instance.setup(options.merge(audit_trail: audit_options.merge(redact: { 'users' => ['email'] }))) + allow(instance.customizer).to receive(:use) + + instance.build + + store = instance.container.resolve(:config)[:audit_trail][:store] + expect(store).to be_a(AuditTrail::Store) + expect(instance.customizer).to have_received(:use) + .with(AuditTrail::Capture, { store: store, redact: { 'users' => ['email'] } }) + end + end + describe 'generate_schema_only' do it 'generates schema and writes to default path' do instance = described_class.instance diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/correlation_id_middleware_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/correlation_id_middleware_spec.rb new file mode 100644 index 000000000..474329e1c --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/correlation_id_middleware_spec.rb @@ -0,0 +1,46 @@ +require 'spec_helper' + +module ForestAdminAgent + module Http + describe CorrelationIdMiddleware do + after { CorrelationId.reset! } + + it 'echoes the id generated during the request on the response header' do + app = ->(_env) { [200, {}, [CorrelationId.current]] } + + _status, headers, body = described_class.new(app).call({}) + + expect(headers[CorrelationId::HEADER]).to eq(body.first) + end + + it 'does not set the header when no id was generated during the request' do + app = ->(_env) { [200, {}, ['ok']] } + + _status, headers, = described_class.new(app).call({}) + + expect(headers).not_to have_key(CorrelationId::HEADER) + end + + it 'resets any leaked id before handling the request' do + CorrelationId.current = 'stale' + seen = 'unset' + app = lambda do |_env| + seen = CorrelationId.current? + [200, {}, ['ok']] + end + + described_class.new(app).call({}) + + expect(seen).to be_nil + end + + it 'clears the id after the request so the thread is not reused with a stale id' do + app = ->(_env) { [200, {}, [CorrelationId.current]] } + + described_class.new(app).call({}) + + expect(CorrelationId.current?).to be_nil + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/correlation_id_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/correlation_id_spec.rb new file mode 100644 index 000000000..6809ce09f --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/correlation_id_spec.rb @@ -0,0 +1,29 @@ +require 'spec_helper' + +module ForestAdminAgent + module Http + describe CorrelationId do + after { described_class.reset! } + + it 'lazily generates and memoizes an id within the thread' do + id = described_class.current + + expect(id).to match(/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/) + expect(described_class.current).to eq(id) + end + + it 'can be seeded by the host' do + described_class.current = 'req-1' + + expect(described_class.current).to eq('req-1') + end + + it 'reset! clears it so a fresh id is generated next' do + first = described_class.current + described_class.reset! + + expect(described_class.current).not_to eq(first) + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/router_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/router_spec.rb index 39054100b..6916ae740 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/router_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/router_spec.rb @@ -3,6 +3,21 @@ module ForestAdminAgent module Http describe Router do + # Rails matches in definition order, so a literal-prefixed path registered after the + # `:collection_name` routes is read as a collection name instead (a 2-segment + # `/_audit-trail/correlations` would 404 as collection `_audit-trail`, id `correlations`). + describe 'route order' do + it 'registers the audit-trail routes before the ones matching on :collection_name' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: Object.new } }) + names = described_class.routes.keys + + expect(names.index('forest_audit_trail_correlations')).to be < names.index('forest_show') + expect(names.index('forest_audit_trail_correlations')).to be < names.index('forest_list') + expect(names.index('forest_audit_trail_correlations')).to be < names.index('forest_audit_trail') + end + end + describe '.cached_routes' do before do described_class.reset_cached_routes! diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb new file mode 100644 index 000000000..2a42a043c --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb @@ -0,0 +1,165 @@ +require 'spec_helper' + +module ForestAdminAgent + module Routes + module Action + include ForestAdminDatasourceToolkit::Schema + include ForestAdminDatasourceToolkit::Components::Query::ConditionTree + + describe Actions do + subject(:route) { described_class.new(collection, 'Refund') } + + let(:action_scope) { ForestAdminDatasourceCustomizer::Decorators::Action::Types::ActionScope::BULK } + let(:collection) do + build_collection( + name: 'orders', + schema: { + fields: { + 'id' => ColumnSchema.new( + column_type: 'Number', is_primary_key: true, + filter_operators: [Operators::IN, Operators::EQUAL] + ) + }, + actions: { 'Refund' => double('action', scope: action_scope) } + }, + list: [{ 'id' => 4 }, { 'id' => 7 }] + ) + end + let(:context) { double('context', collection: collection, caller: build_caller) } + let(:filter) { ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: nil) } + + describe 'the records it audits' do + def audited_ids + route.send(:audited_record_ids, context, filter) + end + + # The ids a client sends are a claim. In a compliance record, asserting an operator acted on a record + # their scope excludes is worse than a missing row, so the selection is read back through the caller's + # own filter. + it 'reads the targets back through the caller filter rather than trusting the request' do + expect(audited_ids).to eq(%w[4 7]) + expect(collection).to have_received(:list) do |_caller, listed, projection| + expect(listed.page.limit).to eq(ForestAdminAgent::AuditTrail::MAX_RECORDS_PER_OPERATION + 1) + expect(projection).to eq(['id']) + end + end + + # Same rule as a bulk write: one unattached row for a run that touched more is a partial audit. + it 'refuses a selection wider than the cap when critical is on' do + allow(ForestAdminAgent::AuditTrail).to receive(:critical?).and_return(true) + cap = ForestAdminAgent::AuditTrail::MAX_RECORDS_PER_OPERATION + allow(collection).to receive(:list).and_return((1..(cap + 1)).map { |id| { 'id' => id } }) + + expect { audited_ids }.to raise_error( + ForestAdminDatasourceToolkit::Exceptions::ForestException, /cannot record an operation touching more/ + ) + end + + it 'records a selection wider than the cap as attached to no record, and says so' do + logger = instance_spy(Services::LoggerService) + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) + cap = ForestAdminAgent::AuditTrail::MAX_RECORDS_PER_OPERATION + allow(collection).to receive(:list).and_return((1..(cap + 1)).map { |id| { 'id' => id } }) + + expect(audited_ids).to eq([]) + expect(logger).to have_received(:log).with('Warn', /records audited/) + end + + context 'with a global action' do + let(:action_scope) { ForestAdminDatasourceCustomizer::Decorators::Action::Types::ActionScope::GLOBAL } + + it 'names no record, and does not even query' do + expect(audited_ids).to eq([]) + expect(collection).not_to have_received(:list) + end + end + end + + describe 'auditing the run' do + let(:store) { double('store', append_all: [10, 11], confirm: nil) } + + before do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: store, redact: { 'orders' => ['reason'] } } }) + end + + def execute(result: { type: 'Success', message: 'done' }, &raising) + allow(collection).to receive(:execute, &(raising || proc { result })) + + route.send(:execute_and_audit, context, {}, { 'amount' => 30, 'reason' => 'damaged' }, filter) + end + + # Pending before the action, confirmed after: an action that takes the process down with it still + # leaves evidence that it started. + it 'records the run before it happens, one row per target' do + execute + + expect(store).to have_received(:append_all) do |rows| + expect(rows.map(&:record_id)).to eq(%w[4 7]) + expect(rows.map(&:status).uniq).to eq([ForestAdminAgent::AuditTrail::Recording::PENDING]) + expect(rows.first.action_name).to eq('Refund') + expect(rows.first.previous_values).to eq({ 'amount' => 30, 'reason' => '[redacted]' }) + end + end + + it 'confirms both rows with what the action answered, and returns it' do + expect(execute).to eq({ type: 'Success', message: 'done' }) + + expect(store).to have_received(:confirm).with( + 10, hash_including(operation: 'action', new_values: { 'type' => 'Success', 'message' => 'done' }) + ) + expect(store).to have_received(:confirm).with(11, any_args) + end + + it 'confirms as failed and re-raises when the action raises' do + expect { execute { raise StandardError, 'boom' } }.to raise_error(StandardError, 'boom') + + expect(store).to have_received(:confirm).with(10, hash_including(operation: 'action_failed')) + end + + it 'confirms as failed when the action answers with an Error result' do + execute(result: { type: 'Error', message: 'not allowed' }) + + expect(store).to have_received(:confirm).with(10, hash_including(operation: 'action_failed')) + end + + it 'does nothing at all without an audit database, not even reading the selection' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache).and_return({}) + + expect(execute).to eq({ type: 'Success', message: 'done' }) + expect(collection).not_to have_received(:list) + expect(store).not_to have_received(:append_all) + end + + # The action has not run yet, so refusing costs nothing to repair — but only when asked to. + it 'refuses the action when the pending row cannot be written and critical is on' do + allow(ForestAdminAgent::AuditTrail).to receive(:critical?).and_return(true) + allow(store).to receive(:append_all).and_raise(StandardError, 'audit db is down') + + expect { execute }.to raise_error(StandardError, 'audit db is down') + expect(collection).not_to have_received(:execute) + end + + it 'runs the action anyway when the pending row cannot be written and critical is off' do + logger = instance_spy(Services::LoggerService) + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) + allow(store).to receive(:append_all).and_raise(StandardError, 'audit db is down') + + expect(execute).to eq({ type: 'Success', message: 'done' }) + expect(logger).to have_received(:log).with('Error', /audit db is down/) + end + + # Reading the selection happens inside the gate too, so a failure there cannot 500 an action. + it 'runs the action anyway when the selection cannot be read' do + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return( + instance_spy(Services::LoggerService) + ) + allow(collection).to receive(:list).and_raise(StandardError, 'datasource down') + + expect(execute).to eq({ type: 'Success', message: 'done' }) + end + end + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/capabilities/collections_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/capabilities/collections_spec.rb index a265e153c..d3a5482d1 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/capabilities/collections_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/capabilities/collections_spec.rb @@ -82,13 +82,24 @@ module Capabilities ) end + it 'announces the audit trail once a store is configured' do + # Merged rather than replaced: the rest of the request reads this config too. + configured = ForestAdminAgent::Facades::Container.config_from_cache.merge( + audit_trail: { store: Object.new } + ) + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache).and_return(configured) + + expect(result[:content][:agentCapabilities][:canUseAuditTrail]).to be true + end + it 'returns agentCapabilities' do expect(result[:content][:agentCapabilities]).to eq( { canUseProjectionOnGetOne: true, canUseProjectionViaHeader: true, canUseProjectionViaHeaderOnList: true, - canUseMultipleFieldsProjectionOnRelation: true + canUseMultipleFieldsProjectionOnRelation: true, + canUseAuditTrail: false } ) end @@ -140,7 +151,8 @@ module Capabilities canUseProjectionOnGetOne: true, canUseProjectionViaHeader: true, canUseProjectionViaHeaderOnList: true, - canUseMultipleFieldsProjectionOnRelation: true + canUseMultipleFieldsProjectionOnRelation: true, + canUseAuditTrail: false } ) end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb new file mode 100644 index 000000000..e215f7bec --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb @@ -0,0 +1,172 @@ +require 'spec_helper' + +module ForestAdminAgent + module Routes + module Resources + include ForestAdminDatasourceToolkit::Schema + include ForestAdminDatasourceToolkit::Components::Query::ConditionTree + + describe AuditTrailCorrelation do + let(:store) { double('store') } + let(:permissions) { double('permissions', can?: true, get_scope: nil) } + let(:collection) do + build_collection( + name: 'books', + schema: { + fields: { + 'id' => ColumnSchema.new( + column_type: 'Number', is_primary_key: true, + filter_operators: [Operators::IN, Operators::EQUAL] + ) + } + }, + list: [{ 'id' => 2 }] + ) + end + + def route_with_store(history: []) + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: store } }) + allow(store).to receive_messages(list_by_correlation: history, list_by_correlations: history) + + route = described_class.new + datasource = double('datasource') + allow(datasource).to receive(:get_collection).with('books').and_return(collection) + context = double('context', datasource: datasource, caller: build_caller, permissions: permissions) + allow(route).to receive(:build).and_return(context) + route + end + + it 'returns 404 without touching the store when the record exists outside the caller scope' do + allow(permissions).to receive(:get_scope).and_return(Nodes::ConditionTreeLeaf.new('id', Operators::EQUAL, 9)) + allow(collection).to receive(:list).and_return([], [{ 'id' => 2 }]) + route = route_with_store + + expect do + route.handle_history( + { headers: {}, params: { 'collection' => 'books', 'recordId' => '2', 'correlation_key' => 'req-1' } } + ) + end.to raise_error(Http::Exceptions::NotFoundError) + + expect(store).not_to have_received(:list_by_correlation) + end + + it 'registers the correlation routes when a store is configured' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: Object.new } }) + + expect(described_class.new.routes.keys).to include( + 'forest_audit_trail_correlation', 'forest_audit_trail_correlations', 'forest_audit_trail_correlations_batch' + ) + end + + it 'does not register when no store is configured' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache).and_return({}) + + expect(described_class.new.routes).to be_empty + end + + it 'reads a single correlation history scoped to the record' do + entry = { operation: 'update', record_id: '2', new_values: { 'first_name' => 'Jo' } } + route = route_with_store(history: [double('entry', to_h: entry)]) + + # Through the registered closure rather than the handler, so the wiring is covered too. + result = route.routes['forest_audit_trail_correlation'][:closure].call( + { headers: {}, params: { 'collection' => 'books', 'recordId' => '2', 'correlation_key' => 'req-1' } } + ) + + expect(store).to have_received(:list_by_correlation).with( + collection: 'books', record_id: '2', correlation_key: 'req-1' + ) + # Same serialization as the per-record route: camelCase on top, column names left alone. + expect(result[:content]).to eq( + { data: [{ 'operation' => 'update', 'recordId' => '2', 'newValues' => { 'first_name' => 'Jo' } }] } + ) + end + + it 'reads a batch history from comma-separated query keys (GET)' do + route = route_with_store(history: [double('entry', to_h: { operation: 'update' })]) + + route.routes['forest_audit_trail_correlations'][:closure].call( + { headers: {}, params: { 'collection' => 'books', 'recordId' => '2', 'correlationKeys' => 'a, b' } } + ) + + expect(store).to have_received(:list_by_correlations).with( + collection: 'books', record_id: '2', correlation_keys: %w[a b] + ) + end + + it 'reads a batch history from a body array (POST)' do + route = route_with_store + + route.routes['forest_audit_trail_correlations_batch'][:closure].call( + { headers: {}, params: { 'collection' => 'books', 'recordId' => '2', 'correlationKeys' => %w[a b] } } + ) + + expect(store).to have_received(:list_by_correlations).with( + collection: 'books', record_id: '2', correlation_keys: %w[a b] + ) + end + + it 'returns an empty batch without hitting the store when no keys are given' do + route = route_with_store + + result = route.handle_batch({ headers: {}, params: { 'collection' => 'books', 'recordId' => '2' } }) + + expect(store).not_to have_received(:list_by_correlations) + expect(result[:content]).to eq({ data: [] }) + end + + it 'answers 404 for a collection the datasource does not know' do + route = route_with_store + datasource = double('datasource') + allow(datasource).to receive(:get_collection).with('ghosts') + .and_raise(ForestAdminDatasourceToolkit::Exceptions::ForestException, + "Collection 'ghosts' not found") + allow(route).to receive(:build).and_return( + double('context', datasource: datasource, caller: build_caller, permissions: permissions) + ) + + expect do + route.handle_history( + { headers: {}, params: { 'collection' => 'ghosts', 'recordId' => '2', 'correlation_key' => 'req-1' } } + ) + end.to raise_error(Http::Exceptions::NotFoundError, /not found/) + end + + it 'passes through a datasource error that is not a missing collection' do + route = route_with_store + datasource = double('datasource') + allow(datasource).to receive(:get_collection).with('books') + .and_raise(ForestAdminDatasourceToolkit::Exceptions::ForestException, + 'connection lost') + allow(route).to receive(:build).and_return( + double('context', datasource: datasource, caller: build_caller, permissions: permissions) + ) + + expect do + route.handle_history( + { headers: {}, params: { 'collection' => 'books', 'recordId' => '2', 'correlation_key' => 'req-1' } } + ) + end.to raise_error(ForestAdminDatasourceToolkit::Exceptions::ForestException, /connection lost/) + end + + it 'rejects a missing collection' do + route = route_with_store + + expect do + route.handle_history({ headers: {}, params: { 'recordId' => '2', 'correlation_key' => 'req-1' } }) + end.to raise_error(Http::Exceptions::ValidationError, /Missing collection/) + end + + it 'rejects a missing recordId' do + route = route_with_store + + expect do + route.handle_history({ headers: {}, params: { 'collection' => 'books', 'correlation_key' => 'req-1' } }) + end.to raise_error(Http::Exceptions::ValidationError, /Missing recordId/) + end + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb new file mode 100644 index 000000000..f45216649 --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -0,0 +1,539 @@ +require 'spec_helper' + +module ForestAdminAgent + module Routes + module Resources + include ForestAdminDatasourceToolkit::Schema + include ForestAdminDatasourceToolkit::Components::Query::ConditionTree + + describe AuditTrail do + let(:store) { double('store') } + let(:permissions) { double('permissions', can?: true, get_scope: nil) } + let(:collection) do + build_collection( + name: 'projects', + schema: { + fields: { + 'id' => ColumnSchema.new( + column_type: 'Number', is_primary_key: true, + filter_operators: [Operators::IN, Operators::EQUAL] + ), + 'status' => ColumnSchema.new(column_type: 'String') + } + }, + list: [{ 'id' => 4 }] + ) + end + + def route_with_store(records: []) + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: store } }) + allow(store).to receive_messages(list_by_record: records, count_by_record: records.length, + authors_by_record: [], renamed_from: []) + + route = described_class.new + context = double('context', collection: collection, caller: build_caller, permissions: permissions) + allow(route).to receive(:build).and_return(context) + route + end + + describe 'state reconstruction' do + def state_route(entries: [], record: { 'id' => 4, 'status' => 'shipped' }) + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: store } }) + allow(store).to receive_messages(list_since: entries, renamed_from: []) + allow(collection).to receive(:list).and_return([record].compact) + + route = described_class.new + context = double('context', collection: collection, caller: build_caller, permissions: permissions) + allow(route).to receive(:build).and_return(context) + route + end + + def entry(operation, previous_values = {}, new_values = {}) + ForestAdminAgent::AuditTrail::AuditRecord.new( + operation: operation, collection: 'projects', record_id: '4', + previous_values: previous_values, new_values: new_values + ) + end + + # Through the registered closure rather than the handler, so the wiring is covered too. + def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) + route.routes['forest_audit_trail_state'][:closure].call( + { headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'timestamp' => timestamp }.merge(extra) } + ) + end + + it 'registers the state route' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: Object.new } }) + + expect(described_class.new.routes).to include('forest_audit_trail_state') + end + + it 'returns the record with every later entry undone' do + route = state_route(entries: [entry('update', { 'status' => 'paid' }, { 'status' => 'shipped' })]) + + result = get_state(route) + + expect(result[:content]).to eq({ data: { 'id' => 4, 'status' => 'paid' } }) + end + + # Authorization and read are one query: a scoped check followed by an unscoped read would hand + # back a row the check never covered. + it 'reads the record through the caller scope, in a single query' do + scope = Nodes::ConditionTreeLeaf.new('id', Operators::EQUAL, 4) + allow(permissions).to receive(:get_scope).and_return(scope) + route = state_route + + get_state(route) + + expect(collection).to have_received(:list).once + expect(collection).to have_received(:list) do |_caller, filter, projection| + expect(filter.condition_tree.conditions).to include(scope) + expect(projection).to include('status') + end + end + + it 'refuses a record that exists outside the caller scope' do + allow(permissions).to receive(:get_scope).and_return(Nodes::ConditionTreeLeaf.new('id', + Operators::EQUAL, 9)) + route = state_route(record: nil) + # Nothing in scope, but the record does exist without it: someone else's. + allow(collection).to receive(:list).and_return([], [{ 'id' => 4 }]) + + expect { get_state(route) }.to raise_error(Http::Exceptions::NotFoundError) + expect(store).not_to have_received(:list_since) + end + + it 'rebuilds a deleted record from its history' do + allow(permissions).to receive(:get_scope).and_return(Nodes::ConditionTreeLeaf.new('id', + Operators::EQUAL, 4)) + route = state_route(entries: [entry('delete', { 'status' => 'shipped' }, {})], record: nil) + allow(collection).to receive(:list).and_return([], []) + + expect(get_state(route)[:content][:data]).to eq({ 'status' => 'shipped' }) + end + + # Rows written before an update moved a writable primary key stay under the id they were true of. + it 'reconstructs state from every id the record has been filed under' do + route = state_route + allow(store).to receive(:renamed_from).and_return([{ id: '1', until: nil, until_row: nil }], []) + + get_state(route) + + expect(store).to have_received(:list_since).with( + hash_including(record_id: [{ id: '4', until: nil, until_row: nil }, + { id: '1', until: nil, until_row: nil }]) + ) + end + + # Strictly after the requested instant: an entry stamped exactly at it belongs to that state. + it 'asks the store for entries strictly newer than the instant' do + route = state_route + get_state(route) + + expect(store).to have_received(:list_since).with( + collection: 'projects', record_id: [{ id: '4', until: nil, until_row: nil }], + timestamp: '2026-01-02T10:00:00.000Z' + ) + end + + it 'returns no data when the record did not exist yet' do + route = state_route(entries: [entry('create', {}, { 'status' => 'draft' })]) + + expect(get_state(route)[:content][:data]).to be_nil + end + + it 'reads a wall-clock instant in the request timezone' do + route = state_route + get_state(route, timestamp: '2026-01-02T08:30', extra: { 'timezone' => 'America/New_York' }) + + expect(store).to have_received(:list_since).with(hash_including(timestamp: '2026-01-02T13:30:00.000Z')) + end + + # Seconds make it parse as ISO-8601, which would silently read it in the server's timezone. + it 'reads a wall-clock instant carrying seconds in the request timezone too' do + route = state_route + get_state(route, timestamp: '2026-01-02T08:30:15', extra: { 'timezone' => 'America/New_York' }) + + expect(store).to have_received(:list_since).with(hash_including(timestamp: '2026-01-02T13:30:15.000Z')) + end + + it 'reads a bare day in the request timezone' do + route = state_route + get_state(route, timestamp: '2026-01-02', extra: { 'timezone' => 'America/New_York' }) + + expect(store).to have_received(:list_since).with(hash_including(timestamp: '2026-01-02T05:00:00.000Z')) + end + + it 'honours an explicit offset instead of the request timezone' do + route = state_route + get_state(route, timestamp: '2026-01-02T08:30:15+02:00', extra: { 'timezone' => 'America/New_York' }) + + expect(store).to have_received(:list_since).with(hash_including(timestamp: '2026-01-02T06:30:15.000Z')) + end + + it 'rejects a missing timestamp' do + route = state_route + + expect { get_state(route, timestamp: '') }.to raise_error( + Http::Exceptions::ValidationError, /Missing timestamp/ + ) + end + + it 'rejects an unparsable timestamp' do + route = state_route + + expect { get_state(route, timestamp: 'yesterday') }.to raise_error(Http::Exceptions::ValidationError) + end + end + + it 'registers the record-history route when an audit_trail store is configured' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: Object.new } }) + + expect(described_class.new.routes).to include('forest_audit_trail') + end + + it 'does not register the route when no audit_trail store is configured' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache).and_return({}) + + expect(described_class.new.routes).not_to include('forest_audit_trail') + end + + it 'reads the history scoped to the packed id and returns data + filtered count' do + entry = { operation: 'update', record_id: '4', previous_values: { 'first_name' => 'Jo' } } + route = route_with_store(records: [double('entry', to_h: entry)]) + + # Through the registered closure rather than the handler, so the wiring is covered too. + result = route.routes['forest_audit_trail'][:closure].call( + { headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } } + ) + + expect(store).to have_received(:list_by_record).with( + collection: 'projects', record_id: [{ id: '4', until: nil, until_row: nil }], skip: 0, limit: 20, order: 'desc' + ) + expect(store).to have_received(:count_by_record) + .with(collection: 'projects', record_id: [{ id: '4', until: nil, until_row: nil }]) + # Top-level keys are camelCased for the frontend; nested value hashes keep the column names. + expect(result[:content]).to eq( + { + data: [{ 'operation' => 'update', 'recordId' => '4', 'previousValues' => { 'first_name' => 'Jo' } }], + meta: { count: 1, availableUsers: [] } + } + ) + end + + it 'intersects the record with the permission scope before reading any history' do + scope = Nodes::ConditionTreeLeaf.new('id', Operators::EQUAL, 4) + allow(permissions).to receive(:get_scope).and_return(scope) + route = route_with_store + + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) + + expect(collection).to have_received(:list) do |_caller, filter, _projection| + expect(filter.condition_tree.conditions).to include(scope) + end + end + + it 'returns 404 without touching the store when the record exists outside the caller scope' do + allow(permissions).to receive(:get_scope).and_return(Nodes::ConditionTreeLeaf.new('id', Operators::EQUAL, 9)) + # Empty under the scope, found without it: the record is someone else's, not a deleted one. + allow(collection).to receive(:list).and_return([], [{ 'id' => 4 }]) + route = route_with_store + + expect do + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) + end.to raise_error(Http::Exceptions::NotFoundError) + + expect(store).not_to have_received(:list_by_record) + end + + it 'still serves the history of a deleted record, which is much of the point of an audit trail' do + allow(permissions).to receive(:get_scope).and_return(Nodes::ConditionTreeLeaf.new('id', Operators::EQUAL, 4)) + allow(collection).to receive(:list).and_return([]) + route = route_with_store(records: [double('entry', to_h: { operation: 'delete', record_id: '4' })]) + + result = route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) + + expect(result[:content][:data]).to eq([{ 'operation' => 'delete', 'recordId' => '4' }]) + end + + # Rows written before an update moved a writable primary key stay under the id they were true of, so + # asking for the current id alone would start the story at the rename. + describe 'a record that was renamed' do + it 'reads the history of every id it has been filed under' do + route = route_with_store + allow(store).to receive(:renamed_from) + .and_return([{ id: '1', until: '2026-01-02T00:00:05.000Z', until_row: 12 }], []) + + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) + + expected = [{ id: '4', until: nil, until_row: nil }, + { id: '1', until: '2026-01-02T00:00:05.000Z', until_row: 12 }] + expect(store).to have_received(:list_by_record).with(hash_including(record_id: expected)) + expect(store).to have_received(:count_by_record).with(hash_including(record_id: expected)) + end + + # Two hops means two real bounds to compare, which is the path a single rename never reaches. + it 'carries the earlier bound down a chain of two renames' do + route = route_with_store + allow(store).to receive(:renamed_from).and_return( + [{ id: '7', until: '2026-01-02T00:00:09.000Z', until_row: 20 }], + [{ id: '1', until: '2026-01-02T00:00:05.000Z', until_row: 10 }], + [] + ) + + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '9' } }) + + expect(store).to have_received(:list_by_record).with( + hash_including(record_id: [ + { id: '9', until: nil, until_row: nil }, + { id: '7', until: '2026-01-02T00:00:09.000Z', until_row: 20 }, + { id: '1', until: '2026-01-02T00:00:05.000Z', until_row: 10 } + ]) + ) + end + + # The middle id was left later than the one before it, so the older segment keeps its own, earlier + # bound rather than inheriting the looser one. + it 'keeps the earlier of the two bounds when the chain reports a later one' do + route = route_with_store + allow(store).to receive(:renamed_from).and_return( + [{ id: '7', until: '2026-01-02T00:00:05.000Z', until_row: 10 }], + [{ id: '1', until: '2026-01-02T00:00:09.000Z', until_row: 20 }], + [] + ) + + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '9' } }) + + expect(store).to have_received(:list_by_record).with( + hash_including(record_id: [ + { id: '9', until: nil, until_row: nil }, + { id: '7', until: '2026-01-02T00:00:05.000Z', until_row: 10 }, + { id: '1', until: '2026-01-02T00:00:05.000Z', until_row: 10 } + ]) + ) + end + + it 'stops walking rather than looping on a chain that comes back to itself' do + route = route_with_store + allow(store).to receive(:renamed_from).and_return( + [{ id: '1', until: nil, until_row: nil }], [{ id: '4', until: nil, until_row: nil }] + ) + + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) + + expect(store).to have_received(:list_by_record).with( + hash_including(record_id: [{ id: '4', until: nil, until_row: nil }, + { id: '1', until: nil, until_row: nil }]) + ) + end + end + + describe 'meta.availableUsers' do + let(:authors) do + [{ user_id: 12, user_first_name: 'Ada', user_last_name: 'L', user_email: 'ada@test' }] + end + + # The distinct authors of what the filters match, whatever page was asked for, in the shape the + # filter dropdown wants. + it 'lists the authors of the matching entries on the first fetch' do + route = route_with_store + allow(store).to receive(:authors_by_record).and_return(authors) + + result = route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) + + expect(result[:content][:meta][:availableUsers]).to eq( + [{ id: 12, firstName: 'Ada', lastName: 'L', email: 'ada@test' }] + ) + expect(store).to have_received(:authors_by_record) + .with(collection: 'projects', record_id: [{ id: '4', until: nil, until_row: nil }]) + end + + # The front keeps the list it saw, so later pages leave it out. + it 'leaves it out past the first page, and does not even ask for it' do + route = route_with_store + allow(store).to receive(:authors_by_record).and_return(authors) + + result = route.handle_request( + { headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'page' => { 'number' => '2' } } } + ) + + expect(result[:content][:meta]).to eq({ count: 0 }) + expect(store).not_to have_received(:authors_by_record) + end + + it 'answers the active filters, not the whole history' do + route = route_with_store + allow(store).to receive(:authors_by_record).and_return(authors) + + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'userIds' => '12' } }) + + expect(store).to have_received(:authors_by_record).with(hash_including(user_ids: [12])) + end + end + + it 'defaults to newest-first and switches to oldest-first on sort=timestamp' do + route = route_with_store + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4', 'sort' => 'timestamp' } }) + + expect(store).to have_received(:list_by_record).with(hash_including(order: 'asc')) + end + + it 'caps page[size] at 100 and honors page[number]' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'page' => { 'size' => '500', 'number' => '3' } } }) + + expect(store).to have_received(:list_by_record).with(hash_including(skip: 200, limit: 100)) + end + + it 'falls back to the default page when page is not a hash' do + route = route_with_store + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4', 'page' => 'foo' } }) + + expect(store).to have_received(:list_by_record).with(hash_including(skip: 0, limit: 20)) + end + + it 'passes a search term through, trimmed' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'search' => ' Lyon ' } }) + + expect(store).to have_received(:list_by_record).with(hash_including(search: 'Lyon')) + # The count has to agree with the filter, like every other one. + expect(store).to have_received(:count_by_record).with(hash_including(search: 'Lyon')) + end + + it 'sends no search when the term is blank' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'search' => ' ' } }) + + expect(store).to have_received(:list_by_record).with(hash_excluding(:search)) + end + + it 'combines a search with the other filters as one AND' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'search' => 'Lyon', + 'userIds' => '12', 'fields' => 'address.city' } }) + + expect(store).to have_received(:list_by_record).with( + hash_including(search: 'Lyon', user_ids: [12], fields: ['address.city']) + ) + end + + it 'passes a fields filter through, keeping names that hold a dot' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'fields' => 'status, address.city ,' } }) + + expect(store).to have_received(:list_by_record).with(hash_including(fields: ['status', 'address.city'])) + expect(store).to have_received(:count_by_record).with(hash_including(fields: ['status', 'address.city'])) + end + + it 'sends no fields filter when the param is absent or empty' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'fields' => ' , ' } }) + + expect(store).to have_received(:list_by_record).with(hash_excluding(:fields)) + end + + it 'parses userIds, dropping non-numeric tokens' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'userIds' => '7, x ,9' } }) + + expect(store).to have_received(:list_by_record).with(hash_including(user_ids: [7, 9])) + end + + it 'parses a date range into inclusive UTC boundaries' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'startDate' => '2026-01-02', 'endDate' => '2026-01-02' } }) + + expect(store).to have_received(:list_by_record).with( + hash_including(start_timestamp: '2026-01-02T00:00:00.000Z', + end_timestamp: '2026-01-02T23:59:59.999Z') + ) + end + + it 'reads dates as local time in the request timezone' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'timezone' => 'America/New_York', 'startDate' => '2026-01-02' } }) + + # 2026-01-02 00:00 in New York (UTC-5) is 05:00 UTC. + expect(store).to have_received(:list_by_record).with(hash_including(start_timestamp: '2026-01-02T05:00:00.000Z')) + end + + it 'reads a wall-clock datetime, completing a minutes-only end boundary to :59.999' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'startDate' => '2026-01-02T08:30', 'endDate' => '2026-01-02 09:30' } }) + + expect(store).to have_received(:list_by_record).with( + hash_including(start_timestamp: '2026-01-02T08:30:00.000Z', + end_timestamp: '2026-01-02T09:30:59.999Z') + ) + end + + it 'keeps explicit seconds as given on both bounds' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'startDate' => '2026-01-02T08:30:15', + 'endDate' => '2026-01-02T09:30:45' } }) + + expect(store).to have_received(:list_by_record).with( + hash_including(start_timestamp: '2026-01-02T08:30:15.000Z', + end_timestamp: '2026-01-02T09:30:45.000Z') + ) + end + + # Right shape, impossible instant: the regex accepts it, the zone refuses to parse it. + it 'rejects a well-formed datetime that is out of range' do + route = route_with_store + + expect do + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'startDate' => '2026-01-02T99:00' } }) + end.to raise_error(Http::Exceptions::ValidationError, /Invalid date/) + end + + it 'rejects an unparsable date' do + route = route_with_store + + expect do + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'startDate' => 'nope' } }) + end.to raise_error(Http::Exceptions::ValidationError, /Invalid date/) + end + + it 'rejects an unknown timezone' do + route = route_with_store + + expect do + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'timezone' => 'Mars/Phobos', 'startDate' => '2026-01-02' } }) + end.to raise_error(Http::Exceptions::ValidationError, /Invalid timezone/) + end + end + end + end +end diff --git a/packages/forest_admin_agent/spec/spec_helper.rb b/packages/forest_admin_agent/spec/spec_helper.rb index d7148dd8c..91a64768f 100644 --- a/packages/forest_admin_agent/spec/spec_helper.rb +++ b/packages/forest_admin_agent/spec/spec_helper.rb @@ -1,4 +1,6 @@ require 'filecache' +require 'active_record' +require 'sqlite3' require 'simplecov' require 'simplecov_json_formatter' require 'simplecov-html' @@ -37,6 +39,10 @@ config.include ForestAdminTestToolkit::Factory::Column config.before do + # The audit connection is class-level, so it is handed back between examples: a store pointed at another + # database would otherwise be refused. + ForestAdminAgent::AuditTrail::Sql::AuditConnectionBase.disconnect! + cache = FileCache.new('app', 'tmp/cache/forest_admin') cache.clear diff --git a/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/collection_customizer.rb b/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/collection_customizer.rb index c4d3ee68d..7cdab3d46 100644 --- a/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/collection_customizer.rb +++ b/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/collection_customizer.rb @@ -263,8 +263,10 @@ def add_chart(name, &definition) # .add_hook('before', 'list') do |context| # # Do something before the list action # end - def add_hook(position, type, &handler) - push_customization { @stack.hook.get_collection(@name).add_hook(position, type, handler) } + def add_hook(position, type, prepend: false, &handler) + push_customization do + @stack.hook.get_collection(@name).add_hook(position, type, handler, prepend: prepend) + end end # Add a new segment on the collection. diff --git a/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hook_collection_decorator.rb b/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hook_collection_decorator.rb index f12455a27..9e47ebd9e 100644 --- a/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hook_collection_decorator.rb +++ b/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hook_collection_decorator.rb @@ -18,8 +18,8 @@ def initialize(child_collection, datasource) } end - def add_hook(position, type, hook) - @hooks[type].add_handler(position, hook) + def add_hook(position, type, hook, prepend: false) + @hooks[type].add_handler(position, hook, prepend: prepend) end def create(caller, data) diff --git a/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hooks.rb b/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hooks.rb index fc644bb5a..a249bf48e 100644 --- a/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hooks.rb +++ b/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hooks.rb @@ -17,8 +17,13 @@ def execute_after(context) @after.each { |hook| hook.call(context) } end - def add_handler(position, hook) - position == 'After' ? @after << hook : @before << hook + # `prepend` puts the handler ahead of the ones already registered. `execute_after` stops at the + # first exception, so a handler that must run whatever a sibling does (the audit trail recording a + # write that already happened) cannot afford to be last. + def add_handler(position, hook, prepend: false) + handlers = position == 'After' ? @after : @before + + prepend ? handlers.unshift(hook) : handlers.push(hook) end end end diff --git a/packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/decorators/hook/hooks_spec.rb b/packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/decorators/hook/hooks_spec.rb index a053293e4..d3f0af571 100644 --- a/packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/decorators/hook/hooks_spec.rb +++ b/packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/decorators/hook/hooks_spec.rb @@ -80,6 +80,32 @@ def initialize end end + describe 'add_handler with prepend' do + it 'runs the prepended handler before the ones already registered' do + calls = [] + + hooks = described_class.new + hooks.add_handler('After', proc { calls << :existing }) + hooks.add_handler('After', proc { calls << :prepended }, prepend: true) + hooks.execute_after(fake_hook_context.new) + + expect(calls).to eq(%i[prepended existing]) + end + + # execute_after stops at the first exception: a handler that must run whatever a sibling does + # cannot afford to be last. + it 'runs the prepended handler even when a later one raises' do + calls = [] + + hooks = described_class.new + hooks.add_handler('After', proc { raise 'boom' }) + hooks.add_handler('After', proc { calls << :prepended }, prepend: true) + + expect { hooks.execute_after(fake_hook_context.new) }.to raise_error('boom') + expect(calls).to eq([:prepended]) + end + end + describe 'execute_after' do describe 'when multiple after hooks are defined' do it 'call all of them' do diff --git a/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/components/caller.rb b/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/components/caller.rb index 183e6b5ea..1d6cad012 100644 --- a/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/components/caller.rb +++ b/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/components/caller.rb @@ -1,7 +1,8 @@ module ForestAdminDatasourceToolkit module Components class Caller - attr_reader :id, :email, :first_name, :last_name, :tags, :team, :rendering_id, :timezone, :permission_level, :role + attr_reader :id, :email, :first_name, :last_name, :tags, :team, :rendering_id, :timezone, + :permission_level, :role, :request, :request_id def initialize( id:, @@ -15,6 +16,7 @@ def initialize( permission_level:, role: nil, request: {}, + request_id: nil, project: nil, environment: nil, **_extra_args @@ -30,6 +32,7 @@ def initialize( @permission_level = permission_level @role = role @request = request + @request_id = request_id @project = project @environment = environment end diff --git a/packages/forest_admin_rails/lib/forest_admin_rails.rb b/packages/forest_admin_rails/lib/forest_admin_rails.rb index 2606f91be..8d91b5c9b 100644 --- a/packages/forest_admin_rails/lib/forest_admin_rails.rb +++ b/packages/forest_admin_rails/lib/forest_admin_rails.rb @@ -34,6 +34,9 @@ module ForestAdminRails setting :disable_route_cache, default: false setting :rpc_max_polling_threads, default: nil setting :workflow_executor_url, default: nil + # { database: , schema:, table_name:, redact: } — setting `database` + # turns the audit trail on: every change is captured and the `/_audit-trail` routes are registered. + setting :audit_trail, default: nil if defined?(Rails::Railtie) # logic for cors middleware,... here // or it might be into Engine diff --git a/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb b/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb index 829667caf..463c50baf 100644 --- a/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb +++ b/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb @@ -34,6 +34,27 @@ class Engine < ::Rails::Engine agent_factory.setup(ForestAdminRails.config) load_configuration load_cors + load_correlation_id + end + end + + # Echo the agent-generated correlation id on every response (mirrors the Node agent's + # router.use(correlationIdMiddleware)); CORS exposure of the header is handled in load_cors. + # + # Ahead of ShowExceptions rather than at the end of the stack: a Rack middleware can only add a header + # to a response it sees returned, so sitting under the exception handlers meant error responses they + # build on the way out never carried the id. + def load_correlation_id + middleware = ForestAdminAgent::Http::CorrelationIdMiddleware + + begin + # The application's stack, not the engine's: ShowExceptions lives there, and an + # insert_before recorded on the engine's own (empty) proxy raises when it is applied. + Rails.application.config.middleware.insert_before ActionDispatch::ShowExceptions, middleware + rescue StandardError + # No ShowExceptions in this stack (or already gone): the id still reaches every response the app + # returns normally. + config.middleware.use middleware end end @@ -102,7 +123,8 @@ def load_cors hostnames += ENV['CORS_ORIGINS'].split(',') if ENV['CORS_ORIGINS'] origins hostnames - resource '*', headers: :any, methods: :any, credentials: true, max_age: 86_400 + resource '*', headers: :any, methods: :any, credentials: true, max_age: 86_400, + expose: [ForestAdminAgent::Http::CorrelationId::HEADER] end end end diff --git a/packages/forest_admin_rails/spec/lib/forest_admin_rails/engine_spec.rb b/packages/forest_admin_rails/spec/lib/forest_admin_rails/engine_spec.rb index 33b0906ce..a048fb76e 100644 --- a/packages/forest_admin_rails/spec/lib/forest_admin_rails/engine_spec.rb +++ b/packages/forest_admin_rails/spec/lib/forest_admin_rails/engine_spec.rb @@ -272,6 +272,51 @@ def self.setup!; end end end + RSpec.describe 'load_correlation_id' do + let(:engine_class) { ForestAdminRails::Engine } + let(:engine_instance) { engine_class.allocate } + # rubocop:disable RSpec/VerifiedDoubles + let(:engine_middleware) { double('engine middleware') } + let(:app_middleware) { double('application middleware') } + let(:application) { double('application', config: double('app config', middleware: app_middleware)) } + let(:engine_config) { double('engine config', middleware: engine_middleware) } + # rubocop:enable RSpec/VerifiedDoubles + + before do + allow(engine_instance).to receive(:config).and_return(engine_config) + allow(Rails).to receive(:application).and_return(application) + end + + # The application's stack, not the engine's: ShowExceptions lives there, and under the exception + # handlers the 500s they build would never carry the header. + it 'inserts the middleware ahead of the exception handlers of the application stack' do + allow(app_middleware).to receive(:insert_before) + + engine_instance.load_correlation_id + + expect(app_middleware).to have_received(:insert_before) + .with(ActionDispatch::ShowExceptions, ForestAdminAgent::Http::CorrelationIdMiddleware) + end + + it 'falls back to appending when the stack has no exception handler to insert before' do + allow(app_middleware).to receive(:insert_before).and_raise(RuntimeError, 'no such middleware') + allow(engine_middleware).to receive(:use) + + engine_instance.load_correlation_id + + expect(engine_middleware).to have_received(:use).with(ForestAdminAgent::Http::CorrelationIdMiddleware) + end + + it 'falls back to appending when there is no application to reach' do + allow(Rails).to receive(:application).and_return(nil) + allow(engine_middleware).to receive(:use) + + engine_instance.load_correlation_id + + expect(engine_middleware).to have_received(:use).with(ForestAdminAgent::Http::CorrelationIdMiddleware) + end + end + RSpec.describe 'Engine autoload behaviour' do it 'does not register an initializer that adds the host lib/ to autoload_paths' do initializer_names = ForestAdminRails::Engine.initializers.map(&:name) diff --git a/packages/forest_admin_rpc_agent/lib/forest_admin_rpc_agent.rb b/packages/forest_admin_rpc_agent/lib/forest_admin_rpc_agent.rb index 99a0cc1ed..9f941922f 100644 --- a/packages/forest_admin_rpc_agent/lib/forest_admin_rpc_agent.rb +++ b/packages/forest_admin_rpc_agent/lib/forest_admin_rpc_agent.rb @@ -28,6 +28,7 @@ module ForestAdminRpcAgent setting :customize_error_message, default: nil setting :disable_route_cache, default: false setting :rpc_max_polling_threads, default: nil + setting :audit_trail, default: nil begin require 'thor'