From c04359abd93afe18c6343ff69eeb5e0996032f54 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Thu, 18 Jun 2026 17:31:07 +0200 Subject: [PATCH 01/35] feat(audit_trail): add audit trail plugin gem Introduce the forest_admin_audit_trail package: a datasource-agnostic plugin that captures who changed what (before/after diff) for every change Forest performs through its data layer, with pluggable storage (in-memory, log, SQL). Supporting agent/rails wiring: - correlation id generated per request, propagated to the caller as request_id and echoed back via response header (CorrelationIdMiddleware) - record-history route (/_audit-trail/:collection/:id) reading from a configurable store - register the gem in the semantic-release pipeline (.releaserc.js) and exclude its version.rb from rubocop, matching the other packages Co-Authored-By: Claude Opus 4.8 --- .releaserc.js | 7 +- .rubocop.yml | 9 + .../forest_admin_agent/http/correlation_id.rb | 37 ++++ .../http/correlation_id_middleware.rb | 29 ++++ .../lib/forest_admin_agent/http/router.rb | 1 + .../routes/resources/audit_trail.rb | 147 ++++++++++++++++ .../forest_admin_agent/utils/caller_parser.rb | 4 + .../http/correlation_id_middleware_spec.rb | 46 +++++ .../http/correlation_id_spec.rb | 29 ++++ .../routes/resources/audit_trail_spec.rb | 120 +++++++++++++ packages/forest_admin_audit_trail/.gitignore | 7 + packages/forest_admin_audit_trail/.rspec | 3 + packages/forest_admin_audit_trail/Gemfile | 20 +++ packages/forest_admin_audit_trail/README.md | 79 +++++++++ packages/forest_admin_audit_trail/Rakefile | 10 ++ .../forest_admin_audit_trail.gemspec | 37 ++++ .../lib/forest_admin_audit_trail.rb | 10 ++ .../forest_admin_audit_trail/audit_record.rb | 45 +++++ .../lib/forest_admin_audit_trail/diff.rb | 81 +++++++++ .../lib/forest_admin_audit_trail/plugin.rb | 163 ++++++++++++++++++ .../sql/audit_connection_base.rb | 11 ++ .../forest_admin_audit_trail/sql/audit_log.rb | 11 ++ .../forest_admin_audit_trail/sql/migrator.rb | 118 +++++++++++++ .../stores/in_memory_store.rb | 52 ++++++ .../stores/log_store.rb | 26 +++ .../stores/sql_store.rb | 116 +++++++++++++ .../lib/forest_admin_audit_trail/version.rb | 3 + .../lib/forest_admin_audit_trail/diff_spec.rb | 52 ++++++ .../forest_admin_audit_trail/plugin_spec.rb | 121 +++++++++++++ .../stores/in_memory_store_spec.rb | 81 +++++++++ .../stores/sql_store_spec.rb | 105 +++++++++++ .../spec/spec_helper.rb | 25 +++ .../components/caller.rb | 5 +- .../lib/forest_admin_rails.rb | 3 + .../lib/forest_admin_rails/engine.rb | 10 +- 35 files changed, 1619 insertions(+), 4 deletions(-) create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/http/correlation_id.rb create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/http/correlation_id_middleware.rb create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb create mode 100644 packages/forest_admin_agent/spec/lib/forest_admin_agent/http/correlation_id_middleware_spec.rb create mode 100644 packages/forest_admin_agent/spec/lib/forest_admin_agent/http/correlation_id_spec.rb create mode 100644 packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb create mode 100644 packages/forest_admin_audit_trail/.gitignore create mode 100644 packages/forest_admin_audit_trail/.rspec create mode 100644 packages/forest_admin_audit_trail/Gemfile create mode 100644 packages/forest_admin_audit_trail/README.md create mode 100644 packages/forest_admin_audit_trail/Rakefile create mode 100644 packages/forest_admin_audit_trail/forest_admin_audit_trail.gemspec create mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail.rb create mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/audit_record.rb create mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/diff.rb create mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/plugin.rb create mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_connection_base.rb create mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_log.rb create mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb create mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/in_memory_store.rb create mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/log_store.rb create mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb create mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb create mode 100644 packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/diff_spec.rb create mode 100644 packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/plugin_spec.rb create mode 100644 packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/in_memory_store_spec.rb create mode 100644 packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb create mode 100644 packages/forest_admin_audit_trail/spec/spec_helper.rb diff --git a/.releaserc.js b/.releaserc.js index 76e57c8f0..920227235 100644 --- a/.releaserc.js +++ b/.releaserc.js @@ -31,7 +31,8 @@ module.exports = { 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb; '+ 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb; '+ 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb; '+ - 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb; ', + 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb; '+ + 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb; ', successCmd: '( cd packages/forest_admin_agent && gem build && gem push forest_admin_agent-*.gem );' + '( cd packages/forest_admin_datasource_active_record && gem build && gem push forest_admin_datasource_active_record-*.gem );' + @@ -45,7 +46,8 @@ module.exports = { '( cd packages/forest_admin_datasource_zendesk && gem build && gem push forest_admin_datasource_zendesk-*.gem );' + '( cd packages/forest_admin_datasource_snowflake && gem build && gem push forest_admin_datasource_snowflake-*.gem );' + '( cd packages/forest_admin_datasource_mambu_payments && gem build && gem push forest_admin_datasource_mambu_payments-*.gem );' + - '( cd packages/forest_admin_datasource_graphql_hasura && gem build && gem push forest_admin_datasource_graphql_hasura-*.gem );' , + '( cd packages/forest_admin_datasource_graphql_hasura && gem build && gem push forest_admin_datasource_graphql_hasura-*.gem );' + + '( cd packages/forest_admin_audit_trail && gem build && gem push forest_admin_audit_trail-*.gem );' , }, ], [ @@ -68,6 +70,7 @@ module.exports = { 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb', 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb', 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb', + 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb', 'package.json' ], }, diff --git a/.rubocop.yml b/.rubocop.yml index 6458bc528..d309d90d2 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -133,6 +133,7 @@ Style/MutableConstant: - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb' - 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb' + - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb' # Offense count: 38 # This cop supports safe autocorrection (--autocorrect). @@ -217,6 +218,7 @@ Style/StringLiterals: - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb' - 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb' + - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb' # Offense count: 1 # This cop supports safe autocorrection (--autocorrect). @@ -258,6 +260,10 @@ Naming/PredicatePrefix: Metrics/ParameterLists: Exclude: - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/configuration.rb' + - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/audit_record.rb' + - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/plugin.rb' + - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/in_memory_store.rb' + - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb' - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/collections/base_collection.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/datasource.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/routes/query_handler.rb' @@ -362,6 +368,7 @@ Metrics/ClassLength: - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/introspection/schema_converter.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/query/query_builder.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/query/aggregator.rb' + - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/plugin.rb' - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/collections/base_collection.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/collection.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/datasource.rb' @@ -454,6 +461,8 @@ Layout/LineLength: RSpec/VerifiedDoubles: Exclude: + - 'packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/plugin_spec.rb' + - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb' - 'packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/composite_datasource_spec.rb' RSpec/VerifiedDoubleReference: diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/http/correlation_id.rb b/packages/forest_admin_agent/lib/forest_admin_agent/http/correlation_id.rb new file mode 100644 index 000000000..ded3da334 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/http/correlation_id.rb @@ -0,0 +1,37 @@ +require 'securerandom' + +module ForestAdminAgent + module Http + # Per-request correlation id, generated by the agent. Shared between the caller (so the audit + # trail can group every change made within one request) and the response header echoed back to + # the client. Mirrors the Node agent's `context.state.requestId` + `x-forest-correlation-id` + # header: the agent generates the id, never reads it from the incoming request. + # + # Stored thread-locally and generated lazily on first read (e.g. when the caller is parsed). The + # host resets it at the start of each request so a pooled thread never reuses a previous id. + module CorrelationId + HEADER = 'x-forest-correlation-id'.freeze + KEY = :forest_admin_correlation_id + + module_function + + # Lazily generate and memoize the id for the current request/thread. + def current + Thread.current[KEY] ||= SecureRandom.uuid + end + + # The id if one was generated during this request, otherwise nil (does not generate one). + def current? + Thread.current[KEY] + end + + def current=(value) + Thread.current[KEY] = value + end + + def reset! + Thread.current[KEY] = nil + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/http/correlation_id_middleware.rb b/packages/forest_admin_agent/lib/forest_admin_agent/http/correlation_id_middleware.rb new file mode 100644 index 000000000..fbd6c6e99 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/http/correlation_id_middleware.rb @@ -0,0 +1,29 @@ +module ForestAdminAgent + module Http + # Rack middleware echoing the agent-generated correlation id back to the client, mirroring the + # Node agent's `correlationIdMiddleware` (`router.use(...)`). Hosts mount it in their middleware + # stack; CORS exposure of the header is handled by the host's CORS config (see the Rails engine). + # + # The id itself is generated lazily by the agent during the request (see CorrelationId, called + # from CallerParser). This middleware only resets the thread-local around the request — so a + # pooled thread never reuses a previous id — and sets the response header when one was generated. + class CorrelationIdMiddleware + def initialize(app) + @app = app + end + + def call(env) + CorrelationId.reset! + + status, headers, body = @app.call(env) + + id = CorrelationId.current? + headers[CorrelationId::HEADER] = id if id + + [status, headers, body] + ensure + CorrelationId.reset! + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb b/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb index 4bd0b896f..5811efdc9 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb @@ -64,6 +64,7 @@ def self.routes { name: 'dissociate_related', handler: -> { Resources::Related::DissociateRelated.new.routes } }, { name: 'update_related', handler: -> { Resources::Related::UpdateRelated.new.routes } }, { name: 'update_field', handler: -> { Resources::UpdateField.new.routes } }, + { name: 'audit_trail', handler: -> { Resources::AuditTrail.new.routes } }, { name: 'workflow_executor_proxy', handler: -> { Workflow::WorkflowExecutorProxy.new.routes } } ] diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb new file mode 100644 index 000000000..221e55db0 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb @@ -0,0 +1,147 @@ +require 'active_support/time' + +module ForestAdminAgent + module Routes + module Resources + # Record-history route, mirroring the Node agent's `/_audit-trail/{collection}/:id`. + # + # Registered only when an `audit_trail` option carrying a readable store was provided to the + # agent. The store is expected to respond to `list_by_record` and `count_by_record`. + class AuditTrail < AbstractAuthenticatedRoute + include ForestAdminAgent::Utils + + DEFAULT_PAGE_SIZE = 20 + MAX_PAGE_SIZE = 100 + DATE_ONLY = /\A\d{4}-\d{2}-\d{2}\z/ + # Wall-clock datetime, `T` or space separator, seconds optional: `YYYY-MM-DD[T ]HH:mm[:ss]`. + DATE_TIME = /\A(\d{4}-\d{2}-\d{2})[T ](\d{2}):(\d{2})(?::(\d{2}))?\z/ + + def setup_routes + return self unless store + + add_route( + 'forest_audit_trail', + 'get', + '/_audit-trail/:collection_name/:id', + ->(args) { handle_request(args) } + ) + + self + end + + def handle_request(args = {}) + context = build(args) + context.permissions.can?(:read, context.collection) + + skip, limit = parse_pagination(args) + filters = { + collection: context.collection.name, + # args[:params]['id'] is already Forest's packed id, the form the audit store keys on. + record_id: args[:params]['id'], + **parse_filters(args) + } + + history = store.list_by_record(**filters, skip: skip, limit: limit, order: parse_sort(args)) + # `count` reflects the active filters (not the absolute total) and is independent of the page. + count = store.count_by_record(**filters) + + { + name: args[:params]['collection_name'], + content: { data: history.map { |record| serialize(record) }, meta: { count: count } } + } + end + + private + + # Camelize only the top-level keys; previous/new value hashes keep the record's own column names. + def serialize(record) + record.to_h.transform_keys { |key| key.to_s.camelize(:lower) } + end + + # JSON:API `sort`: `timestamp` → oldest first, anything else (absent/unsupported) → newest first. + def parse_sort(args) + args.dig(:params, 'sort').to_s == 'timestamp' ? 'asc' : 'desc' + end + + # JSON:API pagination: 1-based page[number] (default 1) and page[size] (default 20, capped at + # 100). Out-of-bound or non-numeric values fall back to the defaults rather than erroring. + def parse_pagination(args) + size = args.dig(:params, 'page', 'size').to_i + size = DEFAULT_PAGE_SIZE if size < 1 + size = MAX_PAGE_SIZE if size > MAX_PAGE_SIZE + + number = args.dig(:params, 'page', 'number').to_i + number = 1 if number < 1 + + [(number - 1) * size, size] + end + + def parse_filters(args) + timezone = args.dig(:params, 'timezone').to_s + timezone = 'UTC' if timezone.empty? + + { + user_ids: parse_user_ids(args.dig(:params, 'userIds')), + start_timestamp: parse_date_boundary(args.dig(:params, 'startDate'), timezone, :start), + end_timestamp: parse_date_boundary(args.dig(:params, 'endDate'), timezone, :end) + }.compact + end + + # Comma-separated integer ids; non-numeric tokens are dropped. Empty after parsing → no filter. + def parse_user_ids(raw) + return nil if raw.nil? || raw.to_s.empty? + + ids = raw.to_s.split(',').map(&:strip).grep(/\A\d+\z/).map(&:to_i) + ids.empty? ? nil : ids + end + + # `startDate`/`endDate` accept a bare day (`YYYY-MM-DD`) or a wall-clock datetime + # (`YYYY-MM-DD[T ]HH:mm[:ss]`), read as local time in the request timezone and returned as a UTC + # ISO instant the store can compare against stored timestamps. + def parse_date_boundary(raw, timezone, boundary) + return nil if raw.nil? || raw.to_s.empty? + + zone = Time.find_zone(timezone) + raise Http::Exceptions::ValidationError, "Invalid timezone: \"#{timezone}\"" if zone.nil? + + instant = begin + local_instant(zone, raw.to_s, boundary) + rescue ArgumentError + nil + end + + if instant.nil? + raise Http::Exceptions::ValidationError, + "Invalid date: \"#{raw}\" (expected YYYY-MM-DD or YYYY-MM-DDTHH:mm)" + end + + instant.utc.iso8601(3) + end + + def local_instant(zone, raw, boundary) + if DATE_ONLY.match?(raw) + day = zone.parse(raw) + # Bare day → start (00:00:00.000) or end (23:59:59.999) of that local day. + boundary == :end ? day.end_of_day : day.beginning_of_day + elsif (match = DATE_TIME.match(raw)) + date, hours, minutes, seconds = match.captures + base = zone.parse("#{date}T#{hours}:#{minutes}") + if seconds + base.change(sec: seconds.to_i, usec: 0) + elsif boundary == :end + # Minutes-only end boundary stays inclusive to :59.999; start stays at :00.000. + base.change(sec: 59, usec: 999_000) + else + base + end + end + end + + def store + config = ForestAdminAgent::Facades::Container.config_from_cache + config && config[:audit_trail] && config[:audit_trail][:store] + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/utils/caller_parser.rb b/packages/forest_admin_agent/lib/forest_admin_agent/utils/caller_parser.rb index 1fea51906..cf373e77e 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/utils/caller_parser.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/utils/caller_parser.rb @@ -18,6 +18,10 @@ def parse @token_data = decode_token @token_data[:timezone] = extract_timezone @token_data[:request] = { ip: @args[:headers]['action_dispatch.remote_ip'].to_s } + # One id per request, generated by the agent, shared by every operation it triggers (used to + # correlate the audit trail) and echoed back to the client in the response header. + # See ForestAdminAgent::Http::CorrelationId. + @token_data[:request_id] = Http::CorrelationId.current project, environment = extract_forest_context @token_data[:project] = project @token_data[:environment] = environment diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/correlation_id_middleware_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/correlation_id_middleware_spec.rb new file mode 100644 index 000000000..474329e1c --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/correlation_id_middleware_spec.rb @@ -0,0 +1,46 @@ +require 'spec_helper' + +module ForestAdminAgent + module Http + describe CorrelationIdMiddleware do + after { CorrelationId.reset! } + + it 'echoes the id generated during the request on the response header' do + app = ->(_env) { [200, {}, [CorrelationId.current]] } + + _status, headers, body = described_class.new(app).call({}) + + expect(headers[CorrelationId::HEADER]).to eq(body.first) + end + + it 'does not set the header when no id was generated during the request' do + app = ->(_env) { [200, {}, ['ok']] } + + _status, headers, = described_class.new(app).call({}) + + expect(headers).not_to have_key(CorrelationId::HEADER) + end + + it 'resets any leaked id before handling the request' do + CorrelationId.current = 'stale' + seen = 'unset' + app = lambda do |_env| + seen = CorrelationId.current? + [200, {}, ['ok']] + end + + described_class.new(app).call({}) + + expect(seen).to be_nil + end + + it 'clears the id after the request so the thread is not reused with a stale id' do + app = ->(_env) { [200, {}, [CorrelationId.current]] } + + described_class.new(app).call({}) + + expect(CorrelationId.current?).to be_nil + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/correlation_id_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/correlation_id_spec.rb new file mode 100644 index 000000000..6809ce09f --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/correlation_id_spec.rb @@ -0,0 +1,29 @@ +require 'spec_helper' + +module ForestAdminAgent + module Http + describe CorrelationId do + after { described_class.reset! } + + it 'lazily generates and memoizes an id within the thread' do + id = described_class.current + + expect(id).to match(/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/) + expect(described_class.current).to eq(id) + end + + it 'can be seeded by the host' do + described_class.current = 'req-1' + + expect(described_class.current).to eq('req-1') + end + + it 'reset! clears it so a fresh id is generated next' do + first = described_class.current + described_class.reset! + + expect(described_class.current).not_to eq(first) + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb new file mode 100644 index 000000000..753e31c95 --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -0,0 +1,120 @@ +require 'spec_helper' + +module ForestAdminAgent + module Routes + module Resources + describe AuditTrail do + let(:store) { double('store') } + + def route_with_store(records: []) + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: store } }) + allow(store).to receive_messages(list_by_record: records, count_by_record: records.length) + + route = described_class.new + context = double('context', collection: double('collection', name: 'projects'), + permissions: double('permissions', can?: true)) + allow(route).to receive(:build).and_return(context) + route + end + + it 'registers the record-history route when an audit_trail store is configured' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: Object.new } }) + + expect(described_class.new.routes).to include('forest_audit_trail') + end + + it 'does not register the route when no audit_trail store is configured' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache).and_return({}) + + expect(described_class.new.routes).not_to include('forest_audit_trail') + end + + it 'reads the history scoped to the packed id and returns data + filtered count' do + entry = { operation: 'update', record_id: '4', previous_values: { 'first_name' => 'Jo' } } + route = route_with_store(records: [double('entry', to_h: entry)]) + + result = route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) + + expect(store).to have_received(:list_by_record).with( + collection: 'projects', record_id: '4', skip: 0, limit: 20, order: 'desc' + ) + expect(store).to have_received(:count_by_record).with(collection: 'projects', record_id: '4') + # Top-level keys are camelCased for the frontend; nested value hashes keep the column names. + expect(result[:content]).to eq( + { + data: [{ 'operation' => 'update', 'recordId' => '4', 'previousValues' => { 'first_name' => 'Jo' } }], + meta: { count: 1 } + } + ) + end + + it 'defaults to newest-first and switches to oldest-first on sort=timestamp' do + route = route_with_store + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4', 'sort' => 'timestamp' } }) + + expect(store).to have_received(:list_by_record).with(hash_including(order: 'asc')) + end + + it 'caps page[size] at 100 and honors page[number]' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'page' => { 'size' => '500', 'number' => '3' } } }) + + expect(store).to have_received(:list_by_record).with(hash_including(skip: 200, limit: 100)) + end + + it 'parses userIds, dropping non-numeric tokens' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'userIds' => '7, x ,9' } }) + + expect(store).to have_received(:list_by_record).with(hash_including(user_ids: [7, 9])) + end + + it 'parses a date range into inclusive UTC boundaries' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'startDate' => '2026-01-02', 'endDate' => '2026-01-02' } }) + + expect(store).to have_received(:list_by_record).with( + hash_including(start_timestamp: '2026-01-02T00:00:00.000Z', + end_timestamp: '2026-01-02T23:59:59.999Z') + ) + end + + it 'reads dates as local time in the request timezone' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'timezone' => 'America/New_York', 'startDate' => '2026-01-02' } }) + + # 2026-01-02 00:00 in New York (UTC-5) is 05:00 UTC. + expect(store).to have_received(:list_by_record).with(hash_including(start_timestamp: '2026-01-02T05:00:00.000Z')) + end + + it 'rejects an unparsable date' do + route = route_with_store + + expect do + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'startDate' => 'nope' } }) + end.to raise_error(Http::Exceptions::ValidationError, /Invalid date/) + end + + it 'rejects an unknown timezone' do + route = route_with_store + + expect do + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'timezone' => 'Mars/Phobos', 'startDate' => '2026-01-02' } }) + end.to raise_error(Http::Exceptions::ValidationError, /Invalid timezone/) + end + end + end + end +end diff --git a/packages/forest_admin_audit_trail/.gitignore b/packages/forest_admin_audit_trail/.gitignore new file mode 100644 index 000000000..d5402bb42 --- /dev/null +++ b/packages/forest_admin_audit_trail/.gitignore @@ -0,0 +1,7 @@ +/.bundle/ +/coverage/ +/pkg/ +/tmp/ +Gemfile.lock +*.gem +*.sqlite3 diff --git a/packages/forest_admin_audit_trail/.rspec b/packages/forest_admin_audit_trail/.rspec new file mode 100644 index 000000000..34c5164d9 --- /dev/null +++ b/packages/forest_admin_audit_trail/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/packages/forest_admin_audit_trail/Gemfile b/packages/forest_admin_audit_trail/Gemfile new file mode 100644 index 000000000..9ac9e586e --- /dev/null +++ b/packages/forest_admin_audit_trail/Gemfile @@ -0,0 +1,20 @@ +source 'https://rubygems.org' + +gemspec + +gem 'forest_admin_datasource_customizer', path: '../forest_admin_datasource_customizer' +gem 'forest_admin_datasource_toolkit', path: '../forest_admin_datasource_toolkit' +gem 'rake', '~> 13.0' +gem 'rubocop', '1.86.1' +gem 'rubocop-performance', '1.26.1' +gem 'rubocop-rspec', '3.9.0' + +group :development, :test do + gem 'forest_admin_agent', path: '../forest_admin_agent' + gem 'forest_admin_test_toolkit', path: '../forest_admin_test_toolkit' + gem 'rspec', '~> 3.0' + gem 'simplecov', '~> 0.22', require: false + gem 'simplecov-html', '~> 0.12.3' + gem 'simplecov_json_formatter', '~> 0.1.4' + gem 'sqlite3', '>= 2.1' +end diff --git a/packages/forest_admin_audit_trail/README.md b/packages/forest_admin_audit_trail/README.md new file mode 100644 index 000000000..de48e9901 --- /dev/null +++ b/packages/forest_admin_audit_trail/README.md @@ -0,0 +1,79 @@ +# forest_admin_audit_trail + +Capture who changed what (before/after) for every change Forest performs through its data layer, and +persist it into a SQL database. Ruby port of the Node `@forestadmin/plugin-audit-trail`. + +Two decoupled parts: + +- **Capture** — datasource-agnostic. It instruments every collection through the Forest customizer + hooks, so it works the same whether the audited datasource is ActiveRecord, Mongoid, etc. +- **Storage** — where the records are written. Ships a **SQL store** (ActiveRecord) that creates the + `forest` schema and creates/evolves the `audit_logs` table through versioned migrations, plus an + in-memory store for tests. + +## Use it in an agent + +`ForestAdminAuditTrail::Stores::SqlStore` both **writes** every audited change and **reads** the +per-record history back. Pass the **same** instance to the plugin (so writes land) and to the agent +`audit_trail` option (so the record-history route is exposed and reads from the same table). + +```ruby +store = ForestAdminAuditTrail::Stores::SqlStore.new( + connection_string: ENV['AUDIT_TRAIL_DATABASE_URL'] # Postgres URL, or any ActiveRecord config +) + +ForestAdminAgent::Builder::AgentFactory.instance.setup( + auth_secret: ENV['FOREST_AUTH_SECRET'], + env_secret: ENV['FOREST_ENV_SECRET'], + # ...usual options... + audit_trail: { store: store } # exposes GET /forest/_audit-trail/{collection}/:id +) + +agent.use(ForestAdminAuditTrail::Plugin, store: store) +``` + +On the first write or read the store ensures the `forest` schema exists and runs any pending +migrations to create/upgrade `forest.audit_logs`; every create / update / delete performed through +Forest then writes one row per record. + +## What gets stored + +`forest.audit_logs`, one row per audited change: + +| column | description | +| ----------------- | ---------------------------------------------------- | +| `id` | auto-increment primary key | +| `timestamp` | when the change happened | +| `operation` | `create` / `update` / `delete` | +| `collection` | audited collection name | +| `record_id` | packed record id (primary keys joined by `\|`) | +| `user_id` | the Forest user who made the change | +| `correlation_key` | per-request id; groups every change made within one request | +| `previous_values` | values before the change (JSON) | +| `new_values` | values after the change (JSON) | + +`previous_values` / `new_values` store **only the parts that actually changed**: nested hashes and +arrays of hashes are diffed structurally, so a single sub-field change records just that leaf. + +The `correlation_key` is the agent's per-request id (`caller.request_id`), generated by the agent and +echoed back to the client in the `X-Forest-Correlation-Id` response header — so every change made in +one request shares a key, and the caller can tie it to its own activity log. + +## Options + +`ForestAdminAuditTrail::Plugin` options: + +| option | description | +| -------- | ------------------------------------------------------------------------------------ | +| `store` | object responding to `append(record)`. Defaults to a store that only logs. | +| `redact` | `{ 'collection_name' => ['field', ...] }` — values masked while recording the change | + +`SqlStore.new` options: `connection_string:` (required, AR URL or config hash), `schema:` +(default `forest`), `table_name:` (default `audit_logs`). + +## Schema migrations & concurrency + +The table is created/evolved through an ordered, append-only migration list tracked in a dedicated +`forest.audit_migrations` table. On Postgres the migrations run inside a transaction-scoped advisory +lock so several agents booting at once apply them one after another; the schema is created (and +committed, idempotently) first since the lock can't cover a not-yet-existing schema. diff --git a/packages/forest_admin_audit_trail/Rakefile b/packages/forest_admin_audit_trail/Rakefile new file mode 100644 index 000000000..85b387478 --- /dev/null +++ b/packages/forest_admin_audit_trail/Rakefile @@ -0,0 +1,10 @@ +require 'bundler/gem_tasks' +require 'rspec/core/rake_task' + +RSpec::Core::RakeTask.new(:spec) + +require 'rubocop/rake_task' + +RuboCop::RakeTask.new + +task default: %i[spec rubocop] diff --git a/packages/forest_admin_audit_trail/forest_admin_audit_trail.gemspec b/packages/forest_admin_audit_trail/forest_admin_audit_trail.gemspec new file mode 100644 index 000000000..b69391b4f --- /dev/null +++ b/packages/forest_admin_audit_trail/forest_admin_audit_trail.gemspec @@ -0,0 +1,37 @@ +lib = File.expand_path('lib', __dir__) +$LOAD_PATH.unshift lib unless $LOAD_PATH.include?(lib) + +require_relative 'lib/forest_admin_audit_trail/version' + +Gem::Specification.new do |spec| + spec.name = 'forest_admin_audit_trail' + spec.version = ForestAdminAuditTrail::VERSION + spec.authors = ['Forest Admin'] + spec.email = ['contact@forestadmin.com'] + spec.homepage = 'https://www.forestadmin.com' + spec.summary = 'Audit-trail plugin for the Forest Admin Ruby agent.' + spec.description = 'Capture who changed what (before/after) for every change Forest performs through its data ' \ + 'layer, and persist it into a SQL database. Datasource-agnostic capture, decoupled SQL storage.' + spec.license = 'GPL-3.0' + spec.required_ruby_version = '>= 3.0.0' + + spec.metadata['homepage_uri'] = spec.homepage + spec.metadata['source_code_uri'] = 'https://github.com/ForestAdmin/agent-ruby' + spec.metadata['changelog_uri'] = 'https://github.com/ForestAdmin/agent-ruby/blob/main/CHANGELOG.md' + spec.metadata['rubygems_mfa_required'] = 'false' + + spec.files = Dir.chdir(__dir__) do + `git ls-files -z`.split("\x0").reject do |f| + (File.expand_path(f) == __FILE__) || + f.start_with?(*%w[bin/ test/ spec/ .git Gemfile LICENSE]) + end + end + + spec.bindir = 'exe' + spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) } + spec.require_paths = ['lib'] + + spec.add_dependency 'activerecord', '>= 6.1' + spec.add_dependency 'activesupport', '>= 6.1' + spec.add_dependency 'zeitwerk', '~> 2.3' +end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail.rb new file mode 100644 index 000000000..29bb037f4 --- /dev/null +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail.rb @@ -0,0 +1,10 @@ +require_relative 'forest_admin_audit_trail/version' +require 'zeitwerk' + +loader = Zeitwerk::Loader.for_gem +loader.inflector.inflect('sql' => 'Sql') +loader.setup + +module ForestAdminAuditTrail + class Error < StandardError; end +end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/audit_record.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/audit_record.rb new file mode 100644 index 000000000..064c96afb --- /dev/null +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/audit_record.rb @@ -0,0 +1,45 @@ +module ForestAdminAuditTrail + # Plain value object describing one audited change. Mirrors the columns of `forest.audit_logs`. + # + # Only the actor's `user_id` is stored; the rest of the actor identity is correlated elsewhere + # through `correlation_key`. + class AuditRecord + OPERATIONS = %w[create update delete].freeze + + attr_reader :timestamp, :operation, :collection, :record_id, :user_id, :correlation_key, + :previous_values, :new_values + + def initialize( + operation:, + collection:, + record_id:, + previous_values:, + new_values:, + timestamp: nil, + user_id: nil, + correlation_key: nil + ) + @timestamp = timestamp + @operation = operation + @collection = collection + @record_id = record_id + @user_id = user_id + @correlation_key = correlation_key + @previous_values = previous_values || {} + @new_values = new_values || {} + end + + def to_h + { + timestamp: @timestamp, + operation: @operation, + collection: @collection, + record_id: @record_id, + user_id: @user_id, + correlation_key: @correlation_key, + previous_values: @previous_values, + new_values: @new_values + } + end + end +end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/diff.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/diff.rb new file mode 100644 index 000000000..12a2b8965 --- /dev/null +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/diff.rb @@ -0,0 +1,81 @@ +module ForestAdminAuditTrail + # Minimal structural diff. Nested hashes and arrays of hashes are recursed into, so only the + # keys/indexes whose leaf value actually changed are kept — a single sub-field change does not + # store the whole object/array. Scalars, primitive arrays, dates and other values are compared and + # kept as a whole. + # + # Ruby's `==` already performs deep, key-order-independent equality on Hash and (ordered) equality + # on Array, so it is used directly as the equality primitive. + module Diff + module_function + + # @return [Hash{Symbol=>Object}, nil] { previous:, next: } of the changed leaves, or nil when equal. + def diff(before, after) + return nil if before == after + + return diff_hashes(before, after) if before.is_a?(Hash) && after.is_a?(Hash) + + return diff_object_arrays(before, after) if object_array?(before) && object_array?(after) + + { previous: before.nil? ? nil : before, next: after.nil? ? nil : after } + end + + # Build the previous/new value hashes for the writable columns that actually changed. + # + # @param before [Hash] snapshot of the record before the change (string keys) + # @param patch [Hash] the values being written (string keys); only present keys are considered + # @param columns [Array] writable column names to inspect + # @return [Hash{Symbol=>Hash}] { previous_values:, new_values: } + def changed_values(before, patch, columns) + previous_values = {} + new_values = {} + + columns.each do |column| + delta = patch.key?(column) ? diff(before[column], patch[column]) : nil + next unless delta + + previous_values[column] = delta[:previous] + new_values[column] = delta[:next] + end + + { previous_values: previous_values, new_values: new_values } + end + + # Arrays whose every element is a hash (record-like collections, e.g. a workflow history). + def object_array?(value) + value.is_a?(Array) && !value.empty? && value.all?(Hash) + end + + def diff_hashes(before, after) + previous = {} + next_values = {} + + (before.keys | after.keys).each do |key| + sub = diff(before[key], after[key]) + next unless sub + + previous[key] = sub[:previous] + next_values[key] = sub[:next] + end + + { previous: previous, next: next_values } + end + + def diff_object_arrays(before, after) + previous = {} + next_values = {} + + [before.length, after.length].max.times do |index| + sub = diff(before[index], after[index]) + next unless sub + + previous[index] = sub[:previous] + next_values[index] = sub[:next] + end + + { previous: previous, next: next_values } + end + + private_class_method :diff_hashes, :diff_object_arrays + end +end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/plugin.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/plugin.rb new file mode 100644 index 000000000..479a5a685 --- /dev/null +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/plugin.rb @@ -0,0 +1,163 @@ +require 'securerandom' +require 'time' + +module ForestAdminAuditTrail + # Datasource-agnostic capture layer. Instruments every collection (or a single one) through the + # Forest customizer hooks, computes the minimal before/after diff for each change and appends an + # {AuditRecord} to the configured store. + # + # Usage (datasource level — audits every collection): + # agent.use(ForestAdminAuditTrail::Plugin, store: my_store) + # + # Options: + # store: object responding to #append(AuditRecord). Defaults to a store that logs each record. + # redact: { 'collection_name' => ['field', ...] } — values masked while still recording the change. + class Plugin + REDACTED = '[redacted]'.freeze + + def run(datasource_customizer, collection_customizer = nil, options = {}) + options ||= {} + @store = options[:store] || Stores::LogStore.new + @redact = options[:redact] || {} + + collections = if collection_customizer + [collection_customizer] + else + datasource_customizer.collections.values + end + + collections.each { |collection| instrument(collection) } + end + + private + + def instrument(collection_customizer) + schema = collection_customizer.collection.schema + # Writable columns only: Forest audits what it writes. Read-only fields cover computed/virtual + # fields and DB-managed columns, none of which Forest mutates. + columns = schema[:fields].select do |_name, field| + field.type == 'Column' && !field.is_read_only + end.keys + primary_keys = ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(collection_customizer.collection) + # Reads must carry the primary keys (even read-only ones) so the record id can be built; the + # diff itself stays restricted to the writable columns. + projection = ForestAdminDatasourceToolkit::Components::Query::Projection.new( + (primary_keys + columns).uniq + ) + name = collection_customizer.name + redacted = @redact[name] || [] + + add_create_hook(collection_customizer, columns, primary_keys, name, redacted) + add_update_hooks(collection_customizer, columns, primary_keys, name, redacted, projection) + add_delete_hooks(collection_customizer, columns, primary_keys, name, redacted, projection) + end + + def add_create_hook(collection_customizer, columns, primary_keys, name, redacted) + plugin = self + collection_customizer.add_hook('After', 'Create') do |context| + plugin.send( + :emit, + context.caller, 'create', name, + plugin.send(:record_id, context.record, primary_keys), + {}, + plugin.send(:pick, context.record, columns), + redacted + ) + end + end + + def add_update_hooks(collection_customizer, columns, primary_keys, name, redacted, projection) + plugin = self + + collection_customizer.add_hook('Before', 'Update') do |context| + before = context.collection.list(context.filter, projection) + plugin.send(:pending)[context.filter] = before + end + + collection_customizer.add_hook('After', 'Update') do |context| + before = plugin.send(:pending).delete(context.filter) || [] + patch = context.patch + + before.each do |record| + delta = ForestAdminAuditTrail::Diff.changed_values(record, patch, columns) + next if delta[:new_values].empty? + + plugin.send( + :emit, + context.caller, 'update', name, + plugin.send(:record_id, record, primary_keys), + delta[:previous_values], delta[:new_values], + redacted + ) + end + end + end + + def add_delete_hooks(collection_customizer, columns, primary_keys, name, redacted, projection) + plugin = self + + collection_customizer.add_hook('Before', 'Delete') do |context| + before = context.collection.list(context.filter, projection) + plugin.send(:pending)[context.filter] = before + end + + collection_customizer.add_hook('After', 'Delete') do |context| + before = plugin.send(:pending).delete(context.filter) || [] + + before.each do |record| + plugin.send( + :emit, + context.caller, 'delete', name, + plugin.send(:record_id, record, primary_keys), + plugin.send(:pick, record, columns), + {}, + redacted + ) + end + end + end + + # Snapshots taken in a "before" hook and consumed in the matching "after" hook. Keyed by the + # filter object, which the hook decorator passes unchanged to both hooks within the same thread. + def pending + Thread.current[:forest_audit_trail_snapshots] ||= {}.compare_by_identity + end + + def emit(caller, operation, collection, record_id, previous_values, new_values, redacted) + @store.append( + AuditRecord.new( + timestamp: Time.now.utc.iso8601(3), + operation: operation, + collection: collection, + record_id: record_id, + user_id: caller&.id, + # Same id for every change made within one request — set on the caller by the agent + # (see CallerParser), mirroring the Node agent's caller.requestId. + correlation_key: correlation_key_for(caller), + previous_values: redact(previous_values, redacted), + new_values: redact(new_values, redacted) + ) + ) + end + + def correlation_key_for(caller) + (caller.respond_to?(:request_id) && caller.request_id) || SecureRandom.uuid + end + + def record_id(record, primary_keys) + primary_keys.map { |pk| record[pk].to_s }.join('|') + end + + def pick(record, columns) + columns.to_h { |column| [column, record[column]] } + end + + def redact(values, redacted_fields) + return values if redacted_fields.empty? + + values.each_with_object({}) do |(field, value), result| + result[field] = redacted_fields.include?(field) ? REDACTED : value + end + end + end +end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_connection_base.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_connection_base.rb new file mode 100644 index 000000000..2622aebbd --- /dev/null +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_connection_base.rb @@ -0,0 +1,11 @@ +require 'active_record' + +module ForestAdminAuditTrail + module Sql + # Dedicated abstract base so the audit storage keeps its own connection pool, isolated from the + # host application's ActiveRecord::Base connection. + class AuditConnectionBase < ActiveRecord::Base + self.abstract_class = true + end + end +end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_log.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_log.rb new file mode 100644 index 000000000..725260935 --- /dev/null +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_log.rb @@ -0,0 +1,11 @@ +module ForestAdminAuditTrail + module Sql + # ActiveRecord model for the audit table. Its table name (and schema) is assigned at runtime by + # the SqlStore once the connection and migrations are ready. The JSON attribute overrides force + # Hash <-> JSON casting on every adapter (Postgres json, or text on SQLite). + class AuditLog < AuditConnectionBase + attribute :previous_values, :json + attribute :new_values, :json + end + end +end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb new file mode 100644 index 000000000..5da907848 --- /dev/null +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb @@ -0,0 +1,118 @@ +module ForestAdminAuditTrail + module Sql + # Creates and evolves the audit table through an ordered, append-only list of migrations, tracked + # in a dedicated `audit_migrations` table (namespaced in the `forest` schema on Postgres). + # + # On Postgres the migrations run inside a transaction-scoped advisory lock, so several agent + # instances booting at once apply them one after another instead of racing on the same DDL. The + # schema is created (and committed) first, made idempotent (CREATE SCHEMA IF NOT EXISTS + + # tolerating a concurrent create), because the lock cannot cover a not-yet-existing schema. + class Migrator + MIGRATIONS_TABLE = 'audit_migrations'.freeze + # Arbitrary but stable key pair identifying the audit-trail migration critical section. + ADVISORY_LOCK = [0x464f, 0x5254].freeze # "FO", "RT" + + MIGRATIONS = [ + { + name: '001-create-audit-logs', + up: lambda do |connection, table| + connection.create_table(table) do |t| + t.datetime :timestamp, null: false + t.string :operation, null: false + t.string :collection, null: false + t.string :record_id, null: false + t.integer :user_id + t.string :correlation_key + t.json :previous_values + t.json :new_values + end + end + }, + { + name: '002-index-record-and-correlation', + up: lambda do |connection, table| + base = table.split('.').last + connection.add_index(table, :record_id, name: "#{base}_record_id") + connection.add_index(table, :correlation_key, name: "#{base}_correlation_key") + connection.add_index(table, :user_id, name: "#{base}_user_id") + end + } + ].freeze + + def initialize(connection, schema:, table_name:) + @connection = connection + @schema = schema # nil on adapters without schema support + @table_name = table_name + end + + def run + ensure_schema + + if postgres? + @connection.transaction do + @connection.execute("SELECT pg_advisory_xact_lock(#{ADVISORY_LOCK[0]}, #{ADVISORY_LOCK[1]})") + apply_pending + end + else + apply_pending + end + end + + private + + def postgres? + @connection.adapter_name.downcase.include?('postgres') + end + + def schema? + postgres? && !@schema.nil? + end + + def qualified(name) + schema? ? "#{@schema}.#{name}" : name + end + + # Create the schema first and commit it: the migrations open DDL on the same connection, and a + # CREATE SCHEMA still pending in the lock transaction would not be visible to them. + def ensure_schema + return unless schema? + + @connection.execute("CREATE SCHEMA IF NOT EXISTS #{@connection.quote_schema_name(@schema)}") + rescue ActiveRecord::StatementInvalid => e + # Another instance created it concurrently — CREATE SCHEMA is not fully race-free. + raise unless /already exists|duplicate/i.match?(e.message) + end + + def apply_pending + done = applied_migrations + table = qualified(@table_name) + + MIGRATIONS.each do |migration| + next if done.include?(migration[:name]) + + migration[:up].call(@connection, table) + @connection.execute( + "INSERT INTO #{@connection.quote_table_name(migrations_table)} (name) " \ + "VALUES (#{@connection.quote(migration[:name])})" + ) + end + end + + def applied_migrations + ensure_migrations_table + + @connection.select_values("SELECT name FROM #{@connection.quote_table_name(migrations_table)}") + end + + def ensure_migrations_table + @connection.create_table(migrations_table, id: false, if_not_exists: true) do |t| + t.string :name, null: false + end + end + + def migrations_table + qualified(MIGRATIONS_TABLE) + end + end + end +end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/in_memory_store.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/in_memory_store.rb new file mode 100644 index 000000000..2d70f676d --- /dev/null +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/in_memory_store.rb @@ -0,0 +1,52 @@ +require 'time' + +module ForestAdminAuditTrail + module Stores + # Read/write store kept in memory. Handy for tests and for exercising the record-history route + # without a database. + class InMemoryStore + def initialize + @records = [] + end + + def append(record) + @records << record + end + + def list_by_record(collection:, record_id:, skip: 0, limit: nil, + user_ids: nil, start_timestamp: nil, end_timestamp: nil, order: 'asc') + matches = sorted(matching(collection, record_id, user_ids, start_timestamp, end_timestamp), order) + + skip ||= 0 + ending = limit.nil? ? matches.length : skip + limit + matches[skip...ending] || [] + end + + def count_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil, end_timestamp: nil) + matching(collection, record_id, user_ids, start_timestamp, end_timestamp).length + end + + private + + def matching(collection, record_id, user_ids, start_timestamp, end_timestamp) + @records.select do |record| + record.collection == collection && record.record_id == record_id && + (user_ids.nil? || user_ids.include?(record.user_id)) && + (start_timestamp.nil? || as_time(record.timestamp) >= as_time(start_timestamp)) && + (end_timestamp.nil? || as_time(record.timestamp) <= as_time(end_timestamp)) + end + end + + # Ties on equal timestamps fall back to append order (index) in both directions — the in-memory + # equivalent of the SQL store's auto-increment id tie-break. + def sorted(records, order) + sign = order.to_s == 'desc' ? -1 : 1 + records.each_with_index.sort_by { |record, i| [sign * as_time(record.timestamp).to_f, i] }.map(&:first) + end + + def as_time(value) + value.is_a?(::Time) ? value : ::Time.iso8601(value.to_s) + end + end + end +end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/log_store.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/log_store.rb new file mode 100644 index 000000000..30a610084 --- /dev/null +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/log_store.rb @@ -0,0 +1,26 @@ +require 'json' + +module ForestAdminAuditTrail + module Stores + # Default store used when none is configured: it only logs each change. Write-only — the + # record-history route reading from it always gets an empty result. + class LogStore + def initialize(logger: nil) + @logger = logger + end + + def append(record) + message = "[audit-trail] #{record.to_h.to_json}" + @logger ? @logger.log('Info', message) : warn(message) + end + + def list_by_record(**) + [] + end + + def count_by_record(**) + 0 + end + end + end +end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb new file mode 100644 index 000000000..bd11ada90 --- /dev/null +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb @@ -0,0 +1,116 @@ +require 'time' + +module ForestAdminAuditTrail + module Stores + # SQL-backed store that both writes every audited change and reads the per-record history back. + # + # Construction is cheap; the connection is opened lazily on the first append or read, at which + # point the `forest` schema and the `audit_logs` table are created/evolved through migrations. + # Pass the SAME instance to the plugin (`store:`) and to the agent (`audit_trail: { store: }`) so + # writes and the record-history route agree on storage. + class SqlStore + DEFAULT_SCHEMA = 'forest'.freeze + DEFAULT_TABLE = 'audit_logs'.freeze + + def initialize(connection_string:, schema: DEFAULT_SCHEMA, table_name: DEFAULT_TABLE) + @connection_string = connection_string + @schema = schema + @table_name = table_name + @mutex = Mutex.new + @ready = false + end + + def append(record) + model.create!(to_row(record)) + end + + def list_by_record(collection:, record_id:, skip: 0, limit: nil, + user_ids: nil, start_timestamp: nil, end_timestamp: nil, order: 'asc') + # `id` (insertion order) breaks ties on equal timestamps in both directions, keeping pages + # deterministic and stable. + relation = scope(collection, record_id, user_ids, start_timestamp, end_timestamp) + .order(timestamp: order.to_s == 'desc' ? :desc : :asc, id: :asc) + .offset(skip || 0) + relation = relation.limit(limit) unless limit.nil? + + relation.map { |row| from_row(row) } + end + + def count_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil, end_timestamp: nil) + scope(collection, record_id, user_ids, start_timestamp, end_timestamp).count + end + + private + + def scope(collection, record_id, user_ids, start_timestamp, end_timestamp) + relation = model.where(collection: collection, record_id: record_id) + relation = relation.where(user_id: user_ids) if user_ids + # Compare as Time so ActiveRecord casts the bound to the datetime column's storage format + # (raw ISO strings with a `Z` would compare lexically against the cast rows and never match). + relation = relation.where('timestamp >= ?', as_time(start_timestamp)) if start_timestamp + relation = relation.where('timestamp <= ?', as_time(end_timestamp)) if end_timestamp + relation + end + + def as_time(value) + value.is_a?(::Time) ? value : ::Time.iso8601(value.to_s) + end + + def model + ensure_ready + Sql::AuditLog + end + + def ensure_ready + return if @ready + + @mutex.synchronize do + return if @ready + + Sql::AuditConnectionBase.establish_connection(@connection_string) + connection = Sql::AuditConnectionBase.connection + Sql::Migrator.new(connection, schema: schema_for(connection), table_name: @table_name).run + Sql::AuditLog.table_name = qualified(connection) + # The table was just created at runtime: refresh the model's cached column metadata. + Sql::AuditLog.reset_column_information + @ready = true + end + end + + def schema_for(connection) + connection.adapter_name.downcase.include?('postgres') ? @schema : nil + end + + def qualified(connection) + schema = schema_for(connection) + schema ? "#{schema}.#{@table_name}" : @table_name + end + + def to_row(record) + { + timestamp: record.timestamp, + operation: record.operation, + collection: record.collection, + record_id: record.record_id, + user_id: record.user_id, + correlation_key: record.correlation_key, + previous_values: record.previous_values, + new_values: record.new_values + } + end + + def from_row(row) + AuditRecord.new( + timestamp: row.timestamp.respond_to?(:iso8601) ? row.timestamp.iso8601(3) : row.timestamp.to_s, + operation: row.operation, + collection: row.collection, + record_id: row.record_id, + user_id: row.user_id, + correlation_key: row.correlation_key, + previous_values: row.previous_values || {}, + new_values: row.new_values || {} + ) + end + end + end +end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb new file mode 100644 index 000000000..5eacc1525 --- /dev/null +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb @@ -0,0 +1,3 @@ +module ForestAdminAuditTrail + VERSION = "1.33.1" +end diff --git a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/diff_spec.rb b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/diff_spec.rb new file mode 100644 index 000000000..c9d7ba9cc --- /dev/null +++ b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/diff_spec.rb @@ -0,0 +1,52 @@ +require 'spec_helper' + +module ForestAdminAuditTrail + describe Diff do + describe '.diff' do + it 'returns nil when values are deeply equal regardless of hash key order' do + expect(described_class.diff({ 'a' => 1, 'b' => 2 }, { 'b' => 2, 'a' => 1 })).to be_nil + end + + it 'keeps only the changed leaf of a nested object' do + before = { 'city' => 'Paris', 'zip' => '75001' } + after = { 'city' => 'Lyon', 'zip' => '75001' } + + expect(described_class.diff(before, after)).to eq( + previous: { 'city' => 'Paris' }, + next: { 'city' => 'Lyon' } + ) + end + + it 'diffs an array of objects index by index' do + before = [{ 'name' => 'a' }, { 'name' => 'b' }] + after = [{ 'name' => 'a' }, { 'name' => 'c' }] + + expect(described_class.diff(before, after)).to eq( + previous: { 1 => { 'name' => 'b' } }, + next: { 1 => { 'name' => 'c' } } + ) + end + + it 'keeps scalars and primitive arrays whole' do + expect(described_class.diff(%w[a b], %w[a c])).to eq(previous: %w[a b], next: %w[a c]) + end + + it 'reports nil for a newly set or cleared value' do + expect(described_class.diff(nil, 'x')).to eq(previous: nil, next: 'x') + expect(described_class.diff('x', nil)).to eq(previous: 'x', next: nil) + end + end + + describe '.changed_values' do + it 'only records writable columns present in the patch that actually changed' do + before = { 'status' => 'open', 'name' => 'Acme', 'ignored' => 1 } + patch = { 'status' => 'closed', 'name' => 'Acme' } + + expect(described_class.changed_values(before, patch, %w[status name])).to eq( + previous_values: { 'status' => 'open' }, + new_values: { 'status' => 'closed' } + ) + end + end + end +end diff --git a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/plugin_spec.rb b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/plugin_spec.rb new file mode 100644 index 000000000..d4f77fd18 --- /dev/null +++ b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/plugin_spec.rb @@ -0,0 +1,121 @@ +require 'spec_helper' + +module ForestAdminAuditTrail + describe Plugin do + let(:column_schema) { ForestAdminDatasourceToolkit::Schema::ColumnSchema } + + let(:store) do + Class.new do + attr_reader :records + + def initialize + @records = [] + end + + def append(record) + @records << record + end + end.new + end + + let(:fields) do + { + 'id' => column_schema.new(column_type: 'Number', is_primary_key: true, is_read_only: true), + 'name' => column_schema.new(column_type: 'String'), + 'address' => column_schema.new(column_type: 'Json') + } + end + + let(:hooks) { {} } + let(:collection) { double('collection') } + let(:caller_double) { double('caller', id: 42, request_id: 'req-xyz') } + + let(:collection_customizer) do + customizer = double('CollectionCustomizer', name: 'companies', collection: collection) + allow(customizer).to receive(:add_hook) { |position, type, &block| hooks["#{position}_#{type}"] = block } + customizer + end + + before do + allow(collection).to receive(:schema).and_return({ fields: fields }) + described_class.new.run(nil, collection_customizer, store: store) + end + + it 'records a create with only the writable columns' do + record = { 'id' => 1, 'name' => 'Acme', 'address' => { 'city' => 'Paris' } } + + hooks['After_Create'].call(double('ctx', caller: caller_double, record: record)) + + audit = store.records.last + expect(audit.operation).to eq('create') + expect(audit.record_id).to eq('1') + expect(audit.user_id).to eq(42) + expect(audit.correlation_key).to eq('req-xyz') + expect(audit.previous_values).to eq({}) + expect(audit.new_values).to eq({ 'name' => 'Acme', 'address' => { 'city' => 'Paris' } }) + end + + it "shares the caller's request id as the correlation key across records of one operation" do + filter = Object.new + allow(collection).to receive(:list).and_return( + [{ 'id' => 1, 'name' => 'A' }, { 'id' => 2, 'name' => 'B' }] + ) + + hooks['Before_Update'].call(double('before', caller: caller_double, filter: filter, collection: collection)) + hooks['After_Update'].call(double('after', caller: caller_double, filter: filter, patch: { 'name' => 'Z' })) + + expect(store.records.map(&:correlation_key)).to eq(%w[req-xyz req-xyz]) + end + + it 'records an update with the minimal nested diff' do + filter = Object.new + before_record = { 'id' => 1, 'name' => 'Acme', 'address' => { 'city' => 'Paris', 'zip' => '1' } } + allow(collection).to receive(:list).and_return([before_record]) + + hooks['Before_Update'].call(double('before', caller: caller_double, filter: filter, collection: collection)) + hooks['After_Update'].call( + double('after', caller: caller_double, filter: filter, + patch: { 'address' => { 'city' => 'Lyon', 'zip' => '1' } }) + ) + + audit = store.records.last + expect(audit.operation).to eq('update') + expect(audit.previous_values).to eq({ 'address' => { 'city' => 'Paris' } }) + expect(audit.new_values).to eq({ 'address' => { 'city' => 'Lyon' } }) + end + + it 'does not record an update when nothing writable changed' do + filter = Object.new + allow(collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'Acme' }]) + + hooks['Before_Update'].call(double('before', caller: caller_double, filter: filter, collection: collection)) + hooks['After_Update'].call(double('after', caller: caller_double, filter: filter, patch: { 'name' => 'Acme' })) + + expect(store.records).to be_empty + end + + it 'records a delete with the previous values' do + filter = Object.new + allow(collection).to receive(:list).and_return([{ 'id' => 7, 'name' => 'Gone', 'address' => nil }]) + + hooks['Before_Delete'].call(double('before', caller: caller_double, filter: filter, collection: collection)) + hooks['After_Delete'].call(double('after', caller: caller_double, filter: filter)) + + audit = store.records.last + expect(audit.operation).to eq('delete') + expect(audit.record_id).to eq('7') + expect(audit.previous_values).to eq({ 'name' => 'Gone', 'address' => nil }) + expect(audit.new_values).to eq({}) + end + + it 'masks redacted fields while still recording the change' do + described_class.new.run(nil, collection_customizer, store: store, redact: { 'companies' => ['name'] }) + + hooks['After_Create'].call( + double('ctx', caller: caller_double, record: { 'id' => 1, 'name' => 'Secret', 'address' => nil }) + ) + + expect(store.records.last.new_values['name']).to eq(described_class::REDACTED) + end + end +end diff --git a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/in_memory_store_spec.rb b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/in_memory_store_spec.rb new file mode 100644 index 000000000..8a76cd91e --- /dev/null +++ b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/in_memory_store_spec.rb @@ -0,0 +1,81 @@ +require 'spec_helper' + +module ForestAdminAuditTrail + module Stores + describe InMemoryStore do + subject(:store) { described_class.new } + + def record(over = {}) + AuditRecord.new( + operation: 'update', collection: 'accounts', record_id: '1', + previous_values: {}, new_values: {}, timestamp: '2026-01-02T03:04:05.000Z', **over + ) + end + + it 'returns rows for the queried record, oldest first' do + store.append(record(timestamp: '2026-01-02T03:04:06.000Z', correlation_key: 'b')) + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'a')) + store.append(record(record_id: '2', correlation_key: 'other')) + + history = store.list_by_record(collection: 'accounts', record_id: '1') + + expect(history.map(&:correlation_key)).to eq(%w[a b]) + end + + it 'honors skip and limit' do + 3.times { |i| store.append(record(timestamp: "2026-01-02T03:04:0#{i}.000Z", correlation_key: i.to_s)) } + + page = store.list_by_record(collection: 'accounts', record_id: '1', skip: 1, limit: 1) + + expect(page.map(&:correlation_key)).to eq(['1']) + end + + it 'returns an empty array when nothing matches' do + store.append(record) + + expect(store.list_by_record(collection: 'accounts', record_id: 'missing')).to eq([]) + end + + it 'sorts newest first when order is desc, ties keeping append order' do + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'a')) + store.append(record(timestamp: '2026-01-02T03:04:06.000Z', correlation_key: 'b')) + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'a2')) + + history = store.list_by_record(collection: 'accounts', record_id: '1', order: 'desc') + + expect(history.map(&:correlation_key)).to eq(%w[b a a2]) + end + + it 'filters by user_ids, dropping non-matching actors' do + store.append(record(user_id: 7, correlation_key: 'keep')) + store.append(record(user_id: 9, correlation_key: 'drop')) + + history = store.list_by_record(collection: 'accounts', record_id: '1', user_ids: [7]) + + expect(history.map(&:correlation_key)).to eq(['keep']) + end + + it 'filters by inclusive start/end timestamps regardless of string format' do + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'before')) + store.append(record(timestamp: '2026-01-02T03:04:07.000Z', correlation_key: 'inside')) + store.append(record(timestamp: '2026-01-02T03:04:09.000Z', correlation_key: 'after')) + + history = store.list_by_record( + collection: 'accounts', record_id: '1', + start_timestamp: '2026-01-02T03:04:06.000Z', end_timestamp: '2026-01-02T03:04:08.000Z' + ) + + expect(history.map(&:correlation_key)).to eq(['inside']) + end + + it 'counts matches independently of skip/limit' do + store.append(record(user_id: 7)) + store.append(record(user_id: 7)) + store.append(record(user_id: 9)) + + expect(store.count_by_record(collection: 'accounts', record_id: '1')).to eq(3) + expect(store.count_by_record(collection: 'accounts', record_id: '1', user_ids: [7])).to eq(2) + end + end + end +end diff --git a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb new file mode 100644 index 000000000..d2595c6b3 --- /dev/null +++ b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb @@ -0,0 +1,105 @@ +require 'spec_helper' +require 'tempfile' + +module ForestAdminAuditTrail + module Stores + describe SqlStore do + let(:db) { Tempfile.new(['audit', '.sqlite3']) } + let(:store) { described_class.new(connection_string: { adapter: 'sqlite3', database: db.path }) } + + after do + ForestAdminAuditTrail::Sql::AuditConnectionBase.remove_connection + db.close! + end + + def record(over = {}) + AuditRecord.new( + operation: 'update', collection: 'accounts', record_id: '1', + previous_values: { 'status' => 'open' }, new_values: { 'status' => 'closed' }, + timestamp: '2026-01-02T03:04:05.000Z', user_id: 42, correlation_key: 'req-1', **over + ) + end + + def connection + ForestAdminAuditTrail::Sql::AuditConnectionBase.connection + end + + it 'creates the audit table with the expected columns on first write' do + store.append(record) + + expect(ForestAdminAuditTrail::Sql::AuditLog.column_names.sort).to eq( + %w[collection correlation_key id new_values operation previous_values record_id timestamp user_id] + ) + end + + it 'persists and reads back a record, decoding the JSON columns' do + store.append(record) + + audit = store.list_by_record(collection: 'accounts', record_id: '1').first + expect(audit.operation).to eq('update') + expect(audit.user_id).to eq(42) + expect(audit.previous_values).to eq({ 'status' => 'open' }) + expect(audit.new_values).to eq({ 'status' => 'closed' }) + end + + it 'returns a record history oldest-first, scoped to the record, honoring skip/limit' do + store.append(record(timestamp: '2026-01-02T03:04:06.000Z', correlation_key: 'b')) + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'a')) + store.append(record(record_id: '2', correlation_key: 'other')) + + history = store.list_by_record(collection: 'accounts', record_id: '1') + expect(history.map(&:correlation_key)).to eq(%w[a b]) + + page = store.list_by_record(collection: 'accounts', record_id: '1', skip: 1, limit: 1) + expect(page.map(&:correlation_key)).to eq(['b']) + end + + it 'sorts newest first when order is desc, breaking ties by insertion order' do + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'a')) + store.append(record(timestamp: '2026-01-02T03:04:06.000Z', correlation_key: 'b')) + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'a2')) + + history = store.list_by_record(collection: 'accounts', record_id: '1', order: 'desc') + expect(history.map(&:correlation_key)).to eq(%w[b a a2]) + end + + it 'filters by user_ids and inclusive timestamp range' do + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', user_id: 7, correlation_key: 'keep')) + store.append(record(timestamp: '2026-01-02T03:04:09.000Z', user_id: 7, correlation_key: 'late')) + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', user_id: 9, correlation_key: 'other')) + + history = store.list_by_record( + collection: 'accounts', record_id: '1', user_ids: [7], + start_timestamp: '2026-01-02T03:04:04.000Z', end_timestamp: '2026-01-02T03:04:06.000Z' + ) + expect(history.map(&:correlation_key)).to eq(['keep']) + end + + it 'counts matches independently of skip/limit, respecting filters' do + store.append(record(user_id: 7)) + store.append(record(user_id: 7)) + store.append(record(user_id: 9)) + + expect(store.count_by_record(collection: 'accounts', record_id: '1')).to eq(3) + expect(store.count_by_record(collection: 'accounts', record_id: '1', user_ids: [7])).to eq(2) + end + + it 'tracks applied migrations and is idempotent across stores' do + store.append(record) + described_class.new(connection_string: { adapter: 'sqlite3', database: db.path }).append(record) + + names = connection.select_values('SELECT name FROM audit_migrations ORDER BY name') + expect(names).to eq(['001-create-audit-logs', '002-index-record-and-correlation']) + end + + it 'indexes record_id, correlation_key and user_id' do + store.append(record) + + index_names = connection.indexes('audit_logs').map(&:name) + expect(index_names).to include( + 'audit_logs_record_id', 'audit_logs_correlation_key', 'audit_logs_user_id' + ) + end + end + end +end diff --git a/packages/forest_admin_audit_trail/spec/spec_helper.rb b/packages/forest_admin_audit_trail/spec/spec_helper.rb new file mode 100644 index 000000000..b1153fb60 --- /dev/null +++ b/packages/forest_admin_audit_trail/spec/spec_helper.rb @@ -0,0 +1,25 @@ +require 'simplecov' +require 'simplecov_json_formatter' +require 'simplecov-html' +require 'active_record' +require 'sqlite3' +require 'forest_admin_datasource_toolkit' +require 'forest_admin_datasource_customizer' +require 'forest_admin_audit_trail' + +SimpleCov.formatters = [SimpleCov::Formatter::JSONFormatter, SimpleCov::Formatter::HTMLFormatter] +SimpleCov.start do + add_filter 'spec' +end + +SimpleCov.coverage_dir 'coverage' +SimpleCov.at_exit do + result = SimpleCov.result + result.format! +end + +RSpec.configure do |config| + config.expect_with :rspec do |c| + c.syntax = :expect + end +end diff --git a/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/components/caller.rb b/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/components/caller.rb index 183e6b5ea..1d6cad012 100644 --- a/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/components/caller.rb +++ b/packages/forest_admin_datasource_toolkit/lib/forest_admin_datasource_toolkit/components/caller.rb @@ -1,7 +1,8 @@ module ForestAdminDatasourceToolkit module Components class Caller - attr_reader :id, :email, :first_name, :last_name, :tags, :team, :rendering_id, :timezone, :permission_level, :role + attr_reader :id, :email, :first_name, :last_name, :tags, :team, :rendering_id, :timezone, + :permission_level, :role, :request, :request_id def initialize( id:, @@ -15,6 +16,7 @@ def initialize( permission_level:, role: nil, request: {}, + request_id: nil, project: nil, environment: nil, **_extra_args @@ -30,6 +32,7 @@ def initialize( @permission_level = permission_level @role = role @request = request + @request_id = request_id @project = project @environment = environment end diff --git a/packages/forest_admin_rails/lib/forest_admin_rails.rb b/packages/forest_admin_rails/lib/forest_admin_rails.rb index 2606f91be..60fb95be7 100644 --- a/packages/forest_admin_rails/lib/forest_admin_rails.rb +++ b/packages/forest_admin_rails/lib/forest_admin_rails.rb @@ -34,6 +34,9 @@ module ForestAdminRails setting :disable_route_cache, default: false setting :rpc_max_polling_threads, default: nil setting :workflow_executor_url, default: nil + # { store: }. When set, the record-history + # route (`/_audit-trail/:collection/:id`) is registered and reads from this store. + setting :audit_trail, default: nil if defined?(Rails::Railtie) # logic for cors middleware,... here // or it might be into Engine diff --git a/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb b/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb index 829667caf..5d82dfa42 100644 --- a/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb +++ b/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb @@ -34,9 +34,16 @@ class Engine < ::Rails::Engine agent_factory.setup(ForestAdminRails.config) load_configuration load_cors + load_correlation_id end end + # Echo the agent-generated correlation id on every response (mirrors the Node agent's + # router.use(correlationIdMiddleware)); CORS exposure of the header is handled in load_cors. + def load_correlation_id + config.middleware.use ForestAdminAgent::Http::CorrelationIdMiddleware + end + def load_configuration return unless running_web_server? return unless create_agent_file_exists? @@ -102,7 +109,8 @@ def load_cors hostnames += ENV['CORS_ORIGINS'].split(',') if ENV['CORS_ORIGINS'] origins hostnames - resource '*', headers: :any, methods: :any, credentials: true, max_age: 86_400 + resource '*', headers: :any, methods: :any, credentials: true, max_age: 86_400, + expose: [ForestAdminAgent::Http::CorrelationId::HEADER] end end end From 05e44158e5ae04be81a40bedcf48d0613b065cf7 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Thu, 18 Jun 2026 17:36:07 +0200 Subject: [PATCH 02/35] chore(audit_trail): exclude gemspec from Gemspec/RequireMFA rubocop cop Matches the other packages, which disable MFA and are excluded from the cop. Co-Authored-By: Claude Opus 4.8 --- .rubocop.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.rubocop.yml b/.rubocop.yml index d309d90d2..fe925b3e6 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -28,6 +28,7 @@ Gemspec/OrderedDependencies: Gemspec/RequireMFA: Exclude: + - 'packages/forest_admin_audit_trail/forest_admin_audit_trail.gemspec' - 'agent_ruby.gemspec' - 'packages/forest_admin_agent/forest_admin_agent.gemspec' - 'packages/forest_admin_rails/forest_admin_rails.gemspec' From e6429a071d7fc4a473e534e6c78757f44dbb2cdb Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Thu, 18 Jun 2026 17:55:10 +0200 Subject: [PATCH 03/35] feat(audit_trail): add correlation-scoped record-history routes Add GET /_audit-trail/correlation/:correlation_key (one key) and the batch GET/POST /_audit-trail/correlations (correlationKeys list, POST body to dodge URL limits), both scoped to a record via collection/recordId params, sharing the per-record auth and store gate. Back them with list_by_correlation / list_by_correlations on the stores (SQL + in-memory + log no-op). Register the correlation source before audit_trail so /_audit-trail/correlation matches it instead of the per-record /_audit-trail/:collection_name/:id (Rails matches in definition order). Mirror the Node README: Rails configuration process plus docs for the record-history, correlation and batch correlation routes. Co-Authored-By: Claude Opus 4.8 (1M context) --- .rubocop.yml | 1 + .../lib/forest_admin_agent/http/router.rb | 3 + .../resources/audit_trail_correlation.rb | 104 ++++++++++++++ .../resources/audit_trail_correlation_spec.rb | 103 ++++++++++++++ packages/forest_admin_audit_trail/README.md | 130 +++++++++++++++++- .../stores/in_memory_store.rb | 15 ++ .../stores/log_store.rb | 8 ++ .../stores/sql_store.rb | 14 ++ .../stores/in_memory_store_spec.rb | 41 ++++++ .../stores/sql_store_spec.rb | 28 ++++ 10 files changed, 441 insertions(+), 6 deletions(-) create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb create mode 100644 packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb diff --git a/.rubocop.yml b/.rubocop.yml index fe925b3e6..6f56e1247 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -464,6 +464,7 @@ RSpec/VerifiedDoubles: Exclude: - 'packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/plugin_spec.rb' - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb' + - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb' - 'packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/composite_datasource_spec.rb' RSpec/VerifiedDoubleReference: diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb b/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb index 5811efdc9..1b12950ae 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb @@ -64,6 +64,9 @@ def self.routes { name: 'dissociate_related', handler: -> { Resources::Related::DissociateRelated.new.routes } }, { name: 'update_related', handler: -> { Resources::Related::UpdateRelated.new.routes } }, { name: 'update_field', handler: -> { Resources::UpdateField.new.routes } }, + # Registered before `audit_trail` so `/_audit-trail/correlation(s)` matches here instead of + # the per-record `/_audit-trail/:collection_name/:id` (Rails matches in definition order). + { name: 'audit_trail_correlation', handler: -> { Resources::AuditTrailCorrelation.new.routes } }, { name: 'audit_trail', handler: -> { Resources::AuditTrail.new.routes } }, { name: 'workflow_executor_proxy', handler: -> { Workflow::WorkflowExecutorProxy.new.routes } } ] diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb new file mode 100644 index 000000000..e7e6d6a46 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb @@ -0,0 +1,104 @@ +module ForestAdminAgent + module Routes + module Resources + # Correlation-scoped record-history routes, mirroring the Node agent's + # `/_audit-trail/correlation/:key` and `/_audit-trail/correlations`. Registered only when an + # `audit_trail` store is configured; the store must respond to `list_by_correlation` and + # `list_by_correlations`. All three routes are scoped to a single record through the + # `collection`/`recordId` query (GET) or body (POST) params and share the per-record auth. + class AuditTrailCorrelation < AbstractAuthenticatedRoute + def setup_routes + return self unless store + + add_route( + 'forest_audit_trail_correlation', + 'get', + '/_audit-trail/correlation/:correlation_key', + ->(args) { handle_history(args) } + ) + # GET carries the keys in `correlationKeys`; POST accepts a body list to dodge URL limits. + add_route( + 'forest_audit_trail_correlations', + 'get', + '/_audit-trail/correlations', + ->(args) { handle_batch(args) } + ) + add_route( + 'forest_audit_trail_correlations_batch', + 'post', + '/_audit-trail/correlations', + ->(args) { handle_batch(args) } + ) + + self + end + + def handle_history(args = {}) + collection, record_id = assert_scope(args) + + history = store.list_by_correlation( + collection: collection.name, + record_id: record_id, + correlation_key: args[:params]['correlation_key'] + ) + + { name: collection.name, content: { data: history.map(&:to_h) } } + end + + def handle_batch(args = {}) + collection, record_id = assert_scope(args) + correlation_keys = parse_correlation_keys(args) + + history = if correlation_keys.empty? + [] + else + store.list_by_correlations( + collection: collection.name, + record_id: record_id, + correlation_keys: correlation_keys + ) + end + + { name: collection.name, content: { data: history.map(&:to_h) } } + end + + private + + def assert_scope(args) + context = build(args) + name = args.dig(:params, 'collection').to_s + record_id = args.dig(:params, 'recordId').to_s + + raise Http::Exceptions::ValidationError, 'Missing collection' if name.empty? + raise Http::Exceptions::ValidationError, 'Missing recordId' if record_id.empty? + + collection = get_collection(context, name) + context.permissions.can?(:read, collection) + + [collection, record_id] + end + + def get_collection(context, name) + context.datasource.get_collection(name) + rescue ForestAdminDatasourceToolkit::Exceptions::ForestException => e + raise Http::Exceptions::NotFoundError, e.message if e.message.include?('not found') + + raise + end + + # Body array (POST) takes precedence, otherwise the comma-separated query param (GET). + def parse_correlation_keys(args) + raw = args.dig(:params, 'correlationKeys') + keys = raw.is_a?(Array) ? raw : raw.to_s.split(',') + + keys.map { |key| key.to_s.strip }.reject(&:empty?) + end + + def store + config = ForestAdminAgent::Facades::Container.config_from_cache + config && config[:audit_trail] && config[:audit_trail][:store] + end + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb new file mode 100644 index 000000000..e4f077e84 --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb @@ -0,0 +1,103 @@ +require 'spec_helper' + +module ForestAdminAgent + module Routes + module Resources + describe AuditTrailCorrelation do + let(:store) { double('store') } + let(:collection) { double('collection', name: 'books') } + + def route_with_store(history: []) + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: store } }) + allow(store).to receive_messages(list_by_correlation: history, list_by_correlations: history) + + route = described_class.new + datasource = double('datasource') + allow(datasource).to receive(:get_collection).with('books').and_return(collection) + context = double('context', datasource: datasource, + permissions: double('permissions', can?: true)) + allow(route).to receive(:build).and_return(context) + route + end + + it 'registers the correlation routes when a store is configured' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: Object.new } }) + + expect(described_class.new.routes.keys).to include( + 'forest_audit_trail_correlation', 'forest_audit_trail_correlations', 'forest_audit_trail_correlations_batch' + ) + end + + it 'does not register when no store is configured' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache).and_return({}) + + expect(described_class.new.routes).to be_empty + end + + it 'reads a single correlation history scoped to the record' do + route = route_with_store(history: [double('entry', to_h: { operation: 'update' })]) + + result = route.handle_history( + { headers: {}, params: { 'collection' => 'books', 'recordId' => '2', 'correlation_key' => 'req-1' } } + ) + + expect(store).to have_received(:list_by_correlation).with( + collection: 'books', record_id: '2', correlation_key: 'req-1' + ) + expect(result[:content]).to eq({ data: [{ operation: 'update' }] }) + end + + it 'reads a batch history from comma-separated query keys (GET)' do + route = route_with_store(history: [double('entry', to_h: { operation: 'update' })]) + + route.handle_batch( + { headers: {}, params: { 'collection' => 'books', 'recordId' => '2', 'correlationKeys' => 'a, b' } } + ) + + expect(store).to have_received(:list_by_correlations).with( + collection: 'books', record_id: '2', correlation_keys: %w[a b] + ) + end + + it 'reads a batch history from a body array (POST)' do + route = route_with_store + + route.handle_batch( + { headers: {}, params: { 'collection' => 'books', 'recordId' => '2', 'correlationKeys' => %w[a b] } } + ) + + expect(store).to have_received(:list_by_correlations).with( + collection: 'books', record_id: '2', correlation_keys: %w[a b] + ) + end + + it 'returns an empty batch without hitting the store when no keys are given' do + route = route_with_store + + result = route.handle_batch({ headers: {}, params: { 'collection' => 'books', 'recordId' => '2' } }) + + expect(store).not_to have_received(:list_by_correlations) + expect(result[:content]).to eq({ data: [] }) + end + + it 'rejects a missing collection' do + route = route_with_store + + expect do + route.handle_history({ headers: {}, params: { 'recordId' => '2', 'correlation_key' => 'req-1' } }) + end.to raise_error(Http::Exceptions::ValidationError, /Missing collection/) + end + + it 'rejects a missing recordId' do + route = route_with_store + + expect do + route.handle_history({ headers: {}, params: { 'collection' => 'books', 'correlation_key' => 'req-1' } }) + end.to raise_error(Http::Exceptions::ValidationError, /Missing recordId/) + end + end + end + end +end diff --git a/packages/forest_admin_audit_trail/README.md b/packages/forest_admin_audit_trail/README.md index de48e9901..9156f30a9 100644 --- a/packages/forest_admin_audit_trail/README.md +++ b/packages/forest_admin_audit_trail/README.md @@ -14,19 +14,63 @@ Two decoupled parts: ## Use it in an agent `ForestAdminAuditTrail::Stores::SqlStore` both **writes** every audited change and **reads** the -per-record history back. Pass the **same** instance to the plugin (so writes land) and to the agent -`audit_trail` option (so the record-history route is exposed and reads from the same table). +per-record history back. The integration has two halves that must share the **same** store instance: + +- the agent `audit_trail` option — so the record-history routes are mounted and read from the table; +- `agent.use(ForestAdminAuditTrail::Plugin, store:)` — so every change is captured into that table. + +Add the gem: + +```ruby +# Gemfile +gem 'forest_admin_audit_trail' +``` + +### Rails (forest_admin_rails) + +Build the store once in an initializer and hand it to the agent through `config.audit_trail`. A +constant keeps the single instance reachable from the agent factory: ```ruby -store = ForestAdminAuditTrail::Stores::SqlStore.new( - connection_string: ENV['AUDIT_TRAIL_DATABASE_URL'] # Postgres URL, or any ActiveRecord config +# config/initializers/forest_admin_rails.rb +AUDIT_TRAIL_STORE = ForestAdminAuditTrail::Stores::SqlStore.new( + connection_string: { # or any ActiveRecord config / URL + adapter: 'postgresql', host: ENV['AUDIT_DB_HOST'], port: ENV['AUDIT_DB_PORT'], + username: ENV['AUDIT_DB_USER'], password: ENV['AUDIT_DB_PASSWORD'], database: ENV['AUDIT_DB_NAME'] + } ) +ForestAdminRails.configure do |config| + config.auth_secret = ENV['FOREST_AUTH_SECRET'] + config.env_secret = ENV['FOREST_ENV_SECRET'] + # exposes GET /forest/_audit-trail/{collection}/:id and the correlation routes below + config.audit_trail = { store: AUDIT_TRAIL_STORE } +end +``` + +Then install the capture plugin where you build the agent (`lib/forest_admin_rails/create_agent.rb`), +reading the **same** store back from the config so writes and reads agree: + +```ruby +# lib/forest_admin_rails/create_agent.rb +@agent = ForestAdminAgent::Builder::AgentFactory.instance + .add_datasource(my_datasource) + +@agent.use(ForestAdminAuditTrail::Plugin, store: ForestAdminRails.config.audit_trail[:store]) + +@agent.build +``` + +### Plain agent (no Rails) + +```ruby +store = ForestAdminAuditTrail::Stores::SqlStore.new(connection_string: ENV['AUDIT_TRAIL_DATABASE_URL']) + ForestAdminAgent::Builder::AgentFactory.instance.setup( auth_secret: ENV['FOREST_AUTH_SECRET'], env_secret: ENV['FOREST_ENV_SECRET'], # ...usual options... - audit_trail: { store: store } # exposes GET /forest/_audit-trail/{collection}/:id + audit_trail: { store: store } ) agent.use(ForestAdminAuditTrail::Plugin, store: store) @@ -34,7 +78,81 @@ agent.use(ForestAdminAuditTrail::Plugin, store: store) On the first write or read the store ensures the `forest` schema exists and runs any pending migrations to create/upgrade `forest.audit_logs`; every create / update / delete performed through -Forest then writes one row per record. +Forest then writes one row per record, and the **Historic** tab in the UI reads from the same table. + +> The record-history routes are mounted **only** when `audit_trail[:store]` is set. Leave it unset +> (the default) and the feature stays off — the plugin can still write to a `LogStore` if used alone. + +## Routes + +All routes live under `/forest/_audit-trail`, are gated on `audit_trail[:store]` being configured, +and require read permission on the target collection (`can?(:read, collection)`). + +### Record-history route + +`GET /forest/_audit-trail/{collection}/{recordId}` returns the current page of history (newest first +by default) together with the filtered total: + +```json +{ "data": [ /* current page rows */ ], "meta": { "count": 137 } } +``` + +`meta.count` is the number of rows matching the active filters (not the absolute total) and is +independent of the page. Optional filters (all combine with `AND`; omit them for the full history): + +| query param | format | effect | +| ----------- | -------------------------------- | ----------------------------------------------- | +| `userIds` | comma-separated integers `12,45` | keep only entries whose `user_id` is in the list | +| `startDate` | `YYYY-MM-DD` or datetime (incl.) | keep entries from this lower bound onward | +| `endDate` | `YYYY-MM-DD` or datetime (incl.) | keep entries up to this upper bound | + +`startDate` / `endDate` are read as **local wall-clock time** in the request `timezone` query param +(e.g. `Europe/Paris`, default `UTC`) and converted to a UTC instant before querying, so filtering +happens in SQL. Two shapes are accepted: + +- **Bare day** `YYYY-MM-DD` — `startDate` snaps to `00:00:00.000`, `endDate` to `23:59:59.999`. +- **Datetime** `YYYY-MM-DD[T| ]HH:mm[:ss]` — `T` or space separator, seconds optional; when seconds + are omitted `endDate` is completed to `:59.999` and `startDate` stays at `:00.000`. + +Both bounds are **inclusive**. Defensive parsing: non-numeric `userIds` tokens are dropped +(`12,abc,45` → `12,45`), and a `startDate` / `endDate` matching no accepted format returns **HTTP +400** (`ValidationError`); an invalid `timezone` likewise returns **400**. + +Pagination follows JSON:API: `page[number]` is 1-based (default `1`), `page[size]` defaults to `20` +and is capped at `100`; out-of-bound or non-numeric values fall back to the defaults rather than +erroring. Sorting follows JSON:API `sort` on `timestamp`: `sort=-timestamp` (or absent/unrecognized) +is newest first, `sort=timestamp` is oldest first. Ties on equal timestamps fall back to insertion +order (the SQL store's auto-increment `id`), so paging is deterministic in either direction. + +### Correlation route + +`GET /forest/_audit-trail/correlation/{correlationKey}` returns `{ "data": [...] }` — the +operation(s) recorded under one `correlation_key` for a single record (usually one), oldest first, or +an empty array if none. Scoped through query params; same auth and gating as above. + +| query param | required | effect | +| ------------ | -------- | ------------------------------------------------------------ | +| `collection` | yes | collection the record belongs to (also the permission scope) | +| `recordId` | yes | packed record id to scope the lookup | + +A missing `collection` or `recordId` returns **HTTP 400** (`ValidationError`). + +### Batch correlation route + +`GET /forest/_audit-trail/correlations` returns `{ "data": [...] }` — a **flat** list of every record +whose `correlation_key` is in `correlationKeys`, scoped to one record (the client groups by +`correlation_key`). Same auth and gating; empty array when nothing matches. + +| query param | required | effect | +| ----------------- | -------- | ------------------------------------------------------------ | +| `correlationKeys` | yes\* | comma-separated keys; blank tokens are dropped | +| `collection` | yes | collection the record belongs to (also the permission scope) | +| `recordId` | yes | packed record id to scope the lookup | + +\* To dodge any URL length limit, the same path also accepts **`POST`** with a JSON body +`{ "correlationKeys": [...], "collection": "...", "recordId": "..." }` (the body array takes +precedence over the query param). An empty/absent key list returns `{ "data": [] }` without hitting +the store. A missing `collection` or `recordId` returns **HTTP 400** (`ValidationError`). ## What gets stored diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/in_memory_store.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/in_memory_store.rb index 2d70f676d..39e12d3ef 100644 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/in_memory_store.rb +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/in_memory_store.rb @@ -26,8 +26,23 @@ def count_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil matching(collection, record_id, user_ids, start_timestamp, end_timestamp).length end + def list_by_correlation(collection:, record_id:, correlation_key:) + by_correlation(collection, record_id) { |record| record.correlation_key == correlation_key } + end + + def list_by_correlations(collection:, record_id:, correlation_keys:) + by_correlation(collection, record_id) { |record| correlation_keys.include?(record.correlation_key) } + end + private + def by_correlation(collection, record_id) + matches = @records.select do |record| + record.collection == collection && record.record_id == record_id && yield(record) + end + sorted(matches, 'asc') + end + def matching(collection, record_id, user_ids, start_timestamp, end_timestamp) @records.select do |record| record.collection == collection && record.record_id == record_id && diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/log_store.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/log_store.rb index 30a610084..3f3c31c27 100644 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/log_store.rb +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/log_store.rb @@ -21,6 +21,14 @@ def list_by_record(**) def count_by_record(**) 0 end + + def list_by_correlation(**) + [] + end + + def list_by_correlations(**) + [] + end end end end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb index bd11ada90..a00245d5e 100644 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb @@ -40,6 +40,20 @@ def count_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil scope(collection, record_id, user_ids, start_timestamp, end_timestamp).count end + def list_by_correlation(collection:, record_id:, correlation_key:) + model.where(collection: collection, record_id: record_id, correlation_key: correlation_key) + .order(:timestamp, :id) + .map { |row| from_row(row) } + end + + def list_by_correlations(collection:, record_id:, correlation_keys:) + return [] if correlation_keys.empty? + + model.where(collection: collection, record_id: record_id, correlation_key: correlation_keys) + .order(:timestamp, :id) + .map { |row| from_row(row) } + end + private def scope(collection, record_id, user_ids, start_timestamp, end_timestamp) diff --git a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/in_memory_store_spec.rb b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/in_memory_store_spec.rb index 8a76cd91e..9ba250d75 100644 --- a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/in_memory_store_spec.rb +++ b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/in_memory_store_spec.rb @@ -76,6 +76,47 @@ def record(over = {}) expect(store.count_by_record(collection: 'accounts', record_id: '1')).to eq(3) expect(store.count_by_record(collection: 'accounts', record_id: '1', user_ids: [7])).to eq(2) end + + describe '#list_by_correlation' do + it 'returns entries under the key for the record, oldest first, scoped to collection/record' do + store.append(record(timestamp: '2026-01-02T00:00:00.000Z', correlation_key: 'req-1', + new_values: { 'n' => 2 })) + store.append(record(timestamp: '2026-01-01T00:00:00.000Z', correlation_key: 'req-1', + new_values: { 'n' => 1 })) + store.append(record(correlation_key: 'req-2', new_values: { 'n' => 9 })) + store.append(record(record_id: '2', correlation_key: 'req-1')) + store.append(record(collection: 'contacts', correlation_key: 'req-1')) + + history = store.list_by_correlation(collection: 'accounts', record_id: '1', correlation_key: 'req-1') + + expect(history.map(&:new_values)).to eq([{ 'n' => 1 }, { 'n' => 2 }]) + end + + it 'returns an empty array when nothing matches' do + store.append(record(correlation_key: 'req-1')) + + expect(store.list_by_correlation(collection: 'accounts', record_id: '1', correlation_key: 'nope')).to eq([]) + end + end + + describe '#list_by_correlations' do + it 'returns a flat list under any of the keys, oldest first' do + store.append(record(timestamp: '2026-01-03T00:00:00.000Z', correlation_key: 'a', new_values: { 'n' => 3 })) + store.append(record(timestamp: '2026-01-01T00:00:00.000Z', correlation_key: 'b', new_values: { 'n' => 1 })) + store.append(record(timestamp: '2026-01-02T00:00:00.000Z', correlation_key: 'a', new_values: { 'n' => 2 })) + store.append(record(timestamp: '2026-01-04T00:00:00.000Z', correlation_key: 'c', new_values: { 'n' => 9 })) + + history = store.list_by_correlations(collection: 'accounts', record_id: '1', correlation_keys: %w[a b]) + + expect(history.map(&:new_values)).to eq([{ 'n' => 1 }, { 'n' => 2 }, { 'n' => 3 }]) + end + + it 'returns an empty array for an empty key list' do + store.append(record(correlation_key: 'a')) + + expect(store.list_by_correlations(collection: 'accounts', record_id: '1', correlation_keys: [])).to eq([]) + end + end end end end diff --git a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb index d2595c6b3..58876c8fa 100644 --- a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb +++ b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb @@ -84,6 +84,34 @@ def connection expect(store.count_by_record(collection: 'accounts', record_id: '1', user_ids: [7])).to eq(2) end + it 'lists entries under a correlation key for the record, scoped and oldest first' do + store.append(record(record_id: '1', correlation_key: 'req-1', timestamp: '2026-01-01T00:00:02.000Z')) + store.append(record(record_id: '1', correlation_key: 'req-1', timestamp: '2026-01-01T00:00:01.000Z')) + store.append(record(record_id: '1', correlation_key: 'req-2')) + store.append(record(record_id: '2', correlation_key: 'req-1')) + + history = store.list_by_correlation(collection: 'accounts', record_id: '1', correlation_key: 'req-1') + expect(history.map(&:timestamp)).to eq(['2026-01-01T00:00:01.000Z', '2026-01-01T00:00:02.000Z']) + end + + it 'lists a flat history across multiple correlation keys, oldest first' do + store.append(record(correlation_key: 'a', timestamp: '2026-01-03T00:00:00.000Z')) + store.append(record(correlation_key: 'b', timestamp: '2026-01-01T00:00:00.000Z')) + store.append(record(correlation_key: 'a', timestamp: '2026-01-02T00:00:00.000Z')) + store.append(record(correlation_key: 'c', timestamp: '2026-01-04T00:00:00.000Z')) + + history = store.list_by_correlations(collection: 'accounts', record_id: '1', correlation_keys: %w[a b]) + expect(history.map(&:timestamp)).to eq( + ['2026-01-01T00:00:00.000Z', '2026-01-02T00:00:00.000Z', '2026-01-03T00:00:00.000Z'] + ) + end + + it 'returns an empty array for an empty correlation key list' do + store.append(record(correlation_key: 'a')) + + expect(store.list_by_correlations(collection: 'accounts', record_id: '1', correlation_keys: [])).to eq([]) + end + it 'tracks applied migrations and is idempotent across stores' do store.append(record) described_class.new(connection_string: { adapter: 'sqlite3', database: db.path }).append(record) From 28adb4e1459dbc9a758fea2fe813cc986db916b9 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Fri, 19 Jun 2026 10:10:06 +0200 Subject: [PATCH 04/35] fix(audit_trail): bind each SqlStore to its own model class Multiple SqlStore instances mutated the shared Sql::AuditLog.table_name, so stores against different Postgres schemas clobbered each other. Make AuditLog an abstract template and build a per-instance concrete subclass bound to the store's own qualified table. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../lib/forest_admin_audit_trail/sql/audit_log.rb | 9 ++++++--- .../forest_admin_audit_trail/stores/sql_store.rb | 13 +++++++++---- .../stores/sql_store_spec.rb | 15 ++++++++++++++- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_log.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_log.rb index 725260935..f2b09487d 100644 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_log.rb +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_log.rb @@ -1,9 +1,12 @@ module ForestAdminAuditTrail module Sql - # ActiveRecord model for the audit table. Its table name (and schema) is assigned at runtime by - # the SqlStore once the connection and migrations are ready. The JSON attribute overrides force - # Hash <-> JSON casting on every adapter (Postgres json, or text on SQLite). + # Abstract template for the audit model. Each SqlStore builds its own concrete subclass bound to + # its own (schema-qualified) table, so stores with different `table_name`/`schema` no longer + # clobber a shared class. The JSON attribute overrides force Hash <-> JSON casting on every + # adapter (Postgres json, or text on SQLite) and are inherited by every subclass. class AuditLog < AuditConnectionBase + self.abstract_class = true + attribute :previous_values, :json attribute :new_values, :json end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb index a00245d5e..f64c28e6c 100644 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb @@ -72,7 +72,7 @@ def as_time(value) def model ensure_ready - Sql::AuditLog + @model end def ensure_ready @@ -84,13 +84,18 @@ def ensure_ready Sql::AuditConnectionBase.establish_connection(@connection_string) connection = Sql::AuditConnectionBase.connection Sql::Migrator.new(connection, schema: schema_for(connection), table_name: @table_name).run - Sql::AuditLog.table_name = qualified(connection) - # The table was just created at runtime: refresh the model's cached column metadata. - Sql::AuditLog.reset_column_information + @model = build_model(qualified(connection)) @ready = true end end + # A per-instance concrete subclass bound to this store's own table, so distinct stores can't + # clobber each other's table name. reset_column_information drops stale metadata for the table + # the migration just created/evolved. + def build_model(table) + Class.new(Sql::AuditLog) { self.table_name = table }.tap(&:reset_column_information) + end + def schema_for(connection) connection.adapter_name.downcase.include?('postgres') ? @schema : nil end diff --git a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb index 58876c8fa..60b29d64a 100644 --- a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb +++ b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb @@ -27,7 +27,7 @@ def connection it 'creates the audit table with the expected columns on first write' do store.append(record) - expect(ForestAdminAuditTrail::Sql::AuditLog.column_names.sort).to eq( + expect(store.send(:model).column_names.sort).to eq( %w[collection correlation_key id new_values operation previous_values record_id timestamp user_id] ) end @@ -120,6 +120,19 @@ def connection expect(names).to eq(['001-create-audit-logs', '002-index-record-and-correlation']) end + it 'binds each store to its own model class instead of mutating a shared one' do + other = described_class.new(connection_string: { adapter: 'sqlite3', database: db.path }) + store.append(record(correlation_key: 'main')) + other.append(record(correlation_key: 'other')) + + # No shared mutable model: AuditLog is an abstract template, each store owns a distinct subclass. + expect(ForestAdminAuditTrail::Sql::AuditLog.abstract_class?).to be(true) + expect(store.send(:model)).not_to equal(other.send(:model)) + expect(store.send(:model).table_name).to eq('audit_logs') + expect(store.list_by_record(collection: 'accounts', record_id: '1').map(&:correlation_key)) + .to eq(%w[main other]) + end + it 'indexes record_id, correlation_key and user_id' do store.append(record) From d4574fb5858419de390392a8ab90268b3af1c7d6 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Fri, 19 Jun 2026 10:10:16 +0200 Subject: [PATCH 05/35] fix(audit_trail): make migration DDL idempotent for non-pg race Without the Postgres advisory lock, two instances booting at once can both see a migration as pending and both run its create_table/add_index, crashing the loser with "already exists". Add if_not_exists: true to the table and index DDL. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../lib/forest_admin_audit_trail/sql/migrator.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb index 5da907848..25b1bc66f 100644 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb @@ -16,7 +16,8 @@ class Migrator { name: '001-create-audit-logs', up: lambda do |connection, table| - connection.create_table(table) do |t| + # if_not_exists: a non-PG race (no advisory lock) can let two instances both reach here. + connection.create_table(table, if_not_exists: true) do |t| t.datetime :timestamp, null: false t.string :operation, null: false t.string :collection, null: false @@ -32,9 +33,9 @@ class Migrator name: '002-index-record-and-correlation', up: lambda do |connection, table| base = table.split('.').last - connection.add_index(table, :record_id, name: "#{base}_record_id") - connection.add_index(table, :correlation_key, name: "#{base}_correlation_key") - connection.add_index(table, :user_id, name: "#{base}_user_id") + connection.add_index(table, :record_id, name: "#{base}_record_id", if_not_exists: true) + connection.add_index(table, :correlation_key, name: "#{base}_correlation_key", if_not_exists: true) + connection.add_index(table, :user_id, name: "#{base}_user_id", if_not_exists: true) end } ].freeze From 552b63561ed3e4817db013b0ff15b68ad64e501f Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Fri, 19 Jun 2026 10:10:25 +0200 Subject: [PATCH 06/35] fix(audit_trail): treat a blank schema as no schema An empty-string schema made schema? true, producing invalid identifiers like ".audit_migrations". Use present? so nil, "" and whitespace all mean no schema qualification. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../lib/forest_admin_audit_trail/sql/migrator.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb index 25b1bc66f..b29e3dde9 100644 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb +++ b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb @@ -66,7 +66,7 @@ def postgres? end def schema? - postgres? && !@schema.nil? + postgres? && @schema.present? end def qualified(name) From 4869ca2cf8752bb2d7a2f9ff4e4501d3d2a6cb38 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Mon, 10 Aug 2026 16:00:18 +0200 Subject: [PATCH 07/35] refactor(audit_trail): move the plugin into the agent, gated on the database The audit trail is no longer a separate gem to install and wire up: capture, storage and the record-history routes now live in forest_admin_agent and turn on as soon as `config.audit_trail[:database]` is set. - the agent factory builds the store from the configured database and installs the capture layer through the customizer, so `reload!` replays it - drop LogStore and InMemoryStore: with no database the feature stays off, and the specs cover the store through SQLite - AuditRecord becomes a Struct, and the capture hooks call their own methods instead of going through `send` - ActiveRecord is required lazily, so agents without an audit database never load it Co-Authored-By: Claude Opus 5 (1M context) --- .releaserc.js | 7 +- .rubocop.yml | 12 +- .../AUDIT_TRAIL.md} | 129 +++++--------- packages/forest_admin_agent/Gemfile | 2 + packages/forest_admin_agent/Gemfile-test | 2 + .../lib/forest_admin_agent.rb | 1 + .../audit_trail/audit_record.rb | 11 ++ .../forest_admin_agent/audit_trail/capture.rb | 136 +++++++++++++++ .../forest_admin_agent/audit_trail/diff.rb | 83 +++++++++ .../audit_trail/sql/audit_connection_base.rb | 19 ++ .../audit_trail/sql/audit_log.rb | 16 ++ .../audit_trail/sql/migrator.rb | 121 +++++++++++++ .../forest_admin_agent/audit_trail/store.rb} | 16 +- .../builder/agent_factory.rb | 19 ++ .../routes/resources/audit_trail.rb | 4 +- .../resources/audit_trail_correlation.rb | 5 +- .../audit_trail/capture_spec.rb | 127 ++++++++++++++ .../audit_trail/diff_spec.rb | 54 ++++++ .../audit_trail/store_spec.rb} | 18 +- .../builder/agent_factory_spec.rb | 47 +++++ .../forest_admin_agent/spec/spec_helper.rb | 2 + packages/forest_admin_audit_trail/.gitignore | 7 - packages/forest_admin_audit_trail/.rspec | 3 - packages/forest_admin_audit_trail/Gemfile | 20 --- packages/forest_admin_audit_trail/Rakefile | 10 -- .../forest_admin_audit_trail.gemspec | 37 ---- .../lib/forest_admin_audit_trail.rb | 10 -- .../forest_admin_audit_trail/audit_record.rb | 45 ----- .../lib/forest_admin_audit_trail/diff.rb | 81 --------- .../lib/forest_admin_audit_trail/plugin.rb | 163 ------------------ .../sql/audit_connection_base.rb | 11 -- .../forest_admin_audit_trail/sql/audit_log.rb | 14 -- .../forest_admin_audit_trail/sql/migrator.rb | 119 ------------- .../stores/in_memory_store.rb | 67 ------- .../stores/log_store.rb | 34 ---- .../lib/forest_admin_audit_trail/version.rb | 3 - .../lib/forest_admin_audit_trail/diff_spec.rb | 52 ------ .../forest_admin_audit_trail/plugin_spec.rb | 121 ------------- .../stores/in_memory_store_spec.rb | 122 ------------- .../spec/spec_helper.rb | 25 --- .../lib/forest_admin_rails.rb | 4 +- 41 files changed, 713 insertions(+), 1066 deletions(-) rename packages/{forest_admin_audit_trail/README.md => forest_admin_agent/AUDIT_TRAIL.md} (61%) create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_log.rb create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb rename packages/{forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb => forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb} (89%) create mode 100644 packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb create mode 100644 packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb rename packages/{forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb => forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb} (91%) delete mode 100644 packages/forest_admin_audit_trail/.gitignore delete mode 100644 packages/forest_admin_audit_trail/.rspec delete mode 100644 packages/forest_admin_audit_trail/Gemfile delete mode 100644 packages/forest_admin_audit_trail/Rakefile delete mode 100644 packages/forest_admin_audit_trail/forest_admin_audit_trail.gemspec delete mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail.rb delete mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/audit_record.rb delete mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/diff.rb delete mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/plugin.rb delete mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_connection_base.rb delete mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_log.rb delete mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb delete mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/in_memory_store.rb delete mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/log_store.rb delete mode 100644 packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb delete mode 100644 packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/diff_spec.rb delete mode 100644 packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/plugin_spec.rb delete mode 100644 packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/in_memory_store_spec.rb delete mode 100644 packages/forest_admin_audit_trail/spec/spec_helper.rb diff --git a/.releaserc.js b/.releaserc.js index 920227235..76e57c8f0 100644 --- a/.releaserc.js +++ b/.releaserc.js @@ -31,8 +31,7 @@ module.exports = { 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb; '+ 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb; '+ 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb; '+ - 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb; '+ - 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb; ', + 'sed -i \'s/VERSION = ".*"/VERSION = "${nextRelease.version}"/g\' packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb; ', successCmd: '( cd packages/forest_admin_agent && gem build && gem push forest_admin_agent-*.gem );' + '( cd packages/forest_admin_datasource_active_record && gem build && gem push forest_admin_datasource_active_record-*.gem );' + @@ -46,8 +45,7 @@ module.exports = { '( cd packages/forest_admin_datasource_zendesk && gem build && gem push forest_admin_datasource_zendesk-*.gem );' + '( cd packages/forest_admin_datasource_snowflake && gem build && gem push forest_admin_datasource_snowflake-*.gem );' + '( cd packages/forest_admin_datasource_mambu_payments && gem build && gem push forest_admin_datasource_mambu_payments-*.gem );' + - '( cd packages/forest_admin_datasource_graphql_hasura && gem build && gem push forest_admin_datasource_graphql_hasura-*.gem );' + - '( cd packages/forest_admin_audit_trail && gem build && gem push forest_admin_audit_trail-*.gem );' , + '( cd packages/forest_admin_datasource_graphql_hasura && gem build && gem push forest_admin_datasource_graphql_hasura-*.gem );' , }, ], [ @@ -70,7 +68,6 @@ module.exports = { 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb', 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb', 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/version.rb', - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb', 'package.json' ], }, diff --git a/.rubocop.yml b/.rubocop.yml index 6f56e1247..42d7fe5c2 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -28,7 +28,6 @@ Gemspec/OrderedDependencies: Gemspec/RequireMFA: Exclude: - - 'packages/forest_admin_audit_trail/forest_admin_audit_trail.gemspec' - 'agent_ruby.gemspec' - 'packages/forest_admin_agent/forest_admin_agent.gemspec' - 'packages/forest_admin_rails/forest_admin_rails.gemspec' @@ -134,7 +133,6 @@ Style/MutableConstant: - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb' - 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb' - - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb' # Offense count: 38 # This cop supports safe autocorrection (--autocorrect). @@ -219,7 +217,6 @@ Style/StringLiterals: - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/version.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/version.rb' - 'packages/forest_admin_datasource_mambu_payments/lib/forest_admin_datasource_mambu_payments/version.rb' - - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb' # Offense count: 1 # This cop supports safe autocorrection (--autocorrect). @@ -260,11 +257,9 @@ Naming/PredicatePrefix: Metrics/ParameterLists: Exclude: + - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb' + - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/configuration.rb' - - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/audit_record.rb' - - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/plugin.rb' - - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/in_memory_store.rb' - - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb' - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/collections/base_collection.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/datasource.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/routes/query_handler.rb' @@ -369,7 +364,6 @@ Metrics/ClassLength: - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/introspection/schema_converter.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/query/query_builder.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/query/aggregator.rb' - - 'packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/plugin.rb' - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/collections/base_collection.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/collection.rb' - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/datasource.rb' @@ -462,7 +456,7 @@ Layout/LineLength: RSpec/VerifiedDoubles: Exclude: - - 'packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/plugin_spec.rb' + - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb' - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb' - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb' - 'packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/composite_datasource_spec.rb' diff --git a/packages/forest_admin_audit_trail/README.md b/packages/forest_admin_agent/AUDIT_TRAIL.md similarity index 61% rename from packages/forest_admin_audit_trail/README.md rename to packages/forest_admin_agent/AUDIT_TRAIL.md index 9156f30a9..630139c7f 100644 --- a/packages/forest_admin_audit_trail/README.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -1,92 +1,66 @@ -# forest_admin_audit_trail +# Audit trail Capture who changed what (before/after) for every change Forest performs through its data layer, and -persist it into a SQL database. Ruby port of the Node `@forestadmin/plugin-audit-trail`. +persist it into a SQL database. Built into the agent: it turns on as soon as an audit-trail **database +is configured**, and stays completely off otherwise. -Two decoupled parts: +Two parts, both internal: -- **Capture** — datasource-agnostic. It instruments every collection through the Forest customizer - hooks, so it works the same whether the audited datasource is ActiveRecord, Mongoid, etc. -- **Storage** — where the records are written. Ships a **SQL store** (ActiveRecord) that creates the - `forest` schema and creates/evolves the `audit_logs` table through versioned migrations, plus an - in-memory store for tests. +- **Capture** (`ForestAdminAgent::AuditTrail::Capture`) — datasource-agnostic. It instruments every + collection through the customizer hooks, so it behaves the same whether the audited datasource is + ActiveRecord, Mongoid, etc. +- **Storage** (`ForestAdminAgent::AuditTrail::Store`) — ActiveRecord-backed. It creates the `forest` + schema and creates/evolves the `audit_logs` table through versioned migrations, and reads the + per-record history back for the routes below. -## Use it in an agent +Storage uses ActiveRecord: outside Rails, add `gem 'activerecord'` (and the adapter gem) to your +Gemfile. Nothing is loaded and no connection is opened until the feature is configured. -`ForestAdminAuditTrail::Stores::SqlStore` both **writes** every audited change and **reads** the -per-record history back. The integration has two halves that must share the **same** store instance: - -- the agent `audit_trail` option — so the record-history routes are mounted and read from the table; -- `agent.use(ForestAdminAuditTrail::Plugin, store:)` — so every change is captured into that table. - -Add the gem: - -```ruby -# Gemfile -gem 'forest_admin_audit_trail' -``` +## Turn it on ### Rails (forest_admin_rails) -Build the store once in an initializer and hand it to the agent through `config.audit_trail`. A -constant keeps the single instance reachable from the agent factory: - ```ruby # config/initializers/forest_admin_rails.rb -AUDIT_TRAIL_STORE = ForestAdminAuditTrail::Stores::SqlStore.new( - connection_string: { # or any ActiveRecord config / URL - adapter: 'postgresql', host: ENV['AUDIT_DB_HOST'], port: ENV['AUDIT_DB_PORT'], - username: ENV['AUDIT_DB_USER'], password: ENV['AUDIT_DB_PASSWORD'], database: ENV['AUDIT_DB_NAME'] - } -) - ForestAdminRails.configure do |config| config.auth_secret = ENV['FOREST_AUTH_SECRET'] config.env_secret = ENV['FOREST_ENV_SECRET'] - # exposes GET /forest/_audit-trail/{collection}/:id and the correlation routes below - config.audit_trail = { store: AUDIT_TRAIL_STORE } -end -``` - -Then install the capture plugin where you build the agent (`lib/forest_admin_rails/create_agent.rb`), -reading the **same** store back from the config so writes and reads agree: -```ruby -# lib/forest_admin_rails/create_agent.rb -@agent = ForestAdminAgent::Builder::AgentFactory.instance - .add_datasource(my_datasource) - -@agent.use(ForestAdminAuditTrail::Plugin, store: ForestAdminRails.config.audit_trail[:store]) - -@agent.build + config.audit_trail = { + database: { # or an ActiveRecord URL: ENV['AUDIT_TRAIL_DATABASE_URL'] + adapter: 'postgresql', host: ENV['AUDIT_DB_HOST'], port: ENV['AUDIT_DB_PORT'], + username: ENV['AUDIT_DB_USER'], password: ENV['AUDIT_DB_PASSWORD'], database: ENV['AUDIT_DB_NAME'] + } + } +end ``` ### Plain agent (no Rails) ```ruby -store = ForestAdminAuditTrail::Stores::SqlStore.new(connection_string: ENV['AUDIT_TRAIL_DATABASE_URL']) - ForestAdminAgent::Builder::AgentFactory.instance.setup( auth_secret: ENV['FOREST_AUTH_SECRET'], env_secret: ENV['FOREST_ENV_SECRET'], # ...usual options... - audit_trail: { store: store } + audit_trail: { database: ENV['AUDIT_TRAIL_DATABASE_URL'] } ) - -agent.use(ForestAdminAuditTrail::Plugin, store: store) ``` -On the first write or read the store ensures the `forest` schema exists and runs any pending -migrations to create/upgrade `forest.audit_logs`; every create / update / delete performed through -Forest then writes one row per record, and the **Historic** tab in the UI reads from the same table. +| option | description | +| ------------ | ------------------------------------------------------------------------------------ | +| `database` | ActiveRecord URL or config hash. **Setting it activates the audit trail.** | +| `schema` | Postgres schema holding the table (default `forest`; ignored on other adapters) | +| `table_name` | default `audit_logs` | +| `redact` | `{ 'collection_name' => ['field', ...] }` — values masked while recording the change | -> The record-history routes are mounted **only** when `audit_trail[:store]` is set. Leave it unset -> (the default) and the feature stays off — the plugin can still write to a `LogStore` if used alone. +On the first write or read the store ensures the schema exists and runs any pending migrations; every +create / update / delete performed through Forest then writes one row per record, and the **Historic** +tab in the UI reads from the same table. ## Routes -All routes live under `/forest/_audit-trail`, are gated on `audit_trail[:store]` being configured, -and require read permission on the target collection (`can?(:read, collection)`). +All routes live under `/forest/_audit-trail`, are registered only when `audit_trail[:database]` is +set, and require read permission on the target collection (`can?(:read, collection)`). ### Record-history route @@ -122,7 +96,7 @@ Pagination follows JSON:API: `page[number]` is 1-based (default `1`), `page[size and is capped at `100`; out-of-bound or non-numeric values fall back to the defaults rather than erroring. Sorting follows JSON:API `sort` on `timestamp`: `sort=-timestamp` (or absent/unrecognized) is newest first, `sort=timestamp` is oldest first. Ties on equal timestamps fall back to insertion -order (the SQL store's auto-increment `id`), so paging is deterministic in either direction. +order (the auto-increment `id`), so paging is deterministic in either direction. ### Correlation route @@ -158,37 +132,26 @@ the store. A missing `collection` or `recordId` returns **HTTP 400** (`Validatio `forest.audit_logs`, one row per audited change: -| column | description | -| ----------------- | ---------------------------------------------------- | -| `id` | auto-increment primary key | -| `timestamp` | when the change happened | -| `operation` | `create` / `update` / `delete` | -| `collection` | audited collection name | -| `record_id` | packed record id (primary keys joined by `\|`) | -| `user_id` | the Forest user who made the change | -| `correlation_key` | per-request id; groups every change made within one request | -| `previous_values` | values before the change (JSON) | -| `new_values` | values after the change (JSON) | +| column | description | +| ----------------- | ----------------------------------------------------------- | +| `id` | auto-increment primary key | +| `timestamp` | when the change happened | +| `operation` | `create` / `update` / `delete` | +| `collection` | audited collection name | +| `record_id` | packed record id (primary keys joined by `\|`) | +| `user_id` | the Forest user who made the change | +| `correlation_key` | per-request id; groups every change made within one request | +| `previous_values` | values before the change (JSON) | +| `new_values` | values after the change (JSON) | `previous_values` / `new_values` store **only the parts that actually changed**: nested hashes and -arrays of hashes are diffed structurally, so a single sub-field change records just that leaf. +arrays of hashes are diffed structurally, so a single sub-field change records just that leaf. Only +writable columns are audited — read-only, computed and DB-managed fields are never written by Forest. The `correlation_key` is the agent's per-request id (`caller.request_id`), generated by the agent and echoed back to the client in the `X-Forest-Correlation-Id` response header — so every change made in one request shares a key, and the caller can tie it to its own activity log. -## Options - -`ForestAdminAuditTrail::Plugin` options: - -| option | description | -| -------- | ------------------------------------------------------------------------------------ | -| `store` | object responding to `append(record)`. Defaults to a store that only logs. | -| `redact` | `{ 'collection_name' => ['field', ...] }` — values masked while recording the change | - -`SqlStore.new` options: `connection_string:` (required, AR URL or config hash), `schema:` -(default `forest`), `table_name:` (default `audit_logs`). - ## Schema migrations & concurrency The table is created/evolved through an ordered, append-only migration list tracked in a dedicated diff --git a/packages/forest_admin_agent/Gemfile b/packages/forest_admin_agent/Gemfile index 0e0b74330..d22fe37a2 100644 --- a/packages/forest_admin_agent/Gemfile +++ b/packages/forest_admin_agent/Gemfile @@ -3,6 +3,7 @@ source "https://rubygems.org" gemspec group :development, :test do + gem 'activerecord', '>= 6.1' gem 'forest_admin_datasource_customizer', path: '../forest_admin_datasource_customizer' gem 'forest_admin_datasource_toolkit', path: '../forest_admin_datasource_toolkit' gem 'forest_admin_test_toolkit', path: '../forest_admin_test_toolkit' @@ -13,4 +14,5 @@ group :development, :test do gem 'simplecov', '~> 0.22', require: false gem 'simplecov-html', '~> 0.12.3' gem 'simplecov_json_formatter', '~> 0.1.4' + gem 'sqlite3', '>= 2.1' end diff --git a/packages/forest_admin_agent/Gemfile-test b/packages/forest_admin_agent/Gemfile-test index 0e0b74330..d22fe37a2 100644 --- a/packages/forest_admin_agent/Gemfile-test +++ b/packages/forest_admin_agent/Gemfile-test @@ -3,6 +3,7 @@ source "https://rubygems.org" gemspec group :development, :test do + gem 'activerecord', '>= 6.1' gem 'forest_admin_datasource_customizer', path: '../forest_admin_datasource_customizer' gem 'forest_admin_datasource_toolkit', path: '../forest_admin_datasource_toolkit' gem 'forest_admin_test_toolkit', path: '../forest_admin_test_toolkit' @@ -13,4 +14,5 @@ group :development, :test do gem 'simplecov', '~> 0.22', require: false gem 'simplecov-html', '~> 0.12.3' gem 'simplecov_json_formatter', '~> 0.1.4' + gem 'sqlite3', '>= 2.1' end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent.rb b/packages/forest_admin_agent/lib/forest_admin_agent.rb index be8270393..f375035bc 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent.rb @@ -4,6 +4,7 @@ loader = Zeitwerk::Loader.for_gem loader.inflector.inflect('oauth2' => 'OAuth2') +loader.inflector.inflect('sql' => 'Sql') loader.inflector.inflect('sse_cache_invalidation' => 'SSECacheInvalidation') loader.setup diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb new file mode 100644 index 000000000..011d4238a --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb @@ -0,0 +1,11 @@ +module ForestAdminAgent + module AuditTrail + # One audited change. Mirrors the columns of `forest.audit_logs`; only the actor's `user_id` is + # stored, the rest of the actor identity is correlated elsewhere through `correlation_key`. + AuditRecord = Struct.new( + :timestamp, :operation, :collection, :record_id, :user_id, :correlation_key, + :previous_values, :new_values, + keyword_init: true + ) + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb new file mode 100644 index 000000000..f0f447307 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb @@ -0,0 +1,136 @@ +require 'securerandom' +require 'time' + +module ForestAdminAgent + module AuditTrail + # Datasource-agnostic capture layer, installed by the agent factory as soon as an audit-trail + # database is configured. It instruments every collection through the Forest customizer hooks, so + # it behaves the same whatever the audited datasource is (ActiveRecord, Mongoid, ...), computes the + # minimal before/after diff for each change and appends an {AuditRecord} to the store. + class Capture + REDACTED = '[redacted]'.freeze + + # Signature imposed by DatasourceCustomizer#use; the agent always instruments the whole datasource. + def run(datasource_customizer, _collection_customizer = nil, options = {}) + @store = options[:store] + @redact = options[:redact] || {} + + datasource_customizer.collections.each_value { |collection| instrument(collection) } + end + + private + + def instrument(collection_customizer) + schema = collection_customizer.collection.schema + # Writable columns only: Forest audits what it writes. Read-only fields cover computed/virtual + # fields and DB-managed columns, none of which Forest mutates. + columns = schema[:fields].select do |_name, field| + field.type == 'Column' && !field.is_read_only + end.keys + primary_keys = ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(collection_customizer.collection) + # Reads must carry the primary keys (even read-only ones) so the record id can be built; the + # diff itself stays restricted to the writable columns. + projection = ForestAdminDatasourceToolkit::Components::Query::Projection.new( + (primary_keys + columns).uniq + ) + name = collection_customizer.name + + add_create_hook(collection_customizer, columns, primary_keys, name) + add_update_hooks(collection_customizer, columns, primary_keys, name, projection) + add_delete_hooks(collection_customizer, columns, primary_keys, name, projection) + end + + def add_create_hook(collection_customizer, columns, primary_keys, name) + collection_customizer.add_hook('After', 'Create') do |context| + emit( + context.caller, 'create', name, record_id(context.record, primary_keys), + {}, pick(context.record, columns) + ) + end + end + + def add_update_hooks(collection_customizer, columns, primary_keys, name, projection) + collection_customizer.add_hook('Before', 'Update') do |context| + pending[context.filter] = context.collection.list(context.filter, projection) + end + + collection_customizer.add_hook('After', 'Update') do |context| + before = pending.delete(context.filter) || [] + patch = context.patch + + before.each do |record| + delta = Diff.changed_values(record, patch, columns) + next if delta[:new_values].empty? + + emit( + context.caller, 'update', name, record_id(record, primary_keys), + delta[:previous_values], delta[:new_values] + ) + end + end + end + + def add_delete_hooks(collection_customizer, columns, primary_keys, name, projection) + collection_customizer.add_hook('Before', 'Delete') do |context| + pending[context.filter] = context.collection.list(context.filter, projection) + end + + collection_customizer.add_hook('After', 'Delete') do |context| + before = pending.delete(context.filter) || [] + + before.each do |record| + emit( + context.caller, 'delete', name, record_id(record, primary_keys), + pick(record, columns), {} + ) + end + end + end + + # Snapshots taken in a "before" hook and consumed in the matching "after" hook. Keyed by the + # filter object, which the hook decorator passes unchanged to both hooks within the same thread. + def pending + Thread.current[:forest_audit_trail_snapshots] ||= {}.compare_by_identity + end + + def emit(caller, operation, collection, record_id, previous_values, new_values) + redacted = @redact[collection] || [] + + @store.append( + AuditRecord.new( + timestamp: Time.now.utc.iso8601(3), + operation: operation, + collection: collection, + record_id: record_id, + user_id: caller&.id, + # Same id for every change made within one request — set on the caller by the agent + # (see CallerParser), mirroring the Node agent's caller.requestId. + correlation_key: correlation_key_for(caller), + previous_values: redact(previous_values, redacted), + new_values: redact(new_values, redacted) + ) + ) + end + + def correlation_key_for(caller) + (caller.respond_to?(:request_id) && caller.request_id) || SecureRandom.uuid + end + + def record_id(record, primary_keys) + primary_keys.map { |pk| record[pk].to_s }.join('|') + end + + def pick(record, columns) + columns.to_h { |column| [column, record[column]] } + end + + def redact(values, redacted_fields) + return values if redacted_fields.empty? + + values.each_with_object({}) do |(field, value), result| + result[field] = redacted_fields.include?(field) ? REDACTED : value + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb new file mode 100644 index 000000000..cd6d30095 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb @@ -0,0 +1,83 @@ +module ForestAdminAgent + module AuditTrail + # Minimal structural diff. Nested hashes and arrays of hashes are recursed into, so only the + # keys/indexes whose leaf value actually changed are kept — a single sub-field change does not + # store the whole object/array. Scalars, primitive arrays, dates and other values are compared and + # kept as a whole. + # + # Ruby's `==` already performs deep, key-order-independent equality on Hash and (ordered) equality + # on Array, so it is used directly as the equality primitive. + module Diff + module_function + + # @return [Hash{Symbol=>Object}, nil] { previous:, next: } of the changed leaves, or nil when equal. + def diff(before, after) + return nil if before == after + + return diff_hashes(before, after) if before.is_a?(Hash) && after.is_a?(Hash) + + return diff_object_arrays(before, after) if object_array?(before) && object_array?(after) + + { previous: before.nil? ? nil : before, next: after.nil? ? nil : after } + end + + # Build the previous/new value hashes for the writable columns that actually changed. + # + # @param before [Hash] snapshot of the record before the change (string keys) + # @param patch [Hash] the values being written (string keys); only present keys are considered + # @param columns [Array] writable column names to inspect + # @return [Hash{Symbol=>Hash}] { previous_values:, new_values: } + def changed_values(before, patch, columns) + previous_values = {} + new_values = {} + + columns.each do |column| + delta = patch.key?(column) ? diff(before[column], patch[column]) : nil + next unless delta + + previous_values[column] = delta[:previous] + new_values[column] = delta[:next] + end + + { previous_values: previous_values, new_values: new_values } + end + + # Arrays whose every element is a hash (record-like collections, e.g. a workflow history). + def object_array?(value) + value.is_a?(Array) && !value.empty? && value.all?(Hash) + end + + def diff_hashes(before, after) + previous = {} + next_values = {} + + (before.keys | after.keys).each do |key| + sub = diff(before[key], after[key]) + next unless sub + + previous[key] = sub[:previous] + next_values[key] = sub[:next] + end + + { previous: previous, next: next_values } + end + + def diff_object_arrays(before, after) + previous = {} + next_values = {} + + [before.length, after.length].max.times do |index| + sub = diff(before[index], after[index]) + next unless sub + + previous[index] = sub[:previous] + next_values[index] = sub[:next] + end + + { previous: previous, next: next_values } + end + + private_class_method :diff_hashes, :diff_object_arrays + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb new file mode 100644 index 000000000..9005f1c40 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb @@ -0,0 +1,19 @@ +begin + require 'active_record' +rescue LoadError + raise LoadError, 'config.audit_trail needs the activerecord gem: add `gem "activerecord"` to your Gemfile.' +end + +module ForestAdminAgent + module AuditTrail + module Sql + # Dedicated abstract base so the audit storage keeps its own connection pool, isolated from the + # host application's ActiveRecord::Base connection. Also the level the `attribute` overrides in + # AuditLog need: declaring them straight on an ActiveRecord::Base child resolves the type + # eagerly and blows up before any connection is established. + class AuditConnectionBase < ActiveRecord::Base + self.abstract_class = true + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_log.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_log.rb new file mode 100644 index 000000000..a0eb7c442 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_log.rb @@ -0,0 +1,16 @@ +module ForestAdminAgent + module AuditTrail + module Sql + # Abstract template for the audit model. Each Store builds its own concrete subclass bound to its + # own (schema-qualified) table, so stores with different `table_name`/`schema` can't clobber a + # shared one. The JSON attribute overrides force Hash <-> JSON casting on every adapter (Postgres + # json, or text on SQLite) and are inherited by every subclass. + class AuditLog < AuditConnectionBase + self.abstract_class = true + + attribute :previous_values, :json + attribute :new_values, :json + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb new file mode 100644 index 000000000..36c9aab73 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb @@ -0,0 +1,121 @@ +module ForestAdminAgent + module AuditTrail + module Sql + # Creates and evolves the audit table through an ordered, append-only list of migrations, tracked + # in a dedicated `audit_migrations` table (namespaced in the `forest` schema on Postgres). + # + # On Postgres the migrations run inside a transaction-scoped advisory lock, so several agent + # instances booting at once apply them one after another instead of racing on the same DDL. The + # schema is created (and committed) first, made idempotent (CREATE SCHEMA IF NOT EXISTS + + # tolerating a concurrent create), because the lock cannot cover a not-yet-existing schema. + class Migrator + MIGRATIONS_TABLE = 'audit_migrations'.freeze + # Arbitrary but stable key pair identifying the audit-trail migration critical section. + ADVISORY_LOCK = [0x464f, 0x5254].freeze # "FO", "RT" + + MIGRATIONS = [ + { + name: '001-create-audit-logs', + up: lambda do |connection, table| + # if_not_exists: a non-PG race (no advisory lock) can let two instances both reach here. + connection.create_table(table, if_not_exists: true) do |t| + t.datetime :timestamp, null: false + t.string :operation, null: false + t.string :collection, null: false + t.string :record_id, null: false + t.integer :user_id + t.string :correlation_key + t.json :previous_values + t.json :new_values + end + end + }, + { + name: '002-index-record-and-correlation', + up: lambda do |connection, table| + base = table.split('.').last + connection.add_index(table, :record_id, name: "#{base}_record_id", if_not_exists: true) + connection.add_index(table, :correlation_key, name: "#{base}_correlation_key", if_not_exists: true) + connection.add_index(table, :user_id, name: "#{base}_user_id", if_not_exists: true) + end + } + ].freeze + + def initialize(connection, schema:, table_name:) + @connection = connection + @schema = schema # nil on adapters without schema support + @table_name = table_name + end + + def run + ensure_schema + + if postgres? + @connection.transaction do + @connection.execute("SELECT pg_advisory_xact_lock(#{ADVISORY_LOCK[0]}, #{ADVISORY_LOCK[1]})") + apply_pending + end + else + apply_pending + end + end + + private + + def postgres? + @connection.adapter_name.downcase.include?('postgres') + end + + def schema? + postgres? && @schema.present? + end + + def qualified(name) + schema? ? "#{@schema}.#{name}" : name + end + + # Create the schema first and commit it: the migrations open DDL on the same connection, and a + # CREATE SCHEMA still pending in the lock transaction would not be visible to them. + def ensure_schema + return unless schema? + + @connection.execute("CREATE SCHEMA IF NOT EXISTS #{@connection.quote_schema_name(@schema)}") + rescue ActiveRecord::StatementInvalid => e + # Another instance created it concurrently — CREATE SCHEMA is not fully race-free. + raise unless /already exists|duplicate/i.match?(e.message) + end + + def apply_pending + done = applied_migrations + table = qualified(@table_name) + + MIGRATIONS.each do |migration| + next if done.include?(migration[:name]) + + migration[:up].call(@connection, table) + @connection.execute( + "INSERT INTO #{@connection.quote_table_name(migrations_table)} (name) " \ + "VALUES (#{@connection.quote(migration[:name])})" + ) + end + end + + def applied_migrations + ensure_migrations_table + + @connection.select_values("SELECT name FROM #{@connection.quote_table_name(migrations_table)}") + end + + def ensure_migrations_table + @connection.create_table(migrations_table, id: false, if_not_exists: true) do |t| + t.string :name, null: false + end + end + + def migrations_table + qualified(MIGRATIONS_TABLE) + end + end + end + end +end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb similarity index 89% rename from packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb rename to packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb index f64c28e6c..8c5d9b4e7 100644 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/sql_store.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb @@ -1,19 +1,17 @@ require 'time' -module ForestAdminAuditTrail - module Stores - # SQL-backed store that both writes every audited change and reads the per-record history back. +module ForestAdminAgent + module AuditTrail + # SQL-backed storage that both writes every audited change and reads the per-record history back. # # Construction is cheap; the connection is opened lazily on the first append or read, at which # point the `forest` schema and the `audit_logs` table are created/evolved through migrations. - # Pass the SAME instance to the plugin (`store:`) and to the agent (`audit_trail: { store: }`) so - # writes and the record-history route agree on storage. - class SqlStore + class Store DEFAULT_SCHEMA = 'forest'.freeze DEFAULT_TABLE = 'audit_logs'.freeze - def initialize(connection_string:, schema: DEFAULT_SCHEMA, table_name: DEFAULT_TABLE) - @connection_string = connection_string + def initialize(database:, schema: DEFAULT_SCHEMA, table_name: DEFAULT_TABLE) + @database = database @schema = schema @table_name = table_name @mutex = Mutex.new @@ -81,7 +79,7 @@ def ensure_ready @mutex.synchronize do return if @ready - Sql::AuditConnectionBase.establish_connection(@connection_string) + Sql::AuditConnectionBase.establish_connection(@database) connection = Sql::AuditConnectionBase.connection Sql::Migrator.new(connection, schema: schema_for(connection), table_name: @table_name).run @model = build_model(qualified(connection)) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/builder/agent_factory.rb b/packages/forest_admin_agent/lib/forest_admin_agent/builder/agent_factory.rb index 1974d6baa..8099232f4 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/builder/agent_factory.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/builder/agent_factory.rb @@ -58,6 +58,7 @@ def use(plugin, options = {}) end def build + install_audit_trail @container.register(:datasource, @customizer.datasource(@logger)) # Reset route cache to ensure routes are computed with all customizations @@ -289,12 +290,30 @@ def build_cache @options[:customize_error_message] = clean_option_value(@options[:customize_error_message], 'config.customize_error_message =') @options[:logger] = clean_option_value(@options[:logger], 'config.logger =') + build_audit_trail_store @container.register(:config, @options.to_h) configure_rpc_polling_pool if @options[:rpc_max_polling_threads] end + # The audit trail switches on as soon as a database is configured. The store is built here + # (cheap: the connection only opens on the first write) and shared through the config, so the + # record-history routes read the very table the capture layer writes to. + def build_audit_trail_store + options = @options[:audit_trail] + return unless options && options[:database] + + options[:store] = AuditTrail::Store.new(**options.slice(:database, :schema, :table_name).compact) + end + + def install_audit_trail + options = @options[:audit_trail] + return if options.nil? || options[:store].nil? + + @customizer.use(AuditTrail::Capture, { store: options[:store], redact: options[:redact] }) + end + def configure_rpc_polling_pool max_threads = @options[:rpc_max_polling_threads].to_i return unless max_threads.positive? diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb index 221e55db0..1012ff221 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb @@ -5,8 +5,8 @@ module Routes module Resources # Record-history route, mirroring the Node agent's `/_audit-trail/{collection}/:id`. # - # Registered only when an `audit_trail` option carrying a readable store was provided to the - # agent. The store is expected to respond to `list_by_record` and `count_by_record`. + # Registered only when `config.audit_trail[:database]` is set, in which case the agent factory + # built the store the capture layer writes to. class AuditTrail < AbstractAuthenticatedRoute include ForestAdminAgent::Utils diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb index e7e6d6a46..6ed916ee0 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb @@ -2,9 +2,8 @@ module ForestAdminAgent module Routes module Resources # Correlation-scoped record-history routes, mirroring the Node agent's - # `/_audit-trail/correlation/:key` and `/_audit-trail/correlations`. Registered only when an - # `audit_trail` store is configured; the store must respond to `list_by_correlation` and - # `list_by_correlations`. All three routes are scoped to a single record through the + # `/_audit-trail/correlation/:key` and `/_audit-trail/correlations`. Registered only when + # `config.audit_trail[:database]` is set. All three routes are scoped to a single record through the # `collection`/`recordId` query (GET) or body (POST) params and share the per-record auth. class AuditTrailCorrelation < AbstractAuthenticatedRoute def setup_routes diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb new file mode 100644 index 000000000..c26321e2b --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb @@ -0,0 +1,127 @@ +require 'spec_helper' + +module ForestAdminAgent + module AuditTrail + describe Capture do + let(:column_schema) { ForestAdminDatasourceToolkit::Schema::ColumnSchema } + + let(:store) do + Class.new do + attr_reader :records + + def initialize + @records = [] + end + + def append(record) + @records << record + end + end.new + end + + let(:fields) do + { + 'id' => column_schema.new(column_type: 'Number', is_primary_key: true, is_read_only: true), + 'name' => column_schema.new(column_type: 'String'), + 'address' => column_schema.new(column_type: 'Json') + } + end + + let(:hooks) { {} } + let(:collection) { double('collection') } + let(:caller_double) { double('caller', id: 42, request_id: 'req-xyz') } + + let(:collection_customizer) do + customizer = double('CollectionCustomizer', name: 'companies', collection: collection) + allow(customizer).to receive(:add_hook) { |position, type, &block| hooks["#{position}_#{type}"] = block } + customizer + end + + let(:datasource_customizer) do + double('DatasourceCustomizer', collections: { 'companies' => collection_customizer }) + end + + before do + allow(collection).to receive(:schema).and_return({ fields: fields }) + described_class.new.run(datasource_customizer, nil, store: store) + end + + it 'records a create with only the writable columns' do + record = { 'id' => 1, 'name' => 'Acme', 'address' => { 'city' => 'Paris' } } + + hooks['After_Create'].call(double('ctx', caller: caller_double, record: record)) + + audit = store.records.last + expect(audit.operation).to eq('create') + expect(audit.record_id).to eq('1') + expect(audit.user_id).to eq(42) + expect(audit.correlation_key).to eq('req-xyz') + expect(audit.previous_values).to eq({}) + expect(audit.new_values).to eq({ 'name' => 'Acme', 'address' => { 'city' => 'Paris' } }) + end + + it "shares the caller's request id as the correlation key across records of one operation" do + filter = Object.new + allow(collection).to receive(:list).and_return( + [{ 'id' => 1, 'name' => 'A' }, { 'id' => 2, 'name' => 'B' }] + ) + + hooks['Before_Update'].call(double('before', caller: caller_double, filter: filter, collection: collection)) + hooks['After_Update'].call(double('after', caller: caller_double, filter: filter, patch: { 'name' => 'Z' })) + + expect(store.records.map(&:correlation_key)).to eq(%w[req-xyz req-xyz]) + end + + it 'records an update with the minimal nested diff' do + filter = Object.new + before_record = { 'id' => 1, 'name' => 'Acme', 'address' => { 'city' => 'Paris', 'zip' => '1' } } + allow(collection).to receive(:list).and_return([before_record]) + + hooks['Before_Update'].call(double('before', caller: caller_double, filter: filter, collection: collection)) + hooks['After_Update'].call( + double('after', caller: caller_double, filter: filter, + patch: { 'address' => { 'city' => 'Lyon', 'zip' => '1' } }) + ) + + audit = store.records.last + expect(audit.operation).to eq('update') + expect(audit.previous_values).to eq({ 'address' => { 'city' => 'Paris' } }) + expect(audit.new_values).to eq({ 'address' => { 'city' => 'Lyon' } }) + end + + it 'does not record an update when nothing writable changed' do + filter = Object.new + allow(collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'Acme' }]) + + hooks['Before_Update'].call(double('before', caller: caller_double, filter: filter, collection: collection)) + hooks['After_Update'].call(double('after', caller: caller_double, filter: filter, patch: { 'name' => 'Acme' })) + + expect(store.records).to be_empty + end + + it 'records a delete with the previous values' do + filter = Object.new + allow(collection).to receive(:list).and_return([{ 'id' => 7, 'name' => 'Gone', 'address' => nil }]) + + hooks['Before_Delete'].call(double('before', caller: caller_double, filter: filter, collection: collection)) + hooks['After_Delete'].call(double('after', caller: caller_double, filter: filter)) + + audit = store.records.last + expect(audit.operation).to eq('delete') + expect(audit.record_id).to eq('7') + expect(audit.previous_values).to eq({ 'name' => 'Gone', 'address' => nil }) + expect(audit.new_values).to eq({}) + end + + it 'masks redacted fields while still recording the change' do + described_class.new.run(datasource_customizer, nil, store: store, redact: { 'companies' => ['name'] }) + + hooks['After_Create'].call( + double('ctx', caller: caller_double, record: { 'id' => 1, 'name' => 'Secret', 'address' => nil }) + ) + + expect(store.records.last.new_values['name']).to eq(described_class::REDACTED) + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb new file mode 100644 index 000000000..403ebddb1 --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb @@ -0,0 +1,54 @@ +require 'spec_helper' + +module ForestAdminAgent + module AuditTrail + describe Diff do + describe '.diff' do + it 'returns nil when values are deeply equal regardless of hash key order' do + expect(described_class.diff({ 'a' => 1, 'b' => 2 }, { 'b' => 2, 'a' => 1 })).to be_nil + end + + it 'keeps only the changed leaf of a nested object' do + before = { 'city' => 'Paris', 'zip' => '75001' } + after = { 'city' => 'Lyon', 'zip' => '75001' } + + expect(described_class.diff(before, after)).to eq( + previous: { 'city' => 'Paris' }, + next: { 'city' => 'Lyon' } + ) + end + + it 'diffs an array of objects index by index' do + before = [{ 'name' => 'a' }, { 'name' => 'b' }] + after = [{ 'name' => 'a' }, { 'name' => 'c' }] + + expect(described_class.diff(before, after)).to eq( + previous: { 1 => { 'name' => 'b' } }, + next: { 1 => { 'name' => 'c' } } + ) + end + + it 'keeps scalars and primitive arrays whole' do + expect(described_class.diff(%w[a b], %w[a c])).to eq(previous: %w[a b], next: %w[a c]) + end + + it 'reports nil for a newly set or cleared value' do + expect(described_class.diff(nil, 'x')).to eq(previous: nil, next: 'x') + expect(described_class.diff('x', nil)).to eq(previous: 'x', next: nil) + end + end + + describe '.changed_values' do + it 'only records writable columns present in the patch that actually changed' do + before = { 'status' => 'open', 'name' => 'Acme', 'ignored' => 1 } + patch = { 'status' => 'closed', 'name' => 'Acme' } + + expect(described_class.changed_values(before, patch, %w[status name])).to eq( + previous_values: { 'status' => 'open' }, + new_values: { 'status' => 'closed' } + ) + end + end + end + end +end diff --git a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb similarity index 91% rename from packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb rename to packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb index 60b29d64a..f9d6114be 100644 --- a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/sql_store_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb @@ -1,14 +1,14 @@ require 'spec_helper' require 'tempfile' -module ForestAdminAuditTrail - module Stores - describe SqlStore do +module ForestAdminAgent + module AuditTrail + describe Store do let(:db) { Tempfile.new(['audit', '.sqlite3']) } - let(:store) { described_class.new(connection_string: { adapter: 'sqlite3', database: db.path }) } + let(:store) { described_class.new(database: { adapter: 'sqlite3', database: db.path }) } after do - ForestAdminAuditTrail::Sql::AuditConnectionBase.remove_connection + Sql::AuditConnectionBase.remove_connection db.close! end @@ -21,7 +21,7 @@ def record(over = {}) end def connection - ForestAdminAuditTrail::Sql::AuditConnectionBase.connection + Sql::AuditConnectionBase.connection end it 'creates the audit table with the expected columns on first write' do @@ -114,19 +114,19 @@ def connection it 'tracks applied migrations and is idempotent across stores' do store.append(record) - described_class.new(connection_string: { adapter: 'sqlite3', database: db.path }).append(record) + described_class.new(database: { adapter: 'sqlite3', database: db.path }).append(record) names = connection.select_values('SELECT name FROM audit_migrations ORDER BY name') expect(names).to eq(['001-create-audit-logs', '002-index-record-and-correlation']) end it 'binds each store to its own model class instead of mutating a shared one' do - other = described_class.new(connection_string: { adapter: 'sqlite3', database: db.path }) + other = described_class.new(database: { adapter: 'sqlite3', database: db.path }) store.append(record(correlation_key: 'main')) other.append(record(correlation_key: 'other')) # No shared mutable model: AuditLog is an abstract template, each store owns a distinct subclass. - expect(ForestAdminAuditTrail::Sql::AuditLog.abstract_class?).to be(true) + expect(Sql::AuditLog.abstract_class?).to be(true) expect(store.send(:model)).not_to equal(other.send(:model)) expect(store.send(:model).table_name).to eq('audit_logs') expect(store.list_by_record(collection: 'accounts', record_id: '1').map(&:correlation_key)) diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/builder/agent_factory_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/builder/agent_factory_spec.rb index 0382c327f..2f2559969 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/builder/agent_factory_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/builder/agent_factory_spec.rb @@ -254,6 +254,53 @@ module Builder end end + describe 'audit trail' do + let(:instance) { described_class.instance } + let(:options) do + { + auth_secret: 'cba803d01a4d43b55010cab41fa1ea1f1f51a95e', + env_secret: '89719c6d8e2e2de2694c2f220fe2dbf02d5289487364daf1e4c6b13733ed0cdb', + is_production: false, + schema_path: File.join('tmp', '.forestadmin-schema.json') + } + end + let(:audit_options) { { database: { adapter: 'sqlite3', database: ':memory:' } } } + + before { allow(instance).to receive(:send_schema) } + + it 'stays off when no audit trail database is configured' do + instance.setup(options) + allow(instance.customizer).to receive(:use) + + instance.build + + expect(instance.container.resolve(:config)[:audit_trail]).to be_nil + expect(instance.customizer).not_to have_received(:use) + end + + it 'stays off when the audit trail option carries no database' do + instance.setup(options.merge(audit_trail: { redact: { 'users' => ['email'] } })) + allow(instance.customizer).to receive(:use) + + instance.build + + expect(instance.container.resolve(:config)[:audit_trail][:store]).to be_nil + expect(instance.customizer).not_to have_received(:use) + end + + it 'builds the store from the configured database and installs the capture layer' do + instance.setup(options.merge(audit_trail: audit_options.merge(redact: { 'users' => ['email'] }))) + allow(instance.customizer).to receive(:use) + + instance.build + + store = instance.container.resolve(:config)[:audit_trail][:store] + expect(store).to be_a(AuditTrail::Store) + expect(instance.customizer).to have_received(:use) + .with(AuditTrail::Capture, { store: store, redact: { 'users' => ['email'] } }) + end + end + describe 'generate_schema_only' do it 'generates schema and writes to default path' do instance = described_class.instance diff --git a/packages/forest_admin_agent/spec/spec_helper.rb b/packages/forest_admin_agent/spec/spec_helper.rb index d7148dd8c..686c44ed5 100644 --- a/packages/forest_admin_agent/spec/spec_helper.rb +++ b/packages/forest_admin_agent/spec/spec_helper.rb @@ -1,4 +1,6 @@ require 'filecache' +require 'active_record' +require 'sqlite3' require 'simplecov' require 'simplecov_json_formatter' require 'simplecov-html' diff --git a/packages/forest_admin_audit_trail/.gitignore b/packages/forest_admin_audit_trail/.gitignore deleted file mode 100644 index d5402bb42..000000000 --- a/packages/forest_admin_audit_trail/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -/.bundle/ -/coverage/ -/pkg/ -/tmp/ -Gemfile.lock -*.gem -*.sqlite3 diff --git a/packages/forest_admin_audit_trail/.rspec b/packages/forest_admin_audit_trail/.rspec deleted file mode 100644 index 34c5164d9..000000000 --- a/packages/forest_admin_audit_trail/.rspec +++ /dev/null @@ -1,3 +0,0 @@ ---format documentation ---color ---require spec_helper diff --git a/packages/forest_admin_audit_trail/Gemfile b/packages/forest_admin_audit_trail/Gemfile deleted file mode 100644 index 9ac9e586e..000000000 --- a/packages/forest_admin_audit_trail/Gemfile +++ /dev/null @@ -1,20 +0,0 @@ -source 'https://rubygems.org' - -gemspec - -gem 'forest_admin_datasource_customizer', path: '../forest_admin_datasource_customizer' -gem 'forest_admin_datasource_toolkit', path: '../forest_admin_datasource_toolkit' -gem 'rake', '~> 13.0' -gem 'rubocop', '1.86.1' -gem 'rubocop-performance', '1.26.1' -gem 'rubocop-rspec', '3.9.0' - -group :development, :test do - gem 'forest_admin_agent', path: '../forest_admin_agent' - gem 'forest_admin_test_toolkit', path: '../forest_admin_test_toolkit' - gem 'rspec', '~> 3.0' - gem 'simplecov', '~> 0.22', require: false - gem 'simplecov-html', '~> 0.12.3' - gem 'simplecov_json_formatter', '~> 0.1.4' - gem 'sqlite3', '>= 2.1' -end diff --git a/packages/forest_admin_audit_trail/Rakefile b/packages/forest_admin_audit_trail/Rakefile deleted file mode 100644 index 85b387478..000000000 --- a/packages/forest_admin_audit_trail/Rakefile +++ /dev/null @@ -1,10 +0,0 @@ -require 'bundler/gem_tasks' -require 'rspec/core/rake_task' - -RSpec::Core::RakeTask.new(:spec) - -require 'rubocop/rake_task' - -RuboCop::RakeTask.new - -task default: %i[spec rubocop] diff --git a/packages/forest_admin_audit_trail/forest_admin_audit_trail.gemspec b/packages/forest_admin_audit_trail/forest_admin_audit_trail.gemspec deleted file mode 100644 index b69391b4f..000000000 --- a/packages/forest_admin_audit_trail/forest_admin_audit_trail.gemspec +++ /dev/null @@ -1,37 +0,0 @@ -lib = File.expand_path('lib', __dir__) -$LOAD_PATH.unshift lib unless $LOAD_PATH.include?(lib) - -require_relative 'lib/forest_admin_audit_trail/version' - -Gem::Specification.new do |spec| - spec.name = 'forest_admin_audit_trail' - spec.version = ForestAdminAuditTrail::VERSION - spec.authors = ['Forest Admin'] - spec.email = ['contact@forestadmin.com'] - spec.homepage = 'https://www.forestadmin.com' - spec.summary = 'Audit-trail plugin for the Forest Admin Ruby agent.' - spec.description = 'Capture who changed what (before/after) for every change Forest performs through its data ' \ - 'layer, and persist it into a SQL database. Datasource-agnostic capture, decoupled SQL storage.' - spec.license = 'GPL-3.0' - spec.required_ruby_version = '>= 3.0.0' - - spec.metadata['homepage_uri'] = spec.homepage - spec.metadata['source_code_uri'] = 'https://github.com/ForestAdmin/agent-ruby' - spec.metadata['changelog_uri'] = 'https://github.com/ForestAdmin/agent-ruby/blob/main/CHANGELOG.md' - spec.metadata['rubygems_mfa_required'] = 'false' - - spec.files = Dir.chdir(__dir__) do - `git ls-files -z`.split("\x0").reject do |f| - (File.expand_path(f) == __FILE__) || - f.start_with?(*%w[bin/ test/ spec/ .git Gemfile LICENSE]) - end - end - - spec.bindir = 'exe' - spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) } - spec.require_paths = ['lib'] - - spec.add_dependency 'activerecord', '>= 6.1' - spec.add_dependency 'activesupport', '>= 6.1' - spec.add_dependency 'zeitwerk', '~> 2.3' -end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail.rb deleted file mode 100644 index 29bb037f4..000000000 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail.rb +++ /dev/null @@ -1,10 +0,0 @@ -require_relative 'forest_admin_audit_trail/version' -require 'zeitwerk' - -loader = Zeitwerk::Loader.for_gem -loader.inflector.inflect('sql' => 'Sql') -loader.setup - -module ForestAdminAuditTrail - class Error < StandardError; end -end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/audit_record.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/audit_record.rb deleted file mode 100644 index 064c96afb..000000000 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/audit_record.rb +++ /dev/null @@ -1,45 +0,0 @@ -module ForestAdminAuditTrail - # Plain value object describing one audited change. Mirrors the columns of `forest.audit_logs`. - # - # Only the actor's `user_id` is stored; the rest of the actor identity is correlated elsewhere - # through `correlation_key`. - class AuditRecord - OPERATIONS = %w[create update delete].freeze - - attr_reader :timestamp, :operation, :collection, :record_id, :user_id, :correlation_key, - :previous_values, :new_values - - def initialize( - operation:, - collection:, - record_id:, - previous_values:, - new_values:, - timestamp: nil, - user_id: nil, - correlation_key: nil - ) - @timestamp = timestamp - @operation = operation - @collection = collection - @record_id = record_id - @user_id = user_id - @correlation_key = correlation_key - @previous_values = previous_values || {} - @new_values = new_values || {} - end - - def to_h - { - timestamp: @timestamp, - operation: @operation, - collection: @collection, - record_id: @record_id, - user_id: @user_id, - correlation_key: @correlation_key, - previous_values: @previous_values, - new_values: @new_values - } - end - end -end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/diff.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/diff.rb deleted file mode 100644 index 12a2b8965..000000000 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/diff.rb +++ /dev/null @@ -1,81 +0,0 @@ -module ForestAdminAuditTrail - # Minimal structural diff. Nested hashes and arrays of hashes are recursed into, so only the - # keys/indexes whose leaf value actually changed are kept — a single sub-field change does not - # store the whole object/array. Scalars, primitive arrays, dates and other values are compared and - # kept as a whole. - # - # Ruby's `==` already performs deep, key-order-independent equality on Hash and (ordered) equality - # on Array, so it is used directly as the equality primitive. - module Diff - module_function - - # @return [Hash{Symbol=>Object}, nil] { previous:, next: } of the changed leaves, or nil when equal. - def diff(before, after) - return nil if before == after - - return diff_hashes(before, after) if before.is_a?(Hash) && after.is_a?(Hash) - - return diff_object_arrays(before, after) if object_array?(before) && object_array?(after) - - { previous: before.nil? ? nil : before, next: after.nil? ? nil : after } - end - - # Build the previous/new value hashes for the writable columns that actually changed. - # - # @param before [Hash] snapshot of the record before the change (string keys) - # @param patch [Hash] the values being written (string keys); only present keys are considered - # @param columns [Array] writable column names to inspect - # @return [Hash{Symbol=>Hash}] { previous_values:, new_values: } - def changed_values(before, patch, columns) - previous_values = {} - new_values = {} - - columns.each do |column| - delta = patch.key?(column) ? diff(before[column], patch[column]) : nil - next unless delta - - previous_values[column] = delta[:previous] - new_values[column] = delta[:next] - end - - { previous_values: previous_values, new_values: new_values } - end - - # Arrays whose every element is a hash (record-like collections, e.g. a workflow history). - def object_array?(value) - value.is_a?(Array) && !value.empty? && value.all?(Hash) - end - - def diff_hashes(before, after) - previous = {} - next_values = {} - - (before.keys | after.keys).each do |key| - sub = diff(before[key], after[key]) - next unless sub - - previous[key] = sub[:previous] - next_values[key] = sub[:next] - end - - { previous: previous, next: next_values } - end - - def diff_object_arrays(before, after) - previous = {} - next_values = {} - - [before.length, after.length].max.times do |index| - sub = diff(before[index], after[index]) - next unless sub - - previous[index] = sub[:previous] - next_values[index] = sub[:next] - end - - { previous: previous, next: next_values } - end - - private_class_method :diff_hashes, :diff_object_arrays - end -end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/plugin.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/plugin.rb deleted file mode 100644 index 479a5a685..000000000 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/plugin.rb +++ /dev/null @@ -1,163 +0,0 @@ -require 'securerandom' -require 'time' - -module ForestAdminAuditTrail - # Datasource-agnostic capture layer. Instruments every collection (or a single one) through the - # Forest customizer hooks, computes the minimal before/after diff for each change and appends an - # {AuditRecord} to the configured store. - # - # Usage (datasource level — audits every collection): - # agent.use(ForestAdminAuditTrail::Plugin, store: my_store) - # - # Options: - # store: object responding to #append(AuditRecord). Defaults to a store that logs each record. - # redact: { 'collection_name' => ['field', ...] } — values masked while still recording the change. - class Plugin - REDACTED = '[redacted]'.freeze - - def run(datasource_customizer, collection_customizer = nil, options = {}) - options ||= {} - @store = options[:store] || Stores::LogStore.new - @redact = options[:redact] || {} - - collections = if collection_customizer - [collection_customizer] - else - datasource_customizer.collections.values - end - - collections.each { |collection| instrument(collection) } - end - - private - - def instrument(collection_customizer) - schema = collection_customizer.collection.schema - # Writable columns only: Forest audits what it writes. Read-only fields cover computed/virtual - # fields and DB-managed columns, none of which Forest mutates. - columns = schema[:fields].select do |_name, field| - field.type == 'Column' && !field.is_read_only - end.keys - primary_keys = ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(collection_customizer.collection) - # Reads must carry the primary keys (even read-only ones) so the record id can be built; the - # diff itself stays restricted to the writable columns. - projection = ForestAdminDatasourceToolkit::Components::Query::Projection.new( - (primary_keys + columns).uniq - ) - name = collection_customizer.name - redacted = @redact[name] || [] - - add_create_hook(collection_customizer, columns, primary_keys, name, redacted) - add_update_hooks(collection_customizer, columns, primary_keys, name, redacted, projection) - add_delete_hooks(collection_customizer, columns, primary_keys, name, redacted, projection) - end - - def add_create_hook(collection_customizer, columns, primary_keys, name, redacted) - plugin = self - collection_customizer.add_hook('After', 'Create') do |context| - plugin.send( - :emit, - context.caller, 'create', name, - plugin.send(:record_id, context.record, primary_keys), - {}, - plugin.send(:pick, context.record, columns), - redacted - ) - end - end - - def add_update_hooks(collection_customizer, columns, primary_keys, name, redacted, projection) - plugin = self - - collection_customizer.add_hook('Before', 'Update') do |context| - before = context.collection.list(context.filter, projection) - plugin.send(:pending)[context.filter] = before - end - - collection_customizer.add_hook('After', 'Update') do |context| - before = plugin.send(:pending).delete(context.filter) || [] - patch = context.patch - - before.each do |record| - delta = ForestAdminAuditTrail::Diff.changed_values(record, patch, columns) - next if delta[:new_values].empty? - - plugin.send( - :emit, - context.caller, 'update', name, - plugin.send(:record_id, record, primary_keys), - delta[:previous_values], delta[:new_values], - redacted - ) - end - end - end - - def add_delete_hooks(collection_customizer, columns, primary_keys, name, redacted, projection) - plugin = self - - collection_customizer.add_hook('Before', 'Delete') do |context| - before = context.collection.list(context.filter, projection) - plugin.send(:pending)[context.filter] = before - end - - collection_customizer.add_hook('After', 'Delete') do |context| - before = plugin.send(:pending).delete(context.filter) || [] - - before.each do |record| - plugin.send( - :emit, - context.caller, 'delete', name, - plugin.send(:record_id, record, primary_keys), - plugin.send(:pick, record, columns), - {}, - redacted - ) - end - end - end - - # Snapshots taken in a "before" hook and consumed in the matching "after" hook. Keyed by the - # filter object, which the hook decorator passes unchanged to both hooks within the same thread. - def pending - Thread.current[:forest_audit_trail_snapshots] ||= {}.compare_by_identity - end - - def emit(caller, operation, collection, record_id, previous_values, new_values, redacted) - @store.append( - AuditRecord.new( - timestamp: Time.now.utc.iso8601(3), - operation: operation, - collection: collection, - record_id: record_id, - user_id: caller&.id, - # Same id for every change made within one request — set on the caller by the agent - # (see CallerParser), mirroring the Node agent's caller.requestId. - correlation_key: correlation_key_for(caller), - previous_values: redact(previous_values, redacted), - new_values: redact(new_values, redacted) - ) - ) - end - - def correlation_key_for(caller) - (caller.respond_to?(:request_id) && caller.request_id) || SecureRandom.uuid - end - - def record_id(record, primary_keys) - primary_keys.map { |pk| record[pk].to_s }.join('|') - end - - def pick(record, columns) - columns.to_h { |column| [column, record[column]] } - end - - def redact(values, redacted_fields) - return values if redacted_fields.empty? - - values.each_with_object({}) do |(field, value), result| - result[field] = redacted_fields.include?(field) ? REDACTED : value - end - end - end -end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_connection_base.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_connection_base.rb deleted file mode 100644 index 2622aebbd..000000000 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_connection_base.rb +++ /dev/null @@ -1,11 +0,0 @@ -require 'active_record' - -module ForestAdminAuditTrail - module Sql - # Dedicated abstract base so the audit storage keeps its own connection pool, isolated from the - # host application's ActiveRecord::Base connection. - class AuditConnectionBase < ActiveRecord::Base - self.abstract_class = true - end - end -end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_log.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_log.rb deleted file mode 100644 index f2b09487d..000000000 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/audit_log.rb +++ /dev/null @@ -1,14 +0,0 @@ -module ForestAdminAuditTrail - module Sql - # Abstract template for the audit model. Each SqlStore builds its own concrete subclass bound to - # its own (schema-qualified) table, so stores with different `table_name`/`schema` no longer - # clobber a shared class. The JSON attribute overrides force Hash <-> JSON casting on every - # adapter (Postgres json, or text on SQLite) and are inherited by every subclass. - class AuditLog < AuditConnectionBase - self.abstract_class = true - - attribute :previous_values, :json - attribute :new_values, :json - end - end -end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb deleted file mode 100644 index b29e3dde9..000000000 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/sql/migrator.rb +++ /dev/null @@ -1,119 +0,0 @@ -module ForestAdminAuditTrail - module Sql - # Creates and evolves the audit table through an ordered, append-only list of migrations, tracked - # in a dedicated `audit_migrations` table (namespaced in the `forest` schema on Postgres). - # - # On Postgres the migrations run inside a transaction-scoped advisory lock, so several agent - # instances booting at once apply them one after another instead of racing on the same DDL. The - # schema is created (and committed) first, made idempotent (CREATE SCHEMA IF NOT EXISTS + - # tolerating a concurrent create), because the lock cannot cover a not-yet-existing schema. - class Migrator - MIGRATIONS_TABLE = 'audit_migrations'.freeze - # Arbitrary but stable key pair identifying the audit-trail migration critical section. - ADVISORY_LOCK = [0x464f, 0x5254].freeze # "FO", "RT" - - MIGRATIONS = [ - { - name: '001-create-audit-logs', - up: lambda do |connection, table| - # if_not_exists: a non-PG race (no advisory lock) can let two instances both reach here. - connection.create_table(table, if_not_exists: true) do |t| - t.datetime :timestamp, null: false - t.string :operation, null: false - t.string :collection, null: false - t.string :record_id, null: false - t.integer :user_id - t.string :correlation_key - t.json :previous_values - t.json :new_values - end - end - }, - { - name: '002-index-record-and-correlation', - up: lambda do |connection, table| - base = table.split('.').last - connection.add_index(table, :record_id, name: "#{base}_record_id", if_not_exists: true) - connection.add_index(table, :correlation_key, name: "#{base}_correlation_key", if_not_exists: true) - connection.add_index(table, :user_id, name: "#{base}_user_id", if_not_exists: true) - end - } - ].freeze - - def initialize(connection, schema:, table_name:) - @connection = connection - @schema = schema # nil on adapters without schema support - @table_name = table_name - end - - def run - ensure_schema - - if postgres? - @connection.transaction do - @connection.execute("SELECT pg_advisory_xact_lock(#{ADVISORY_LOCK[0]}, #{ADVISORY_LOCK[1]})") - apply_pending - end - else - apply_pending - end - end - - private - - def postgres? - @connection.adapter_name.downcase.include?('postgres') - end - - def schema? - postgres? && @schema.present? - end - - def qualified(name) - schema? ? "#{@schema}.#{name}" : name - end - - # Create the schema first and commit it: the migrations open DDL on the same connection, and a - # CREATE SCHEMA still pending in the lock transaction would not be visible to them. - def ensure_schema - return unless schema? - - @connection.execute("CREATE SCHEMA IF NOT EXISTS #{@connection.quote_schema_name(@schema)}") - rescue ActiveRecord::StatementInvalid => e - # Another instance created it concurrently — CREATE SCHEMA is not fully race-free. - raise unless /already exists|duplicate/i.match?(e.message) - end - - def apply_pending - done = applied_migrations - table = qualified(@table_name) - - MIGRATIONS.each do |migration| - next if done.include?(migration[:name]) - - migration[:up].call(@connection, table) - @connection.execute( - "INSERT INTO #{@connection.quote_table_name(migrations_table)} (name) " \ - "VALUES (#{@connection.quote(migration[:name])})" - ) - end - end - - def applied_migrations - ensure_migrations_table - - @connection.select_values("SELECT name FROM #{@connection.quote_table_name(migrations_table)}") - end - - def ensure_migrations_table - @connection.create_table(migrations_table, id: false, if_not_exists: true) do |t| - t.string :name, null: false - end - end - - def migrations_table - qualified(MIGRATIONS_TABLE) - end - end - end -end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/in_memory_store.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/in_memory_store.rb deleted file mode 100644 index 39e12d3ef..000000000 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/in_memory_store.rb +++ /dev/null @@ -1,67 +0,0 @@ -require 'time' - -module ForestAdminAuditTrail - module Stores - # Read/write store kept in memory. Handy for tests and for exercising the record-history route - # without a database. - class InMemoryStore - def initialize - @records = [] - end - - def append(record) - @records << record - end - - def list_by_record(collection:, record_id:, skip: 0, limit: nil, - user_ids: nil, start_timestamp: nil, end_timestamp: nil, order: 'asc') - matches = sorted(matching(collection, record_id, user_ids, start_timestamp, end_timestamp), order) - - skip ||= 0 - ending = limit.nil? ? matches.length : skip + limit - matches[skip...ending] || [] - end - - def count_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil, end_timestamp: nil) - matching(collection, record_id, user_ids, start_timestamp, end_timestamp).length - end - - def list_by_correlation(collection:, record_id:, correlation_key:) - by_correlation(collection, record_id) { |record| record.correlation_key == correlation_key } - end - - def list_by_correlations(collection:, record_id:, correlation_keys:) - by_correlation(collection, record_id) { |record| correlation_keys.include?(record.correlation_key) } - end - - private - - def by_correlation(collection, record_id) - matches = @records.select do |record| - record.collection == collection && record.record_id == record_id && yield(record) - end - sorted(matches, 'asc') - end - - def matching(collection, record_id, user_ids, start_timestamp, end_timestamp) - @records.select do |record| - record.collection == collection && record.record_id == record_id && - (user_ids.nil? || user_ids.include?(record.user_id)) && - (start_timestamp.nil? || as_time(record.timestamp) >= as_time(start_timestamp)) && - (end_timestamp.nil? || as_time(record.timestamp) <= as_time(end_timestamp)) - end - end - - # Ties on equal timestamps fall back to append order (index) in both directions — the in-memory - # equivalent of the SQL store's auto-increment id tie-break. - def sorted(records, order) - sign = order.to_s == 'desc' ? -1 : 1 - records.each_with_index.sort_by { |record, i| [sign * as_time(record.timestamp).to_f, i] }.map(&:first) - end - - def as_time(value) - value.is_a?(::Time) ? value : ::Time.iso8601(value.to_s) - end - end - end -end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/log_store.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/log_store.rb deleted file mode 100644 index 3f3c31c27..000000000 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/stores/log_store.rb +++ /dev/null @@ -1,34 +0,0 @@ -require 'json' - -module ForestAdminAuditTrail - module Stores - # Default store used when none is configured: it only logs each change. Write-only — the - # record-history route reading from it always gets an empty result. - class LogStore - def initialize(logger: nil) - @logger = logger - end - - def append(record) - message = "[audit-trail] #{record.to_h.to_json}" - @logger ? @logger.log('Info', message) : warn(message) - end - - def list_by_record(**) - [] - end - - def count_by_record(**) - 0 - end - - def list_by_correlation(**) - [] - end - - def list_by_correlations(**) - [] - end - end - end -end diff --git a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb b/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb deleted file mode 100644 index 5eacc1525..000000000 --- a/packages/forest_admin_audit_trail/lib/forest_admin_audit_trail/version.rb +++ /dev/null @@ -1,3 +0,0 @@ -module ForestAdminAuditTrail - VERSION = "1.33.1" -end diff --git a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/diff_spec.rb b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/diff_spec.rb deleted file mode 100644 index c9d7ba9cc..000000000 --- a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/diff_spec.rb +++ /dev/null @@ -1,52 +0,0 @@ -require 'spec_helper' - -module ForestAdminAuditTrail - describe Diff do - describe '.diff' do - it 'returns nil when values are deeply equal regardless of hash key order' do - expect(described_class.diff({ 'a' => 1, 'b' => 2 }, { 'b' => 2, 'a' => 1 })).to be_nil - end - - it 'keeps only the changed leaf of a nested object' do - before = { 'city' => 'Paris', 'zip' => '75001' } - after = { 'city' => 'Lyon', 'zip' => '75001' } - - expect(described_class.diff(before, after)).to eq( - previous: { 'city' => 'Paris' }, - next: { 'city' => 'Lyon' } - ) - end - - it 'diffs an array of objects index by index' do - before = [{ 'name' => 'a' }, { 'name' => 'b' }] - after = [{ 'name' => 'a' }, { 'name' => 'c' }] - - expect(described_class.diff(before, after)).to eq( - previous: { 1 => { 'name' => 'b' } }, - next: { 1 => { 'name' => 'c' } } - ) - end - - it 'keeps scalars and primitive arrays whole' do - expect(described_class.diff(%w[a b], %w[a c])).to eq(previous: %w[a b], next: %w[a c]) - end - - it 'reports nil for a newly set or cleared value' do - expect(described_class.diff(nil, 'x')).to eq(previous: nil, next: 'x') - expect(described_class.diff('x', nil)).to eq(previous: 'x', next: nil) - end - end - - describe '.changed_values' do - it 'only records writable columns present in the patch that actually changed' do - before = { 'status' => 'open', 'name' => 'Acme', 'ignored' => 1 } - patch = { 'status' => 'closed', 'name' => 'Acme' } - - expect(described_class.changed_values(before, patch, %w[status name])).to eq( - previous_values: { 'status' => 'open' }, - new_values: { 'status' => 'closed' } - ) - end - end - end -end diff --git a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/plugin_spec.rb b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/plugin_spec.rb deleted file mode 100644 index d4f77fd18..000000000 --- a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/plugin_spec.rb +++ /dev/null @@ -1,121 +0,0 @@ -require 'spec_helper' - -module ForestAdminAuditTrail - describe Plugin do - let(:column_schema) { ForestAdminDatasourceToolkit::Schema::ColumnSchema } - - let(:store) do - Class.new do - attr_reader :records - - def initialize - @records = [] - end - - def append(record) - @records << record - end - end.new - end - - let(:fields) do - { - 'id' => column_schema.new(column_type: 'Number', is_primary_key: true, is_read_only: true), - 'name' => column_schema.new(column_type: 'String'), - 'address' => column_schema.new(column_type: 'Json') - } - end - - let(:hooks) { {} } - let(:collection) { double('collection') } - let(:caller_double) { double('caller', id: 42, request_id: 'req-xyz') } - - let(:collection_customizer) do - customizer = double('CollectionCustomizer', name: 'companies', collection: collection) - allow(customizer).to receive(:add_hook) { |position, type, &block| hooks["#{position}_#{type}"] = block } - customizer - end - - before do - allow(collection).to receive(:schema).and_return({ fields: fields }) - described_class.new.run(nil, collection_customizer, store: store) - end - - it 'records a create with only the writable columns' do - record = { 'id' => 1, 'name' => 'Acme', 'address' => { 'city' => 'Paris' } } - - hooks['After_Create'].call(double('ctx', caller: caller_double, record: record)) - - audit = store.records.last - expect(audit.operation).to eq('create') - expect(audit.record_id).to eq('1') - expect(audit.user_id).to eq(42) - expect(audit.correlation_key).to eq('req-xyz') - expect(audit.previous_values).to eq({}) - expect(audit.new_values).to eq({ 'name' => 'Acme', 'address' => { 'city' => 'Paris' } }) - end - - it "shares the caller's request id as the correlation key across records of one operation" do - filter = Object.new - allow(collection).to receive(:list).and_return( - [{ 'id' => 1, 'name' => 'A' }, { 'id' => 2, 'name' => 'B' }] - ) - - hooks['Before_Update'].call(double('before', caller: caller_double, filter: filter, collection: collection)) - hooks['After_Update'].call(double('after', caller: caller_double, filter: filter, patch: { 'name' => 'Z' })) - - expect(store.records.map(&:correlation_key)).to eq(%w[req-xyz req-xyz]) - end - - it 'records an update with the minimal nested diff' do - filter = Object.new - before_record = { 'id' => 1, 'name' => 'Acme', 'address' => { 'city' => 'Paris', 'zip' => '1' } } - allow(collection).to receive(:list).and_return([before_record]) - - hooks['Before_Update'].call(double('before', caller: caller_double, filter: filter, collection: collection)) - hooks['After_Update'].call( - double('after', caller: caller_double, filter: filter, - patch: { 'address' => { 'city' => 'Lyon', 'zip' => '1' } }) - ) - - audit = store.records.last - expect(audit.operation).to eq('update') - expect(audit.previous_values).to eq({ 'address' => { 'city' => 'Paris' } }) - expect(audit.new_values).to eq({ 'address' => { 'city' => 'Lyon' } }) - end - - it 'does not record an update when nothing writable changed' do - filter = Object.new - allow(collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'Acme' }]) - - hooks['Before_Update'].call(double('before', caller: caller_double, filter: filter, collection: collection)) - hooks['After_Update'].call(double('after', caller: caller_double, filter: filter, patch: { 'name' => 'Acme' })) - - expect(store.records).to be_empty - end - - it 'records a delete with the previous values' do - filter = Object.new - allow(collection).to receive(:list).and_return([{ 'id' => 7, 'name' => 'Gone', 'address' => nil }]) - - hooks['Before_Delete'].call(double('before', caller: caller_double, filter: filter, collection: collection)) - hooks['After_Delete'].call(double('after', caller: caller_double, filter: filter)) - - audit = store.records.last - expect(audit.operation).to eq('delete') - expect(audit.record_id).to eq('7') - expect(audit.previous_values).to eq({ 'name' => 'Gone', 'address' => nil }) - expect(audit.new_values).to eq({}) - end - - it 'masks redacted fields while still recording the change' do - described_class.new.run(nil, collection_customizer, store: store, redact: { 'companies' => ['name'] }) - - hooks['After_Create'].call( - double('ctx', caller: caller_double, record: { 'id' => 1, 'name' => 'Secret', 'address' => nil }) - ) - - expect(store.records.last.new_values['name']).to eq(described_class::REDACTED) - end - end -end diff --git a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/in_memory_store_spec.rb b/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/in_memory_store_spec.rb deleted file mode 100644 index 9ba250d75..000000000 --- a/packages/forest_admin_audit_trail/spec/lib/forest_admin_audit_trail/stores/in_memory_store_spec.rb +++ /dev/null @@ -1,122 +0,0 @@ -require 'spec_helper' - -module ForestAdminAuditTrail - module Stores - describe InMemoryStore do - subject(:store) { described_class.new } - - def record(over = {}) - AuditRecord.new( - operation: 'update', collection: 'accounts', record_id: '1', - previous_values: {}, new_values: {}, timestamp: '2026-01-02T03:04:05.000Z', **over - ) - end - - it 'returns rows for the queried record, oldest first' do - store.append(record(timestamp: '2026-01-02T03:04:06.000Z', correlation_key: 'b')) - store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'a')) - store.append(record(record_id: '2', correlation_key: 'other')) - - history = store.list_by_record(collection: 'accounts', record_id: '1') - - expect(history.map(&:correlation_key)).to eq(%w[a b]) - end - - it 'honors skip and limit' do - 3.times { |i| store.append(record(timestamp: "2026-01-02T03:04:0#{i}.000Z", correlation_key: i.to_s)) } - - page = store.list_by_record(collection: 'accounts', record_id: '1', skip: 1, limit: 1) - - expect(page.map(&:correlation_key)).to eq(['1']) - end - - it 'returns an empty array when nothing matches' do - store.append(record) - - expect(store.list_by_record(collection: 'accounts', record_id: 'missing')).to eq([]) - end - - it 'sorts newest first when order is desc, ties keeping append order' do - store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'a')) - store.append(record(timestamp: '2026-01-02T03:04:06.000Z', correlation_key: 'b')) - store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'a2')) - - history = store.list_by_record(collection: 'accounts', record_id: '1', order: 'desc') - - expect(history.map(&:correlation_key)).to eq(%w[b a a2]) - end - - it 'filters by user_ids, dropping non-matching actors' do - store.append(record(user_id: 7, correlation_key: 'keep')) - store.append(record(user_id: 9, correlation_key: 'drop')) - - history = store.list_by_record(collection: 'accounts', record_id: '1', user_ids: [7]) - - expect(history.map(&:correlation_key)).to eq(['keep']) - end - - it 'filters by inclusive start/end timestamps regardless of string format' do - store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'before')) - store.append(record(timestamp: '2026-01-02T03:04:07.000Z', correlation_key: 'inside')) - store.append(record(timestamp: '2026-01-02T03:04:09.000Z', correlation_key: 'after')) - - history = store.list_by_record( - collection: 'accounts', record_id: '1', - start_timestamp: '2026-01-02T03:04:06.000Z', end_timestamp: '2026-01-02T03:04:08.000Z' - ) - - expect(history.map(&:correlation_key)).to eq(['inside']) - end - - it 'counts matches independently of skip/limit' do - store.append(record(user_id: 7)) - store.append(record(user_id: 7)) - store.append(record(user_id: 9)) - - expect(store.count_by_record(collection: 'accounts', record_id: '1')).to eq(3) - expect(store.count_by_record(collection: 'accounts', record_id: '1', user_ids: [7])).to eq(2) - end - - describe '#list_by_correlation' do - it 'returns entries under the key for the record, oldest first, scoped to collection/record' do - store.append(record(timestamp: '2026-01-02T00:00:00.000Z', correlation_key: 'req-1', - new_values: { 'n' => 2 })) - store.append(record(timestamp: '2026-01-01T00:00:00.000Z', correlation_key: 'req-1', - new_values: { 'n' => 1 })) - store.append(record(correlation_key: 'req-2', new_values: { 'n' => 9 })) - store.append(record(record_id: '2', correlation_key: 'req-1')) - store.append(record(collection: 'contacts', correlation_key: 'req-1')) - - history = store.list_by_correlation(collection: 'accounts', record_id: '1', correlation_key: 'req-1') - - expect(history.map(&:new_values)).to eq([{ 'n' => 1 }, { 'n' => 2 }]) - end - - it 'returns an empty array when nothing matches' do - store.append(record(correlation_key: 'req-1')) - - expect(store.list_by_correlation(collection: 'accounts', record_id: '1', correlation_key: 'nope')).to eq([]) - end - end - - describe '#list_by_correlations' do - it 'returns a flat list under any of the keys, oldest first' do - store.append(record(timestamp: '2026-01-03T00:00:00.000Z', correlation_key: 'a', new_values: { 'n' => 3 })) - store.append(record(timestamp: '2026-01-01T00:00:00.000Z', correlation_key: 'b', new_values: { 'n' => 1 })) - store.append(record(timestamp: '2026-01-02T00:00:00.000Z', correlation_key: 'a', new_values: { 'n' => 2 })) - store.append(record(timestamp: '2026-01-04T00:00:00.000Z', correlation_key: 'c', new_values: { 'n' => 9 })) - - history = store.list_by_correlations(collection: 'accounts', record_id: '1', correlation_keys: %w[a b]) - - expect(history.map(&:new_values)).to eq([{ 'n' => 1 }, { 'n' => 2 }, { 'n' => 3 }]) - end - - it 'returns an empty array for an empty key list' do - store.append(record(correlation_key: 'a')) - - expect(store.list_by_correlations(collection: 'accounts', record_id: '1', correlation_keys: [])).to eq([]) - end - end - end - end -end diff --git a/packages/forest_admin_audit_trail/spec/spec_helper.rb b/packages/forest_admin_audit_trail/spec/spec_helper.rb deleted file mode 100644 index b1153fb60..000000000 --- a/packages/forest_admin_audit_trail/spec/spec_helper.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'simplecov' -require 'simplecov_json_formatter' -require 'simplecov-html' -require 'active_record' -require 'sqlite3' -require 'forest_admin_datasource_toolkit' -require 'forest_admin_datasource_customizer' -require 'forest_admin_audit_trail' - -SimpleCov.formatters = [SimpleCov::Formatter::JSONFormatter, SimpleCov::Formatter::HTMLFormatter] -SimpleCov.start do - add_filter 'spec' -end - -SimpleCov.coverage_dir 'coverage' -SimpleCov.at_exit do - result = SimpleCov.result - result.format! -end - -RSpec.configure do |config| - config.expect_with :rspec do |c| - c.syntax = :expect - end -end diff --git a/packages/forest_admin_rails/lib/forest_admin_rails.rb b/packages/forest_admin_rails/lib/forest_admin_rails.rb index 60fb95be7..8d91b5c9b 100644 --- a/packages/forest_admin_rails/lib/forest_admin_rails.rb +++ b/packages/forest_admin_rails/lib/forest_admin_rails.rb @@ -34,8 +34,8 @@ module ForestAdminRails setting :disable_route_cache, default: false setting :rpc_max_polling_threads, default: nil setting :workflow_executor_url, default: nil - # { store: }. When set, the record-history - # route (`/_audit-trail/:collection/:id`) is registered and reads from this store. + # { database: , schema:, table_name:, redact: } — setting `database` + # turns the audit trail on: every change is captured and the `/_audit-trail` routes are registered. setting :audit_trail, default: nil if defined?(Rails::Railtie) From cbf18658a8f14fc26872ccae9f6f2b8f4d46ff2c Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Mon, 10 Aug 2026 16:18:52 +0200 Subject: [PATCH 08/35] fix(audit_trail): scope history reads to the caller, keep gem loadable without activerecord MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rails eager loads every Zeitwerk loader, so the audit `sql/` files were pulled in — and with them `require "active_record"` — in agents that never configure an audit database (the mongoid package's CI job boots such an app). They are excluded from eager loading now. Review findings: - the history routes only checked `can?(:read, collection)`, so a role restricted to a subset of records could read the history of any of them; both routes now intersect the record with the permission scope, like Show - migration bookkeeping keyed the target table, otherwise a store configured with another `table_name` read someone else's migrations as done and never created its table - snapshots pair through a bounded LIFO stack instead of the filter object: the after-context carries the caller's original filter, so a customization replacing it silently dropped the audit record and stranded the snapshot - the effective patch is snapshotted in the before hook, so a replaced patch is recorded as written rather than as sent - a hash key holding nil is no longer diffed as equal to a missing key The reported `list` arity issue is a false positive: hook contexts hand out a RelaxedCollection, whose `list(filter, projection)` already binds the caller. The capture spec now uses a verifying double of it so the contract is asserted. Co-Authored-By: Claude Opus 5 (1M context) --- .../lib/forest_admin_agent.rb | 3 + .../forest_admin_agent/audit_trail/capture.rb | 34 +++++--- .../forest_admin_agent/audit_trail/diff.rb | 12 ++- .../audit_trail/sql/migrator.rb | 10 ++- .../routes/resources/audit_trail.rb | 2 + .../resources/audit_trail_correlation.rb | 3 + .../routes/resources/scoped_record.rb | 28 +++++++ .../audit_trail/capture_spec.rb | 81 ++++++++++++++----- .../audit_trail/diff_spec.rb | 14 ++++ .../audit_trail/store_spec.rb | 11 ++- .../resources/audit_trail_correlation_spec.rb | 35 +++++++- .../routes/resources/audit_trail_spec.rb | 44 +++++++++- 12 files changed, 236 insertions(+), 41 deletions(-) create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/scoped_record.rb diff --git a/packages/forest_admin_agent/lib/forest_admin_agent.rb b/packages/forest_admin_agent/lib/forest_admin_agent.rb index f375035bc..22d65f2b9 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent.rb @@ -6,6 +6,9 @@ loader.inflector.inflect('oauth2' => 'OAuth2') loader.inflector.inflect('sql' => 'Sql') loader.inflector.inflect('sse_cache_invalidation' => 'SSECacheInvalidation') +# ActiveRecord is only needed by agents configuring an audit-trail database, and Rails eager loads +# every gem loader (Zeitwerk::Loader.eager_load_all), so these files must stay strictly autoloaded. +loader.do_not_eager_load("#{__dir__}/forest_admin_agent/audit_trail/sql") loader.setup module ForestAdminAgent diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb index f0f447307..deb970e5f 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb @@ -9,6 +9,8 @@ module AuditTrail # minimal before/after diff for each change and appends an {AuditRecord} to the store. class Capture REDACTED = '[redacted]'.freeze + # ponytail: 16 deep is far past any legitimate nesting; raise it if one ever gets that far. + MAX_SNAPSHOTS = 16 # Signature imposed by DatasourceCustomizer#use; the agent always instruments the whole datasource. def run(datasource_customizer, _collection_customizer = nil, options = {}) @@ -51,15 +53,16 @@ def add_create_hook(collection_customizer, columns, primary_keys, name) def add_update_hooks(collection_customizer, columns, primary_keys, name, projection) collection_customizer.add_hook('Before', 'Update') do |context| - pending[context.filter] = context.collection.list(context.filter, projection) + # Snapshot the patch here: installed last, this hook sees what actually gets written, while + # the after-context is handed the original patch the caller sent. + push_snapshot(records: context.collection.list(context.filter, projection), patch: context.patch) end collection_customizer.add_hook('After', 'Update') do |context| - before = pending.delete(context.filter) || [] - patch = context.patch + snapshot = snapshots.pop - before.each do |record| - delta = Diff.changed_values(record, patch, columns) + snapshot&.fetch(:records)&.each do |record| + delta = Diff.changed_values(record, snapshot[:patch], columns) next if delta[:new_values].empty? emit( @@ -72,13 +75,13 @@ def add_update_hooks(collection_customizer, columns, primary_keys, name, project def add_delete_hooks(collection_customizer, columns, primary_keys, name, projection) collection_customizer.add_hook('Before', 'Delete') do |context| - pending[context.filter] = context.collection.list(context.filter, projection) + push_snapshot(records: context.collection.list(context.filter, projection)) end collection_customizer.add_hook('After', 'Delete') do |context| - before = pending.delete(context.filter) || [] + snapshot = snapshots.pop - before.each do |record| + snapshot&.fetch(:records)&.each do |record| emit( context.caller, 'delete', name, record_id(record, primary_keys), pick(record, columns), {} @@ -87,10 +90,17 @@ def add_delete_hooks(collection_customizer, columns, primary_keys, name, project end end - # Snapshots taken in a "before" hook and consumed in the matching "after" hook. Keyed by the - # filter object, which the hook decorator passes unchanged to both hooks within the same thread. - def pending - Thread.current[:forest_audit_trail_snapshots] ||= {}.compare_by_identity + # Snapshots taken in a "before" hook and consumed in the matching "after" hook. Both bracket one + # operation on one thread, so a LIFO stack pairs them without relying on the filter object + # reaching both hooks unchanged. An operation raising in between strands its entry, hence the cap. + def push_snapshot(snapshot) + stack = snapshots + stack.shift while stack.size >= MAX_SNAPSHOTS + stack.push(snapshot) + end + + def snapshots + Thread.current[:forest_audit_trail_snapshots] ||= [] end def emit(caller, operation, collection, record_id, previous_values, new_values) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb index cd6d30095..8d8ed7a1a 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb @@ -52,7 +52,7 @@ def diff_hashes(before, after) next_values = {} (before.keys | after.keys).each do |key| - sub = diff(before[key], after[key]) + sub = diff_at(before, after, key) next unless sub previous[key] = sub[:previous] @@ -62,6 +62,14 @@ def diff_hashes(before, after) { previous: previous, next: next_values } end + # A key held with a nil value is not the same thing as a missing key: recursing on the values + # alone reads both as nil and reports no change at all. + def diff_at(before, after, key) + return diff(before[key], after[key]) if before.key?(key) == after.key?(key) + + { previous: before[key], next: after[key] } + end + def diff_object_arrays(before, after) previous = {} next_values = {} @@ -77,7 +85,7 @@ def diff_object_arrays(before, after) { previous: previous, next: next_values } end - private_class_method :diff_hashes, :diff_object_arrays + private_class_method :diff_hashes, :diff_at, :diff_object_arrays end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb index 36c9aab73..d05419afb 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb @@ -90,12 +90,18 @@ def apply_pending table = qualified(@table_name) MIGRATIONS.each do |migration| - next if done.include?(migration[:name]) + # The tracking table is shared by every audit table of the database/schema, so the target + # table belongs in the key: a store configured with another `table_name` must still get its + # own table created instead of reading someone else's migration as done. Rows written by + # earlier versions (bare migration name) simply replay, which the `if_not_exists` DDL above + # makes a no-op. + key = "#{table}:#{migration[:name]}" + next if done.include?(key) migration[:up].call(@connection, table) @connection.execute( "INSERT INTO #{@connection.quote_table_name(migrations_table)} (name) " \ - "VALUES (#{@connection.quote(migration[:name])})" + "VALUES (#{@connection.quote(key)})" ) end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb index 1012ff221..6af4cb825 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb @@ -9,6 +9,7 @@ module Resources # built the store the capture layer writes to. class AuditTrail < AbstractAuthenticatedRoute include ForestAdminAgent::Utils + include ScopedRecord DEFAULT_PAGE_SIZE = 20 MAX_PAGE_SIZE = 100 @@ -32,6 +33,7 @@ def setup_routes def handle_request(args = {}) context = build(args) context.permissions.can?(:read, context.collection) + assert_record_in_scope(context, context.collection, args[:params]['id']) skip, limit = parse_pagination(args) filters = { diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb index 6ed916ee0..a9e2431e3 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb @@ -6,6 +6,8 @@ module Resources # `config.audit_trail[:database]` is set. All three routes are scoped to a single record through the # `collection`/`recordId` query (GET) or body (POST) params and share the per-record auth. class AuditTrailCorrelation < AbstractAuthenticatedRoute + include ScopedRecord + def setup_routes return self unless store @@ -73,6 +75,7 @@ def assert_scope(args) collection = get_collection(context, name) context.permissions.can?(:read, collection) + assert_record_in_scope(context, collection, record_id) [collection, record_id] end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/scoped_record.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/scoped_record.rb new file mode 100644 index 000000000..fb6b58225 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/scoped_record.rb @@ -0,0 +1,28 @@ +module ForestAdminAgent + module Routes + module Resources + # Shared by the audit-trail routes: they take a packed record id straight from the request, so + # the caller's permission scope has to be checked against that record before any history is + # returned. `can?(:read, collection)` alone only proves access to the collection — a role + # restricted to a subset of the records would otherwise read the history of any of them. + module ScopedRecord + include ForestAdminDatasourceToolkit::Components::Query + + def assert_record_in_scope(context, collection, packed_id) + scope = context.permissions.get_scope(collection) + record = ConditionTree::ConditionTreeFactory.match_records( + collection, [Utils::Id.unpack_id(collection, packed_id, with_key: true)] + ) + filter = Filter.new( + condition_tree: ConditionTree::ConditionTreeFactory.intersect([record, scope]) + ) + projection = Projection.new(ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(collection)) + + return unless collection.list(context.caller, filter, projection).empty? + + raise Http::Exceptions::NotFoundError, 'Record does not exists' + end + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb index c26321e2b..3b298af21 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb @@ -31,6 +31,12 @@ def append(record) let(:collection) { double('collection') } let(:caller_double) { double('caller', id: 42, request_id: 'req-xyz') } + # What a hook context really hands out: the caller is already bound, so `list` takes + # (filter, projection). A verifying double keeps that contract honest. + let(:relaxed_collection) do + instance_double(ForestAdminDatasourceCustomizer::Context::RelaxedWrappers::RelaxedCollection) + end + let(:collection_customizer) do customizer = double('CollectionCustomizer', name: 'companies', collection: collection) allow(customizer).to receive(:add_hook) { |position, type, &block| hooks["#{position}_#{type}"] = block } @@ -42,6 +48,7 @@ def append(record) end before do + Thread.current[:forest_audit_trail_snapshots] = nil allow(collection).to receive(:schema).and_return({ fields: fields }) described_class.new.run(datasource_customizer, nil, store: store) end @@ -60,28 +67,34 @@ def append(record) expect(audit.new_values).to eq({ 'name' => 'Acme', 'address' => { 'city' => 'Paris' } }) end + def before_hook(type, patch: nil) + hooks["Before_#{type}"].call( + double('before', caller: caller_double, filter: Object.new, + collection: relaxed_collection, patch: patch) + ) + end + + def after_hook(type) + hooks["After_#{type}"].call(double('after', caller: caller_double, filter: Object.new)) + end + it "shares the caller's request id as the correlation key across records of one operation" do - filter = Object.new - allow(collection).to receive(:list).and_return( + allow(relaxed_collection).to receive(:list).and_return( [{ 'id' => 1, 'name' => 'A' }, { 'id' => 2, 'name' => 'B' }] ) - hooks['Before_Update'].call(double('before', caller: caller_double, filter: filter, collection: collection)) - hooks['After_Update'].call(double('after', caller: caller_double, filter: filter, patch: { 'name' => 'Z' })) + before_hook('Update', patch: { 'name' => 'Z' }) + after_hook('Update') expect(store.records.map(&:correlation_key)).to eq(%w[req-xyz req-xyz]) end it 'records an update with the minimal nested diff' do - filter = Object.new before_record = { 'id' => 1, 'name' => 'Acme', 'address' => { 'city' => 'Paris', 'zip' => '1' } } - allow(collection).to receive(:list).and_return([before_record]) + allow(relaxed_collection).to receive(:list).and_return([before_record]) - hooks['Before_Update'].call(double('before', caller: caller_double, filter: filter, collection: collection)) - hooks['After_Update'].call( - double('after', caller: caller_double, filter: filter, - patch: { 'address' => { 'city' => 'Lyon', 'zip' => '1' } }) - ) + before_hook('Update', patch: { 'address' => { 'city' => 'Lyon', 'zip' => '1' } }) + after_hook('Update') audit = store.records.last expect(audit.operation).to eq('update') @@ -90,21 +103,51 @@ def append(record) end it 'does not record an update when nothing writable changed' do - filter = Object.new - allow(collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'Acme' }]) + allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'Acme' }]) - hooks['Before_Update'].call(double('before', caller: caller_double, filter: filter, collection: collection)) - hooks['After_Update'].call(double('after', caller: caller_double, filter: filter, patch: { 'name' => 'Acme' })) + before_hook('Update', patch: { 'name' => 'Acme' }) + after_hook('Update') expect(store.records).to be_empty end + # The before and after hooks are handed different filter objects (the after context always + # carries the caller's original), so pairing must not depend on that object. + it 'pairs the snapshot with its after hook without relying on the filter object' do + allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'Acme' }]) + + before_hook('Update', patch: { 'name' => 'Effective' }) + after_hook('Update') + + expect(store.records.last.new_values).to eq({ 'name' => 'Effective' }) + end + + it 'still audits the next write after one raised between its hooks' do + allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'Acme' }]) + + before_hook('Update', patch: { 'name' => 'Never written' }) + before_hook('Update', patch: { 'name' => 'Z' }) + after_hook('Update') + + expect(store.records.map { |record| record.new_values['name'] }).to eq(['Z']) + end + + it 'drops the oldest stranded snapshots instead of growing the thread-local without bound' do + allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'Acme' }]) + + (described_class::MAX_SNAPSHOTS + 4).times { before_hook('Update', patch: { 'name' => 'stranded' }) } + before_hook('Update', patch: { 'name' => 'Z' }) + after_hook('Update') + + expect(Thread.current[:forest_audit_trail_snapshots].size).to eq(described_class::MAX_SNAPSHOTS - 1) + expect(store.records.map { |record| record.new_values['name'] }).to eq(['Z']) + end + it 'records a delete with the previous values' do - filter = Object.new - allow(collection).to receive(:list).and_return([{ 'id' => 7, 'name' => 'Gone', 'address' => nil }]) + allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 7, 'name' => 'Gone', 'address' => nil }]) - hooks['Before_Delete'].call(double('before', caller: caller_double, filter: filter, collection: collection)) - hooks['After_Delete'].call(double('after', caller: caller_double, filter: filter)) + before_hook('Delete') + after_hook('Delete') audit = store.records.last expect(audit.operation).to eq('delete') diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb index 403ebddb1..10405b2eb 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb @@ -18,6 +18,20 @@ module AuditTrail ) end + it 'reports a key that disappeared, even when its value was nil' do + expect(described_class.diff({ 'flag' => nil }, {})).to eq( + previous: { 'flag' => nil }, + next: { 'flag' => nil } + ) + end + + it 'reports a key that appeared holding nil' do + expect(described_class.diff({}, { 'flag' => nil })).to eq( + previous: { 'flag' => nil }, + next: { 'flag' => nil } + ) + end + it 'diffs an array of objects index by index' do before = [{ 'name' => 'a' }, { 'name' => 'b' }] after = [{ 'name' => 'a' }, { 'name' => 'c' }] diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb index f9d6114be..4b766d463 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb @@ -117,7 +117,16 @@ def connection described_class.new(database: { adapter: 'sqlite3', database: db.path }).append(record) names = connection.select_values('SELECT name FROM audit_migrations ORDER BY name') - expect(names).to eq(['001-create-audit-logs', '002-index-record-and-correlation']) + expect(names).to eq(['audit_logs:001-create-audit-logs', 'audit_logs:002-index-record-and-correlation']) + end + + it 'migrates a second table in the same database instead of reading the first one as done' do + store.append(record) + other = described_class.new(database: { adapter: 'sqlite3', database: db.path }, table_name: 'other_logs') + + expect { other.append(record) }.not_to raise_error + expect(other.list_by_record(collection: 'accounts', record_id: '1').size).to eq(1) + expect(connection.indexes('other_logs').map(&:name)).to include('other_logs_record_id') end it 'binds each store to its own model class instead of mutating a shared one' do diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb index e4f077e84..a5dfcb2dd 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb @@ -3,9 +3,26 @@ module ForestAdminAgent module Routes module Resources + include ForestAdminDatasourceToolkit::Schema + include ForestAdminDatasourceToolkit::Components::Query::ConditionTree + describe AuditTrailCorrelation do let(:store) { double('store') } - let(:collection) { double('collection', name: 'books') } + let(:permissions) { double('permissions', can?: true, get_scope: nil) } + let(:collection) do + build_collection( + name: 'books', + schema: { + fields: { + 'id' => ColumnSchema.new( + column_type: 'Number', is_primary_key: true, + filter_operators: [Operators::IN, Operators::EQUAL] + ) + } + }, + list: [{ 'id' => 2 }] + ) + end def route_with_store(history: []) allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) @@ -15,12 +32,24 @@ def route_with_store(history: []) route = described_class.new datasource = double('datasource') allow(datasource).to receive(:get_collection).with('books').and_return(collection) - context = double('context', datasource: datasource, - permissions: double('permissions', can?: true)) + context = double('context', datasource: datasource, caller: build_caller, permissions: permissions) allow(route).to receive(:build).and_return(context) route end + it 'returns 404 without touching the store when the record is out of the caller scope' do + allow(collection).to receive(:list).and_return([]) + route = route_with_store + + expect do + route.handle_history( + { headers: {}, params: { 'collection' => 'books', 'recordId' => '2', 'correlation_key' => 'req-1' } } + ) + end.to raise_error(Http::Exceptions::NotFoundError) + + expect(store).not_to have_received(:list_by_correlation) + end + it 'registers the correlation routes when a store is configured' do allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) .and_return({ audit_trail: { store: Object.new } }) diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb index 753e31c95..5f43e701b 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -3,8 +3,26 @@ module ForestAdminAgent module Routes module Resources + include ForestAdminDatasourceToolkit::Schema + include ForestAdminDatasourceToolkit::Components::Query::ConditionTree + describe AuditTrail do let(:store) { double('store') } + let(:permissions) { double('permissions', can?: true, get_scope: nil) } + let(:collection) do + build_collection( + name: 'projects', + schema: { + fields: { + 'id' => ColumnSchema.new( + column_type: 'Number', is_primary_key: true, + filter_operators: [Operators::IN, Operators::EQUAL] + ) + } + }, + list: [{ 'id' => 4 }] + ) + end def route_with_store(records: []) allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) @@ -12,8 +30,7 @@ def route_with_store(records: []) allow(store).to receive_messages(list_by_record: records, count_by_record: records.length) route = described_class.new - context = double('context', collection: double('collection', name: 'projects'), - permissions: double('permissions', can?: true)) + context = double('context', collection: collection, caller: build_caller, permissions: permissions) allow(route).to receive(:build).and_return(context) route end @@ -50,6 +67,29 @@ def route_with_store(records: []) ) end + it 'intersects the record with the permission scope before reading any history' do + scope = Nodes::ConditionTreeLeaf.new('id', Operators::EQUAL, 4) + allow(permissions).to receive(:get_scope).and_return(scope) + route = route_with_store + + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) + + expect(collection).to have_received(:list) do |_caller, filter, _projection| + expect(filter.condition_tree.conditions).to include(scope) + end + end + + it 'returns 404 without touching the store when the record is out of the caller scope' do + allow(collection).to receive(:list).and_return([]) + route = route_with_store + + expect do + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) + end.to raise_error(Http::Exceptions::NotFoundError) + + expect(store).not_to have_received(:list_by_record) + end + it 'defaults to newest-first and switches to oldest-first on sort=timestamp' do route = route_with_store route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4', 'sort' => 'timestamp' } }) From 12cd870dd3e36501998f76acf9899f74cf86a755 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Mon, 10 Aug 2026 16:35:00 +0200 Subject: [PATCH 09/35] fix(audit_trail): keep deleted records readable, one payload shape, safe page parsing Review follow-ups: - history stays readable once the record is gone: the scope check now refuses only a record that still exists outside the caller's scope, since inspecting what was deleted is much of the point of an audit trail - the correlation routes serialize like the per-record one (camelCase on top, column names untouched) instead of leaking snake_case keys; the shared behaviour of the audit routes lives in AuditTrailRoute - `?page=foo` no longer raises: a non-Hash `page` falls back to the documented defaults instead of blowing up in Hash#dig Co-Authored-By: Claude Opus 5 (1M context) --- packages/forest_admin_agent/AUDIT_TRAIL.md | 8 ++++ .../routes/resources/audit_trail.rb | 17 ++++--- .../resources/audit_trail_correlation.rb | 6 +-- .../routes/resources/audit_trail_route.rb | 44 +++++++++++++++++++ .../routes/resources/scoped_record.rb | 28 ------------ .../resources/audit_trail_correlation_spec.rb | 13 ++++-- .../routes/resources/audit_trail_spec.rb | 23 +++++++++- 7 files changed, 93 insertions(+), 46 deletions(-) create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb delete mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/scoped_record.rb diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index 630139c7f..fdb47183f 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -98,6 +98,14 @@ erroring. Sorting follows JSON:API `sort` on `timestamp`: `sort=-timestamp` (or is newest first, `sort=timestamp` is oldest first. Ties on equal timestamps fall back to insertion order (the auto-increment `id`), so paging is deterministic in either direction. +All three routes serialize audit records the same way: top-level keys are camelCased +(`recordId`, `userId`, `correlationKey`, `previousValues`, `new_values` → `newValues`), while the +`previousValues` / `newValues` hashes keep the audited record's own column names. + +A record that no longer exists keeps its history: only a record that still exists *outside* the +caller's permission scope is refused (404). Inspecting what was deleted is much of the point of an +audit trail, and the delete event itself is the last thing recorded. + ### Correlation route `GET /forest/_audit-trail/correlation/{correlationKey}` returns `{ "data": [...] }` — the diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb index 6af4cb825..3f577efa9 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb @@ -9,7 +9,7 @@ module Resources # built the store the capture layer writes to. class AuditTrail < AbstractAuthenticatedRoute include ForestAdminAgent::Utils - include ScopedRecord + include AuditTrailRoute DEFAULT_PAGE_SIZE = 20 MAX_PAGE_SIZE = 100 @@ -49,17 +49,12 @@ def handle_request(args = {}) { name: args[:params]['collection_name'], - content: { data: history.map { |record| serialize(record) }, meta: { count: count } } + content: { data: history.map { |record| serialize_record(record) }, meta: { count: count } } } end private - # Camelize only the top-level keys; previous/new value hashes keep the record's own column names. - def serialize(record) - record.to_h.transform_keys { |key| key.to_s.camelize(:lower) } - end - # JSON:API `sort`: `timestamp` → oldest first, anything else (absent/unsupported) → newest first. def parse_sort(args) args.dig(:params, 'sort').to_s == 'timestamp' ? 'asc' : 'desc' @@ -68,11 +63,15 @@ def parse_sort(args) # JSON:API pagination: 1-based page[number] (default 1) and page[size] (default 20, capped at # 100). Out-of-bound or non-numeric values fall back to the defaults rather than erroring. def parse_pagination(args) - size = args.dig(:params, 'page', 'size').to_i + # `?page=foo` reaches us as a bare String, which `dig` refuses to walk into. + page = args.dig(:params, 'page') + page = {} unless page.is_a?(Hash) + + size = page['size'].to_i size = DEFAULT_PAGE_SIZE if size < 1 size = MAX_PAGE_SIZE if size > MAX_PAGE_SIZE - number = args.dig(:params, 'page', 'number').to_i + number = page['number'].to_i number = 1 if number < 1 [(number - 1) * size, size] diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb index a9e2431e3..2743e4a7b 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb @@ -6,7 +6,7 @@ module Resources # `config.audit_trail[:database]` is set. All three routes are scoped to a single record through the # `collection`/`recordId` query (GET) or body (POST) params and share the per-record auth. class AuditTrailCorrelation < AbstractAuthenticatedRoute - include ScopedRecord + include AuditTrailRoute def setup_routes return self unless store @@ -43,7 +43,7 @@ def handle_history(args = {}) correlation_key: args[:params]['correlation_key'] ) - { name: collection.name, content: { data: history.map(&:to_h) } } + { name: collection.name, content: { data: history.map { |record| serialize_record(record) } } } end def handle_batch(args = {}) @@ -60,7 +60,7 @@ def handle_batch(args = {}) ) end - { name: collection.name, content: { data: history.map(&:to_h) } } + { name: collection.name, content: { data: history.map { |record| serialize_record(record) } } } end private diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb new file mode 100644 index 000000000..1efa078b6 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb @@ -0,0 +1,44 @@ +module ForestAdminAgent + module Routes + module Resources + # Behaviour shared by every audit-trail route: they all take a packed record id straight from the + # request, so the caller's permission scope has to be checked against that record before any + # history is returned (`can?(:read, collection)` alone only proves access to the collection — a + # role restricted to a subset of the records would otherwise read the history of any of them), + # and they all serialize audit records the same way. + module AuditTrailRoute + include ForestAdminDatasourceToolkit::Components::Query + + def assert_record_in_scope(context, collection, packed_id) + condition = ConditionTree::ConditionTreeFactory.match_records( + collection, [Utils::Id.unpack_id(collection, packed_id, with_key: true)] + ) + scope = context.permissions.get_scope(collection) + + return if any_record?(context, collection, ConditionTree::ConditionTreeFactory.intersect([condition, scope])) + + # Nothing in scope means the record is either outside it or gone for good. A deleted record + # keeps its history readable — inspecting what was deleted is much of the point of an audit + # trail — so only a record that still exists outside the scope is refused. Without a scope + # the first query already answered the question. + return if scope.nil? || !any_record?(context, collection, condition) + + raise Http::Exceptions::NotFoundError, 'Record does not exists' + end + + # Camelize only the top-level keys; previous/new value hashes keep the record's own column names. + def serialize_record(record) + record.to_h.transform_keys { |key| key.to_s.camelize(:lower) } + end + + def any_record?(context, collection, condition_tree) + collection.list( + context.caller, + Filter.new(condition_tree: condition_tree), + Projection.new(ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(collection)) + ).any? + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/scoped_record.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/scoped_record.rb deleted file mode 100644 index fb6b58225..000000000 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/scoped_record.rb +++ /dev/null @@ -1,28 +0,0 @@ -module ForestAdminAgent - module Routes - module Resources - # Shared by the audit-trail routes: they take a packed record id straight from the request, so - # the caller's permission scope has to be checked against that record before any history is - # returned. `can?(:read, collection)` alone only proves access to the collection — a role - # restricted to a subset of the records would otherwise read the history of any of them. - module ScopedRecord - include ForestAdminDatasourceToolkit::Components::Query - - def assert_record_in_scope(context, collection, packed_id) - scope = context.permissions.get_scope(collection) - record = ConditionTree::ConditionTreeFactory.match_records( - collection, [Utils::Id.unpack_id(collection, packed_id, with_key: true)] - ) - filter = Filter.new( - condition_tree: ConditionTree::ConditionTreeFactory.intersect([record, scope]) - ) - projection = Projection.new(ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(collection)) - - return unless collection.list(context.caller, filter, projection).empty? - - raise Http::Exceptions::NotFoundError, 'Record does not exists' - end - end - end - end -end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb index a5dfcb2dd..66f5e9e7b 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb @@ -37,8 +37,9 @@ def route_with_store(history: []) route end - it 'returns 404 without touching the store when the record is out of the caller scope' do - allow(collection).to receive(:list).and_return([]) + it 'returns 404 without touching the store when the record exists outside the caller scope' do + allow(permissions).to receive(:get_scope).and_return(Nodes::ConditionTreeLeaf.new('id', Operators::EQUAL, 9)) + allow(collection).to receive(:list).and_return([], [{ 'id' => 2 }]) route = route_with_store expect do @@ -66,7 +67,8 @@ def route_with_store(history: []) end it 'reads a single correlation history scoped to the record' do - route = route_with_store(history: [double('entry', to_h: { operation: 'update' })]) + entry = { operation: 'update', record_id: '2', new_values: { 'first_name' => 'Jo' } } + route = route_with_store(history: [double('entry', to_h: entry)]) result = route.handle_history( { headers: {}, params: { 'collection' => 'books', 'recordId' => '2', 'correlation_key' => 'req-1' } } @@ -75,7 +77,10 @@ def route_with_store(history: []) expect(store).to have_received(:list_by_correlation).with( collection: 'books', record_id: '2', correlation_key: 'req-1' ) - expect(result[:content]).to eq({ data: [{ operation: 'update' }] }) + # Same serialization as the per-record route: camelCase on top, column names left alone. + expect(result[:content]).to eq( + { data: [{ 'operation' => 'update', 'recordId' => '2', 'newValues' => { 'first_name' => 'Jo' } }] } + ) end it 'reads a batch history from comma-separated query keys (GET)' do diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb index 5f43e701b..9ade67934 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -79,8 +79,10 @@ def route_with_store(records: []) end end - it 'returns 404 without touching the store when the record is out of the caller scope' do - allow(collection).to receive(:list).and_return([]) + it 'returns 404 without touching the store when the record exists outside the caller scope' do + allow(permissions).to receive(:get_scope).and_return(Nodes::ConditionTreeLeaf.new('id', Operators::EQUAL, 9)) + # Empty under the scope, found without it: the record is someone else's, not a deleted one. + allow(collection).to receive(:list).and_return([], [{ 'id' => 4 }]) route = route_with_store expect do @@ -90,6 +92,16 @@ def route_with_store(records: []) expect(store).not_to have_received(:list_by_record) end + it 'still serves the history of a deleted record, which is much of the point of an audit trail' do + allow(permissions).to receive(:get_scope).and_return(Nodes::ConditionTreeLeaf.new('id', Operators::EQUAL, 4)) + allow(collection).to receive(:list).and_return([]) + route = route_with_store(records: [double('entry', to_h: { operation: 'delete', record_id: '4' })]) + + result = route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) + + expect(result[:content][:data]).to eq([{ 'operation' => 'delete', 'recordId' => '4' }]) + end + it 'defaults to newest-first and switches to oldest-first on sort=timestamp' do route = route_with_store route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4', 'sort' => 'timestamp' } }) @@ -106,6 +118,13 @@ def route_with_store(records: []) expect(store).to have_received(:list_by_record).with(hash_including(skip: 200, limit: 100)) end + it 'falls back to the default page when page is not a hash' do + route = route_with_store + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4', 'page' => 'foo' } }) + + expect(store).to have_received(:list_by_record).with(hash_including(skip: 0, limit: 20)) + end + it 'parses userIds, dropping non-numeric tokens' do route = route_with_store route.handle_request({ headers: {}, From 444b7f03baeb6d3bb6d6fcccb4cbdd72c5d73c4e Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Tue, 11 Aug 2026 15:54:15 +0200 Subject: [PATCH 10/35] fix(audit_trail): add audit_trail option to RPC agent --- packages/forest_admin_rpc_agent/lib/forest_admin_rpc_agent.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/forest_admin_rpc_agent/lib/forest_admin_rpc_agent.rb b/packages/forest_admin_rpc_agent/lib/forest_admin_rpc_agent.rb index 99a0cc1ed..9f941922f 100644 --- a/packages/forest_admin_rpc_agent/lib/forest_admin_rpc_agent.rb +++ b/packages/forest_admin_rpc_agent/lib/forest_admin_rpc_agent.rb @@ -28,6 +28,7 @@ module ForestAdminRpcAgent setting :customize_error_message, default: nil setting :disable_route_cache, default: false setting :rpc_max_polling_threads, default: nil + setting :audit_trail, default: nil begin require 'thor' From f138a1d11480bf3dc9ef7a9e548208206bb7d2d2 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Wed, 12 Aug 2026 09:26:18 +0200 Subject: [PATCH 11/35] fix(audit_trail): register the history routes before the collection routes `GET /_audit-trail/correlations` has two path segments, so the earlier `/:collection_name/:id` route matched it first and answered "Collection '_audit-trail' not found". Both audit-trail sources now sit ahead of every route matching on `:collection_name`; everything registered before them has a literal first segment, so nothing can shadow them and they shadow nothing. Co-Authored-By: Claude Opus 5 (1M context) --- .../lib/forest_admin_agent/http/router.rb | 10 ++++++---- .../lib/forest_admin_agent/http/router_spec.rb | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb b/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb index 1b12950ae..2c7063a2a 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/http/router.rb @@ -50,6 +50,12 @@ def self.routes { name: 'charts', handler: -> { Charts::Charts.new.routes } }, { name: 'collections', handler: -> { Capabilities::Collections.new.routes } }, { name: 'native_query', handler: -> { Resources::NativeQuery.new.routes } }, + # Both must come before the routes matching on `:collection_name`: Rails matches in + # definition order, so `/_audit-trail/correlations` would otherwise be read as + # `/:collection_name/:id` and 404 on a collection named `_audit-trail`. Correlation first, so + # `/_audit-trail/correlations` wins over the per-record `/_audit-trail/:collection_name/:id`. + { name: 'audit_trail_correlation', handler: -> { Resources::AuditTrailCorrelation.new.routes } }, + { name: 'audit_trail', handler: -> { Resources::AuditTrail.new.routes } }, { name: 'count', handler: -> { Resources::Count.new.routes } }, { name: 'delete', handler: -> { Resources::Delete.new.routes } }, { name: 'csv', handler: -> { Resources::Csv.new.routes } }, @@ -64,10 +70,6 @@ def self.routes { name: 'dissociate_related', handler: -> { Resources::Related::DissociateRelated.new.routes } }, { name: 'update_related', handler: -> { Resources::Related::UpdateRelated.new.routes } }, { name: 'update_field', handler: -> { Resources::UpdateField.new.routes } }, - # Registered before `audit_trail` so `/_audit-trail/correlation(s)` matches here instead of - # the per-record `/_audit-trail/:collection_name/:id` (Rails matches in definition order). - { name: 'audit_trail_correlation', handler: -> { Resources::AuditTrailCorrelation.new.routes } }, - { name: 'audit_trail', handler: -> { Resources::AuditTrail.new.routes } }, { name: 'workflow_executor_proxy', handler: -> { Workflow::WorkflowExecutorProxy.new.routes } } ] diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/router_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/router_spec.rb index 39054100b..6916ae740 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/router_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/http/router_spec.rb @@ -3,6 +3,21 @@ module ForestAdminAgent module Http describe Router do + # Rails matches in definition order, so a literal-prefixed path registered after the + # `:collection_name` routes is read as a collection name instead (a 2-segment + # `/_audit-trail/correlations` would 404 as collection `_audit-trail`, id `correlations`). + describe 'route order' do + it 'registers the audit-trail routes before the ones matching on :collection_name' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: Object.new } }) + names = described_class.routes.keys + + expect(names.index('forest_audit_trail_correlations')).to be < names.index('forest_show') + expect(names.index('forest_audit_trail_correlations')).to be < names.index('forest_list') + expect(names.index('forest_audit_trail_correlations')).to be < names.index('forest_audit_trail') + end + end + describe '.cached_routes' do before do described_class.reset_cached_routes! From 545051f10167fc04cd638b40ba2c7ac2196572c4 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Wed, 12 Aug 2026 09:52:10 +0200 Subject: [PATCH 12/35] feat(audit_trail): record smart action runs in the audit table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The customizer has no Execute hook, so the invocation is recorded from the action route: one row per selected record, `operation` `action` (or `action_failed` when it raised), the action name in its own column and the submitted form values in `new_values`, redacted with the same config as columns. A global action or a select-all selection names no target without querying the whole selection, so those runs are recorded once, attached to no record. Recording is best-effort: a failing audit database logs instead of breaking the action. What an action *changes* is unchanged: writes going through the Forest data layer are audited by the existing hooks under the same correlation key, and a direct ORM write stays invisible — documented rather than worked around. Co-Authored-By: Claude Opus 5 (1M context) --- .rubocop.yml | 3 + packages/forest_admin_agent/AUDIT_TRAIL.md | 22 ++++++ .../lib/forest_admin_agent/audit_trail.rb | 16 ++++ .../audit_trail/action_capture.rb | 47 +++++++++++ .../audit_trail/audit_record.rb | 3 +- .../forest_admin_agent/audit_trail/capture.rb | 22 +----- .../audit_trail/recording.rb | 30 +++++++ .../audit_trail/sql/migrator.rb | 6 ++ .../forest_admin_agent/audit_trail/store.rb | 2 + .../routes/action/actions.rb | 45 ++++++++++- .../routes/resources/audit_trail.rb | 5 -- .../resources/audit_trail_correlation.rb | 5 -- .../routes/resources/audit_trail_route.rb | 4 + .../audit_trail/action_capture_spec.rb | 79 +++++++++++++++++++ .../audit_trail/store_spec.rb | 18 ++++- .../routes/action/actions_spec.rb | 53 +++++++++++++ 16 files changed, 327 insertions(+), 33 deletions(-) create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb create mode 100644 packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb create mode 100644 packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb diff --git a/.rubocop.yml b/.rubocop.yml index 42d7fe5c2..4add98f46 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -257,6 +257,7 @@ Naming/PredicatePrefix: Metrics/ParameterLists: Exclude: + - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/configuration.rb' @@ -456,7 +457,9 @@ Layout/LineLength: RSpec/VerifiedDoubles: Exclude: + - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb' - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb' + - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb' - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb' - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb' - 'packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/composite_datasource_spec.rb' diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index fdb47183f..f378d6c9c 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -136,6 +136,27 @@ whose `correlation_key` is in `correlationKeys`, scoped to one record (the clien precedence over the query param). An empty/absent key list returns `{ "data": [] }` without hitting the store. A missing `collection` or `recordId` returns **HTTP 400** (`ValidationError`). +## Smart actions + +Running a smart action writes one row per selected record, in the same table: + +| column | value | +| ------------ | ------------------------------------------------------------------------- | +| `operation` | `action` when it ran, `action_failed` when it raised | +| `actionName` | the action's name | +| `newValues` | the submitted form values (redacted with the same `redact` config) | +| `recordId` | each selected record — empty for a global action or a select-all selection | + +A **global** action targets no record, and a **select-all** selection only tells the agent which ids were +*excluded*, so naming the targets would mean querying the whole selection: those runs are recorded once, +attached to no record. Recording is best-effort — a failing audit database logs an error rather than +breaking the action. + +> **What an action changes is only audited when it goes through Forest.** `context.collection.update(...)` +> passes through the same hooks as any other write, so it produces the usual field-level rows sharing the +> action's `correlationKey`. A direct ORM write (`Customer.find(id).update!(...)`) is invisible to the +> agent, so nothing is recorded for it beyond the invocation row above. + ## What gets stored `forest.audit_logs`, one row per audited change: @@ -149,6 +170,7 @@ the store. A missing `collection` or `recordId` returns **HTTP 400** (`Validatio | `record_id` | packed record id (primary keys joined by `\|`) | | `user_id` | the Forest user who made the change | | `correlation_key` | per-request id; groups every change made within one request | +| `action_name` | smart-action rows only: the action that was run | | `previous_values` | values before the change (JSON) | | `new_values` | values after the change (JSON) | diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb new file mode 100644 index 000000000..56e3d2994 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb @@ -0,0 +1,16 @@ +module ForestAdminAgent + # The audit trail is inert unless `config.audit_trail[:database]` was set: the agent factory builds + # the store during setup and stores it in the config, and everything (capture layers and routes) + # resolves it from here. + module AuditTrail + def self.options + config = Facades::Container.config_from_cache + + (config && config[:audit_trail]) || {} + end + + def self.store + options[:store] + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb new file mode 100644 index 000000000..27dd4c98a --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb @@ -0,0 +1,47 @@ +module ForestAdminAgent + module AuditTrail + # Records smart-action invocations into the same table as the field-level history. + # + # {Capture} cannot see them: the customizer has no `Execute` hook, and an action's writes are only + # audited when it goes through the Forest data layer — a direct ORM write (`Customer.find(id).update!`) + # is invisible to the agent and is therefore not recorded at all. What lands here is the invocation + # itself: who ran which action, on which records, with which form values, and whether it raised. + class ActionCapture + include Recording + + EXECUTED = 'action'.freeze + FAILED = 'action_failed'.freeze + # Bulk runs over a select-all selection, and global actions, target no id we can name without + # querying the whole selection: they get one row attached to no record rather than none at all. + NO_RECORD = ''.freeze + + def initialize(store, redact = {}) + @store = store + @redact = redact || {} + end + + def record(caller:, collection:, action_name:, form_values:, record_ids:, failed: false) + timestamp = now + correlation_key = correlation_key_for(caller) + values = redact(form_values || {}, @redact[collection] || []) + ids = record_ids.empty? ? [NO_RECORD] : record_ids + + ids.each do |record_id| + @store.append( + AuditRecord.new( + timestamp: timestamp, + operation: failed ? FAILED : EXECUTED, + collection: collection, + record_id: record_id, + action_name: action_name, + user_id: caller&.id, + correlation_key: correlation_key, + previous_values: {}, + new_values: values + ) + ) + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb index 011d4238a..31139914a 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb @@ -2,9 +2,10 @@ module ForestAdminAgent module AuditTrail # One audited change. Mirrors the columns of `forest.audit_logs`; only the actor's `user_id` is # stored, the rest of the actor identity is correlated elsewhere through `correlation_key`. + # `action_name` is set on smart-action rows only. AuditRecord = Struct.new( :timestamp, :operation, :collection, :record_id, :user_id, :correlation_key, - :previous_values, :new_values, + :action_name, :previous_values, :new_values, keyword_init: true ) end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb index deb970e5f..6a8eb4f74 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb @@ -1,6 +1,3 @@ -require 'securerandom' -require 'time' - module ForestAdminAgent module AuditTrail # Datasource-agnostic capture layer, installed by the agent factory as soon as an audit-trail @@ -8,7 +5,8 @@ module AuditTrail # it behaves the same whatever the audited datasource is (ActiveRecord, Mongoid, ...), computes the # minimal before/after diff for each change and appends an {AuditRecord} to the store. class Capture - REDACTED = '[redacted]'.freeze + include Recording + # ponytail: 16 deep is far past any legitimate nesting; raise it if one ever gets that far. MAX_SNAPSHOTS = 16 @@ -108,13 +106,11 @@ def emit(caller, operation, collection, record_id, previous_values, new_values) @store.append( AuditRecord.new( - timestamp: Time.now.utc.iso8601(3), + timestamp: now, operation: operation, collection: collection, record_id: record_id, user_id: caller&.id, - # Same id for every change made within one request — set on the caller by the agent - # (see CallerParser), mirroring the Node agent's caller.requestId. correlation_key: correlation_key_for(caller), previous_values: redact(previous_values, redacted), new_values: redact(new_values, redacted) @@ -122,10 +118,6 @@ def emit(caller, operation, collection, record_id, previous_values, new_values) ) end - def correlation_key_for(caller) - (caller.respond_to?(:request_id) && caller.request_id) || SecureRandom.uuid - end - def record_id(record, primary_keys) primary_keys.map { |pk| record[pk].to_s }.join('|') end @@ -133,14 +125,6 @@ def record_id(record, primary_keys) def pick(record, columns) columns.to_h { |column| [column, record[column]] } end - - def redact(values, redacted_fields) - return values if redacted_fields.empty? - - values.each_with_object({}) do |(field, value), result| - result[field] = redacted_fields.include?(field) ? REDACTED : value - end - end end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb new file mode 100644 index 000000000..d99a6e8e2 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb @@ -0,0 +1,30 @@ +require 'securerandom' +require 'time' + +module ForestAdminAgent + module AuditTrail + # Shared by the two capture layers: {Capture} for the changes Forest writes, {ActionCapture} for the + # smart actions it runs. + module Recording + REDACTED = '[redacted]'.freeze + + # Same id for every change made within one request — set on the caller by the agent (see + # CallerParser), mirroring the Node agent's caller.requestId. + def correlation_key_for(caller) + (caller.respond_to?(:request_id) && caller.request_id) || SecureRandom.uuid + end + + def redact(values, redacted_fields) + return values if redacted_fields.empty? + + values.each_with_object({}) do |(field, value), result| + result[field] = redacted_fields.include?(field) ? REDACTED : value + end + end + + def now + Time.now.utc.iso8601(3) + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb index d05419afb..1e3700f86 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb @@ -38,6 +38,12 @@ class Migrator connection.add_index(table, :correlation_key, name: "#{base}_correlation_key", if_not_exists: true) connection.add_index(table, :user_id, name: "#{base}_user_id", if_not_exists: true) end + }, + { + name: '003-add-action-name', + up: lambda do |connection, table| + connection.add_column(table, :action_name, :string, if_not_exists: true) + end } ].freeze diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb index 8c5d9b4e7..3da78d276 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb @@ -111,6 +111,7 @@ def to_row(record) record_id: record.record_id, user_id: record.user_id, correlation_key: record.correlation_key, + action_name: record.action_name, previous_values: record.previous_values, new_values: record.new_values } @@ -124,6 +125,7 @@ def from_row(row) record_id: row.record_id, user_id: row.user_id, correlation_key: row.correlation_key, + action_name: row.action_name, previous_values: row.previous_values || {}, new_values: row.new_values || {} ) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb index 265f56b35..a654d3af0 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb @@ -75,7 +75,7 @@ def handle_request(args = {}) fields.reject { |field| field.type == 'Layout' } ) - result = context.collection.execute(context.caller, @action_name, data, filter_for_caller) + result = execute_and_audit(context, args, data, filter_for_caller) { content: ForestAdminAgent::Utils::ActionResult.parse(result) } end @@ -117,6 +117,49 @@ def handle_hook_request(args = {}) private + # A failed run is worth recording too: "who tried to run this" is usually the interesting part. + # The audit row is best-effort — a broken audit database must not fail the action itself. + def execute_and_audit(context, args, data, filter) + result = context.collection.execute(context.caller, @action_name, data, filter) + audit_action(context, args, data) + + result + rescue StandardError + audit_action(context, args, data, failed: true) + raise + end + + def audit_action(context, args, data, failed: false) + store = ForestAdminAgent::AuditTrail.store + return unless store + + capture = ForestAdminAgent::AuditTrail::ActionCapture.new( + store, ForestAdminAgent::AuditTrail.options[:redact] + ) + capture.record( + caller: context.caller, + collection: context.collection.name, + action_name: @action_name, + form_values: data, + record_ids: audited_record_ids(args, context), + failed: failed + ) + rescue StandardError => e + Facades::Container.logger.log('Error', "[ForestAdmin] Could not audit action: #{e.message}") + end + + # Packed ids, the form the audit store keys on. A global action targets nothing, and a select-all + # selection only tells us which ids were *excluded*: naming the targets would mean querying the + # whole selection, so those runs are recorded once, attached to no record. + def audited_record_ids(args, context) + return [] if context.collection.schema[:actions][@action_name].scope == Types::ActionScope::GLOBAL + + selection = Utils::Id.parse_selection_ids(context.collection, args[:params], with_key: true) + return [] if selection[:are_excluded] + + selection[:ids].map { |record| Utils::Id.pack_id(context.collection, record) } + end + def middleware_custom_action_approval_request_data(args) raise Http::Exceptions::UnprocessableError if args.dig(:params, :data, :attributes, :requester_id) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb index 3f577efa9..1c0453270 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb @@ -137,11 +137,6 @@ def local_instant(zone, raw, boundary) end end end - - def store - config = ForestAdminAgent::Facades::Container.config_from_cache - config && config[:audit_trail] && config[:audit_trail][:store] - end end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb index 2743e4a7b..a2767d7cd 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_correlation.rb @@ -95,11 +95,6 @@ def parse_correlation_keys(args) keys.map { |key| key.to_s.strip }.reject(&:empty?) end - - def store - config = ForestAdminAgent::Facades::Container.config_from_cache - config && config[:audit_trail] && config[:audit_trail][:store] - end end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb index 1efa078b6..631c0c5ee 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb @@ -31,6 +31,10 @@ def serialize_record(record) record.to_h.transform_keys { |key| key.to_s.camelize(:lower) } end + def store + ::ForestAdminAgent::AuditTrail.store + end + def any_record?(context, collection, condition_tree) collection.list( context.caller, diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb new file mode 100644 index 000000000..413a0fa87 --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb @@ -0,0 +1,79 @@ +require 'spec_helper' + +module ForestAdminAgent + module AuditTrail + describe ActionCapture do + let(:store) do + Class.new do + attr_reader :records + + def initialize + @records = [] + end + + def append(record) + @records << record + end + end.new + end + + let(:caller_double) { double('caller', id: 42, request_id: 'req-xyz') } + + def record(capture: described_class.new(store), **over) + invocation = { caller: caller_double, collection: 'orders', action_name: 'Refund', + form_values: { 'amount' => 30, 'reason' => 'damaged' }, record_ids: ['4'] } + invocation.update(over) + + capture.record(**invocation) + store.records + end + + it 'records the invocation against the selected record' do + audit = record.last + + expect(audit.operation).to eq('action') + expect(audit.action_name).to eq('Refund') + expect(audit.collection).to eq('orders') + expect(audit.record_id).to eq('4') + expect(audit.user_id).to eq(42) + expect(audit.correlation_key).to eq('req-xyz') + expect(audit.previous_values).to eq({}) + expect(audit.new_values).to eq({ 'amount' => 30, 'reason' => 'damaged' }) + end + + it 'writes one row per selected record, sharing timestamp and correlation key' do + records = record(record_ids: %w[4 7 9]) + + expect(records.map(&:record_id)).to eq(%w[4 7 9]) + expect(records.map(&:timestamp).uniq.size).to eq(1) + expect(records.map(&:correlation_key).uniq).to eq(['req-xyz']) + end + + # Global actions and select-all selections name no record, but the run still has to leave a trace. + it 'records a run attached to no record when the selection is unknown' do + records = record(record_ids: []) + + expect(records.size).to eq(1) + expect(records.last.record_id).to eq('') + end + + it 'marks a failed run with its own operation' do + expect(record(failed: true).last.operation).to eq('action_failed') + end + + it 'masks redacted fields of the form, keeping the rest' do + capture = described_class.new(store, { 'orders' => ['reason'] }) + + audit = record(capture: capture).last + + expect(audit.new_values).to eq({ 'amount' => 30, 'reason' => Recording::REDACTED }) + end + + it 'falls back to a generated correlation key when the caller carries none' do + audit = record(caller: double('caller', id: 42, request_id: nil)).last + + expect(audit.correlation_key).to match(/\A[0-9a-f-]{36}\z/) + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb index 4b766d463..fdcba0a33 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb @@ -28,10 +28,21 @@ def connection store.append(record) expect(store.send(:model).column_names.sort).to eq( - %w[collection correlation_key id new_values operation previous_values record_id timestamp user_id] + %w[action_name collection correlation_key id new_values operation previous_values record_id + timestamp user_id] ) end + it 'persists the action name of a smart-action row' do + store.append(record(operation: 'action', action_name: 'Refund', previous_values: {}, + new_values: { 'amount' => 30 })) + + audit = store.list_by_record(collection: 'accounts', record_id: '1').first + expect(audit.operation).to eq('action') + expect(audit.action_name).to eq('Refund') + expect(audit.new_values).to eq({ 'amount' => 30 }) + end + it 'persists and reads back a record, decoding the JSON columns' do store.append(record) @@ -117,7 +128,10 @@ def connection described_class.new(database: { adapter: 'sqlite3', database: db.path }).append(record) names = connection.select_values('SELECT name FROM audit_migrations ORDER BY name') - expect(names).to eq(['audit_logs:001-create-audit-logs', 'audit_logs:002-index-record-and-correlation']) + expect(names).to eq( + ['audit_logs:001-create-audit-logs', 'audit_logs:002-index-record-and-correlation', + 'audit_logs:003-add-action-name'] + ) end it 'migrates a second table in the same database instead of reading the first one as done' do diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb new file mode 100644 index 000000000..838153a54 --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb @@ -0,0 +1,53 @@ +require 'spec_helper' + +module ForestAdminAgent + module Routes + module Action + include ForestAdminDatasourceToolkit::Schema + include ForestAdminDatasourceToolkit::Components::Query::ConditionTree + + describe Actions do + subject(:route) { described_class.new(collection, 'Refund') } + + let(:action_scope) { ForestAdminDatasourceCustomizer::Decorators::Action::Types::ActionScope::BULK } + let(:collection) do + build_collection( + name: 'orders', + schema: { + fields: { + 'id' => ColumnSchema.new( + column_type: 'Number', is_primary_key: true, + filter_operators: [Operators::IN, Operators::EQUAL] + ) + }, + actions: { 'Refund' => double('action', scope: action_scope) } + } + ) + end + let(:context) { double('context', collection: collection) } + + def audited_ids(attributes) + route.send(:audited_record_ids, { params: { data: { attributes: attributes } } }, context) + end + + it 'packs the ids of an explicit selection' do + expect(audited_ids({ ids: %w[4 7] })).to eq(%w[4 7]) + end + + # A select-all selection only carries the excluded ids: naming the targets would mean querying + # the whole selection, so the run is recorded unattached instead. + it 'returns no id for a select-all selection' do + expect(audited_ids({ ids: ['4'], all_records: true, all_records_ids_excluded: ['9'] })).to eq([]) + end + + context 'with a global action' do + let(:action_scope) { ForestAdminDatasourceCustomizer::Decorators::Action::Types::ActionScope::GLOBAL } + + it 'returns no id, since a global action targets no record' do + expect(audited_ids({ ids: %w[4 7] })).to eq([]) + end + end + end + end + end +end From bff005b0359afe48e720b254d5e8bf059496cf8e Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Wed, 12 Aug 2026 10:06:38 +0200 Subject: [PATCH 13/35] refactor(audit_trail): drop the action name, the activity logs already have it The Forest activity logs record which action ran, and `correlation_key` joins them to the audit rows, so storing the name again buys nothing. Migration 003 goes with it: the action row keeps `operation` (`action` / `action_failed`), the submitted form values and the targeted record ids. Anyone who ran the intermediate code keeps a stray nullable `action_name` column; nothing reads or writes it, and it can be dropped by hand. Co-Authored-By: Claude Opus 5 (1M context) --- .rubocop.yml | 1 - packages/forest_admin_agent/AUDIT_TRAIL.md | 15 ++++++++------- .../audit_trail/action_capture.rb | 6 +++--- .../audit_trail/audit_record.rb | 3 +-- .../audit_trail/sql/migrator.rb | 6 ------ .../lib/forest_admin_agent/audit_trail/store.rb | 2 -- .../forest_admin_agent/routes/action/actions.rb | 1 - .../audit_trail/action_capture_spec.rb | 3 +-- .../forest_admin_agent/audit_trail/store_spec.rb | 12 ++++-------- 9 files changed, 17 insertions(+), 32 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 4add98f46..4c07c2cbe 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -257,7 +257,6 @@ Naming/PredicatePrefix: Metrics/ParameterLists: Exclude: - - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/configuration.rb' diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index f378d6c9c..dab148e89 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -140,12 +140,14 @@ the store. A missing `collection` or `recordId` returns **HTTP 400** (`Validatio Running a smart action writes one row per selected record, in the same table: -| column | value | -| ------------ | ------------------------------------------------------------------------- | -| `operation` | `action` when it ran, `action_failed` when it raised | -| `actionName` | the action's name | -| `newValues` | the submitted form values (redacted with the same `redact` config) | -| `recordId` | each selected record — empty for a global action or a select-all selection | +| column | value | +| ----------- | -------------------------------------------------------------------------- | +| `operation` | `action` when it ran, `action_failed` when it raised | +| `newValues` | the submitted form values (redacted with the same `redact` config) | +| `recordId` | each selected record — empty for a global action or a select-all selection | + +**Which** action ran is not stored: the Forest activity logs already record it, and +`correlationKey` is the join between the two. A **global** action targets no record, and a **select-all** selection only tells the agent which ids were *excluded*, so naming the targets would mean querying the whole selection: those runs are recorded once, @@ -170,7 +172,6 @@ breaking the action. | `record_id` | packed record id (primary keys joined by `\|`) | | `user_id` | the Forest user who made the change | | `correlation_key` | per-request id; groups every change made within one request | -| `action_name` | smart-action rows only: the action that was run | | `previous_values` | values before the change (JSON) | | `new_values` | values after the change (JSON) | diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb index 27dd4c98a..ea08c7ce7 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb @@ -5,7 +5,8 @@ module AuditTrail # {Capture} cannot see them: the customizer has no `Execute` hook, and an action's writes are only # audited when it goes through the Forest data layer — a direct ORM write (`Customer.find(id).update!`) # is invisible to the agent and is therefore not recorded at all. What lands here is the invocation - # itself: who ran which action, on which records, with which form values, and whether it raised. + # itself: on which records, with which form values, and whether it raised. Which action it was lives in + # the Forest activity logs — `correlation_key` is the join. class ActionCapture include Recording @@ -20,7 +21,7 @@ def initialize(store, redact = {}) @redact = redact || {} end - def record(caller:, collection:, action_name:, form_values:, record_ids:, failed: false) + def record(caller:, collection:, form_values:, record_ids:, failed: false) timestamp = now correlation_key = correlation_key_for(caller) values = redact(form_values || {}, @redact[collection] || []) @@ -33,7 +34,6 @@ def record(caller:, collection:, action_name:, form_values:, record_ids:, failed operation: failed ? FAILED : EXECUTED, collection: collection, record_id: record_id, - action_name: action_name, user_id: caller&.id, correlation_key: correlation_key, previous_values: {}, diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb index 31139914a..011d4238a 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb @@ -2,10 +2,9 @@ module ForestAdminAgent module AuditTrail # One audited change. Mirrors the columns of `forest.audit_logs`; only the actor's `user_id` is # stored, the rest of the actor identity is correlated elsewhere through `correlation_key`. - # `action_name` is set on smart-action rows only. AuditRecord = Struct.new( :timestamp, :operation, :collection, :record_id, :user_id, :correlation_key, - :action_name, :previous_values, :new_values, + :previous_values, :new_values, keyword_init: true ) end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb index 1e3700f86..d05419afb 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb @@ -38,12 +38,6 @@ class Migrator connection.add_index(table, :correlation_key, name: "#{base}_correlation_key", if_not_exists: true) connection.add_index(table, :user_id, name: "#{base}_user_id", if_not_exists: true) end - }, - { - name: '003-add-action-name', - up: lambda do |connection, table| - connection.add_column(table, :action_name, :string, if_not_exists: true) - end } ].freeze diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb index 3da78d276..8c5d9b4e7 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb @@ -111,7 +111,6 @@ def to_row(record) record_id: record.record_id, user_id: record.user_id, correlation_key: record.correlation_key, - action_name: record.action_name, previous_values: record.previous_values, new_values: record.new_values } @@ -125,7 +124,6 @@ def from_row(row) record_id: row.record_id, user_id: row.user_id, correlation_key: row.correlation_key, - action_name: row.action_name, previous_values: row.previous_values || {}, new_values: row.new_values || {} ) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb index a654d3af0..626367a62 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb @@ -139,7 +139,6 @@ def audit_action(context, args, data, failed: false) capture.record( caller: context.caller, collection: context.collection.name, - action_name: @action_name, form_values: data, record_ids: audited_record_ids(args, context), failed: failed diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb index 413a0fa87..6a3c1cf45 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb @@ -20,7 +20,7 @@ def append(record) let(:caller_double) { double('caller', id: 42, request_id: 'req-xyz') } def record(capture: described_class.new(store), **over) - invocation = { caller: caller_double, collection: 'orders', action_name: 'Refund', + invocation = { caller: caller_double, collection: 'orders', form_values: { 'amount' => 30, 'reason' => 'damaged' }, record_ids: ['4'] } invocation.update(over) @@ -32,7 +32,6 @@ def record(capture: described_class.new(store), **over) audit = record.last expect(audit.operation).to eq('action') - expect(audit.action_name).to eq('Refund') expect(audit.collection).to eq('orders') expect(audit.record_id).to eq('4') expect(audit.user_id).to eq(42) diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb index fdcba0a33..c9c9d153b 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb @@ -28,18 +28,15 @@ def connection store.append(record) expect(store.send(:model).column_names.sort).to eq( - %w[action_name collection correlation_key id new_values operation previous_values record_id - timestamp user_id] + %w[collection correlation_key id new_values operation previous_values record_id timestamp user_id] ) end - it 'persists the action name of a smart-action row' do - store.append(record(operation: 'action', action_name: 'Refund', previous_values: {}, - new_values: { 'amount' => 30 })) + it 'persists a smart-action row, form values included' do + store.append(record(operation: 'action', previous_values: {}, new_values: { 'amount' => 30 })) audit = store.list_by_record(collection: 'accounts', record_id: '1').first expect(audit.operation).to eq('action') - expect(audit.action_name).to eq('Refund') expect(audit.new_values).to eq({ 'amount' => 30 }) end @@ -129,8 +126,7 @@ def connection names = connection.select_values('SELECT name FROM audit_migrations ORDER BY name') expect(names).to eq( - ['audit_logs:001-create-audit-logs', 'audit_logs:002-index-record-and-correlation', - 'audit_logs:003-add-action-name'] + ['audit_logs:001-create-audit-logs', 'audit_logs:002-index-record-and-correlation'] ) end From 31ad68a2c684fad71cfc23427540cb0d79d320f6 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Wed, 12 Aug 2026 10:54:00 +0200 Subject: [PATCH 14/35] fix(audit_trail): never let auditing break the request, and cover the gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recording a change raised through the after hook when the audit store was unavailable, so a write that had already succeeded came back as an error the client could retry into a duplicate. Auditing is best-effort now, in one place (`Recording#audit_safely`): the failure is logged and the row dropped, for the change capture, the snapshot read of a before hook (which would otherwise block the write) and the action capture alike. A failed snapshot read pushes an empty snapshot rather than nothing, so the after hook still pops its own entry instead of an unrelated one. Coverage of the new code, following the qlty report: - the action audit path, previously untested (72% diff coverage on actions.rb) - the Postgres-only migrator branches — advisory lock, schema creation and its concurrent-create rescue — through a fake adapter, no server needed - the wall-clock datetime filters, documented but never exercised, including an out-of-range instant - the correlation routes' unknown-collection 404 and its re-raise - three route registrations now dispatch through their closure instead of calling the handler, so the wiring is covered by the tests that were already there Every audit-trail file is at 100% except the `activerecord` LoadError guard, which needs a bundle without ActiveRecord to reach. Co-Authored-By: Claude Opus 5 (1M context) --- .rubocop.yml | 1 + .../audit_trail/action_capture.rb | 37 +++++---- .../forest_admin_agent/audit_trail/capture.rb | 34 +++++---- .../audit_trail/recording.rb | 11 +++ .../routes/action/actions.rb | 9 +-- .../audit_trail/action_capture_spec.rb | 15 ++++ .../audit_trail/capture_spec.rb | 25 ++++++ .../audit_trail/sql/migrator_spec.rb | 76 +++++++++++++++++++ .../routes/action/actions_spec.rb | 40 ++++++++++ .../resources/audit_trail_correlation_spec.rb | 41 +++++++++- .../routes/resources/audit_trail_spec.rb | 41 +++++++++- 11 files changed, 290 insertions(+), 40 deletions(-) create mode 100644 packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb diff --git a/.rubocop.yml b/.rubocop.yml index 4c07c2cbe..0fe2fd95f 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -458,6 +458,7 @@ RSpec/VerifiedDoubles: Exclude: - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb' - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb' + - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb' - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb' - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb' - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb' diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb index ea08c7ce7..f05d36210 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb @@ -21,25 +21,30 @@ def initialize(store, redact = {}) @redact = redact || {} end + # No-op unless an audit database was configured, and best-effort: the action has already run. def record(caller:, collection:, form_values:, record_ids:, failed: false) - timestamp = now - correlation_key = correlation_key_for(caller) - values = redact(form_values || {}, @redact[collection] || []) - ids = record_ids.empty? ? [NO_RECORD] : record_ids + return unless @store - ids.each do |record_id| - @store.append( - AuditRecord.new( - timestamp: timestamp, - operation: failed ? FAILED : EXECUTED, - collection: collection, - record_id: record_id, - user_id: caller&.id, - correlation_key: correlation_key, - previous_values: {}, - new_values: values + audit_safely do + timestamp = now + correlation_key = correlation_key_for(caller) + values = redact(form_values || {}, @redact[collection] || []) + ids = record_ids.empty? ? [NO_RECORD] : record_ids + + ids.each do |record_id| + @store.append( + AuditRecord.new( + timestamp: timestamp, + operation: failed ? FAILED : EXECUTED, + collection: collection, + record_id: record_id, + user_id: caller&.id, + correlation_key: correlation_key, + previous_values: {}, + new_values: values + ) ) - ) + end end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb index 6a8eb4f74..1bcc23118 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb @@ -53,7 +53,7 @@ def add_update_hooks(collection_customizer, columns, primary_keys, name, project collection_customizer.add_hook('Before', 'Update') do |context| # Snapshot the patch here: installed last, this hook sees what actually gets written, while # the after-context is handed the original patch the caller sent. - push_snapshot(records: context.collection.list(context.filter, projection), patch: context.patch) + push_snapshot(records: snapshot(context, projection), patch: context.patch) end collection_customizer.add_hook('After', 'Update') do |context| @@ -73,7 +73,7 @@ def add_update_hooks(collection_customizer, columns, primary_keys, name, project def add_delete_hooks(collection_customizer, columns, primary_keys, name, projection) collection_customizer.add_hook('Before', 'Delete') do |context| - push_snapshot(records: context.collection.list(context.filter, projection)) + push_snapshot(records: snapshot(context, projection)) end collection_customizer.add_hook('After', 'Delete') do |context| @@ -88,6 +88,12 @@ def add_delete_hooks(collection_customizer, columns, primary_keys, name, project end end + # An empty snapshot on failure rather than no snapshot at all: the after hook pops unconditionally, + # so skipping the push would pair it with an unrelated entry. + def snapshot(context, projection) + audit_safely { context.collection.list(context.filter, projection) } || [] + end + # Snapshots taken in a "before" hook and consumed in the matching "after" hook. Both bracket one # operation on one thread, so a LIFO stack pairs them without relying on the filter object # reaching both hooks unchanged. An operation raising in between strands its entry, hence the cap. @@ -104,18 +110,20 @@ def snapshots def emit(caller, operation, collection, record_id, previous_values, new_values) redacted = @redact[collection] || [] - @store.append( - AuditRecord.new( - timestamp: now, - operation: operation, - collection: collection, - record_id: record_id, - user_id: caller&.id, - correlation_key: correlation_key_for(caller), - previous_values: redact(previous_values, redacted), - new_values: redact(new_values, redacted) + audit_safely do + @store.append( + AuditRecord.new( + timestamp: now, + operation: operation, + collection: collection, + record_id: record_id, + user_id: caller&.id, + correlation_key: correlation_key_for(caller), + previous_values: redact(previous_values, redacted), + new_values: redact(new_values, redacted) + ) ) - ) + end end def record_id(record, primary_keys) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb index d99a6e8e2..76bcb39e6 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb @@ -25,6 +25,17 @@ def redact(values, redacted_fields) def now Time.now.utc.iso8601(3) end + + # Auditing must never take the request down with it. By the time a change is recorded the write has + # already happened, so raising would report a failure for an operation that succeeded (and invite a + # retry that duplicates it); a snapshot read failing must not block the write either. Losing the row + # is the lesser evil, so it is logged and dropped. + def audit_safely + yield + rescue StandardError => e + Facades::Container.logger.log('Error', "[ForestAdmin] Audit trail unavailable, skipping: #{e.message}") + nil + end end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb index 626367a62..fdb6e1bc2 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb @@ -118,7 +118,6 @@ def handle_hook_request(args = {}) private # A failed run is worth recording too: "who tried to run this" is usually the interesting part. - # The audit row is best-effort — a broken audit database must not fail the action itself. def execute_and_audit(context, args, data, filter) result = context.collection.execute(context.caller, @action_name, data, filter) audit_action(context, args, data) @@ -130,12 +129,10 @@ def execute_and_audit(context, args, data, filter) end def audit_action(context, args, data, failed: false) - store = ForestAdminAgent::AuditTrail.store - return unless store - capture = ForestAdminAgent::AuditTrail::ActionCapture.new( - store, ForestAdminAgent::AuditTrail.options[:redact] + ForestAdminAgent::AuditTrail.store, ForestAdminAgent::AuditTrail.options[:redact] ) + capture.record( caller: context.caller, collection: context.collection.name, @@ -143,8 +140,6 @@ def audit_action(context, args, data, failed: false) record_ids: audited_record_ids(args, context), failed: failed ) - rescue StandardError => e - Facades::Container.logger.log('Error', "[ForestAdmin] Could not audit action: #{e.message}") end # Packed ids, the form the audit store keys on. A global action targets nothing, and a select-all diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb index 6a3c1cf45..d20756c9d 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb @@ -68,6 +68,21 @@ def record(capture: described_class.new(store), **over) expect(audit.new_values).to eq({ 'amount' => 30, 'reason' => Recording::REDACTED }) end + it 'does nothing without a configured store' do + expect { record(capture: described_class.new(nil)) }.not_to raise_error + end + + # The action already ran: reporting a failure for it because the audit database is down would be + # worse than losing the row. + it 'logs and swallows a store failure instead of breaking the action' do + logger = instance_spy(Services::LoggerService) + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) + allow(store).to receive(:append).and_raise(StandardError, 'audit db is down') + + expect { record }.not_to raise_error + expect(logger).to have_received(:log).with('Error', /audit db is down/) + end + it 'falls back to a generated correlation key when the caller carries none' do audit = record(caller: double('caller', id: 42, request_id: nil)).last diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb index 3b298af21..8276b617b 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb @@ -156,6 +156,31 @@ def after_hook(type) expect(audit.new_values).to eq({}) end + # The write already happened when the after hook runs, so a broken audit database must not turn a + # successful change into a client-visible error. + it 'logs and swallows a store failure instead of failing the write' do + logger = instance_spy(Services::LoggerService) + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) + allow(store).to receive(:append).and_raise(StandardError, 'audit db is down') + + expect do + hooks['After_Create'].call(double('ctx', caller: caller_double, record: { 'id' => 1, 'name' => 'Acme' })) + end.not_to raise_error + expect(logger).to have_received(:log).with('Error', /audit db is down/) + end + + it 'does not block the write when the before-hook snapshot cannot be read' do + logger = instance_spy(Services::LoggerService) + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) + allow(relaxed_collection).to receive(:list).and_raise(StandardError, 'read failed') + + expect do + before_hook('Update', patch: { 'name' => 'Z' }) + after_hook('Update') + end.not_to raise_error + expect(store.records).to be_empty + end + it 'masks redacted fields while still recording the change' do described_class.new.run(datasource_customizer, nil, store: store, redact: { 'companies' => ['name'] }) diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb new file mode 100644 index 000000000..9f5de88b7 --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb @@ -0,0 +1,76 @@ +require 'spec_helper' + +module ForestAdminAgent + module AuditTrail + module Sql + # The store specs cover the SQLite path end to end; these pin the Postgres-only behaviour (schema + # creation and the advisory lock) without needing a Postgres server. + describe Migrator do + subject(:migrator) { described_class.new(connection, schema: 'forest', table_name: 'audit_logs') } + + let(:executed) { [] } + # A plain double on purpose: this fakes the Postgres adapter, which cannot be loaded here (no `pg` + # gem), and `quote_schema_name` only exists on it. + let(:connection) do + double( + 'PostgreSQL connection', + adapter_name: 'PostgreSQL', + create_table: nil, + add_index: nil, + add_column: nil, + select_values: ['forest.audit_logs:001-create-audit-logs', + 'forest.audit_logs:002-index-record-and-correlation'], + quote_schema_name: '"forest"', + quote_table_name: '"forest.audit_migrations"', + quote: "'x'" + ) + end + + before do + allow(connection).to receive(:execute) { |sql| executed << sql } + allow(connection).to receive(:transaction).and_yield + end + + it 'creates the schema before taking the lock, so the migrations can see it' do + migrator.run + + expect(executed.first).to eq('CREATE SCHEMA IF NOT EXISTS "forest"') + end + + it 'runs the migrations under a transaction-scoped advisory lock' do + migrator.run + + expect(connection).to have_received(:transaction) + expect(executed).to include('SELECT pg_advisory_xact_lock(17999, 21076)') + end + + def raise_on_create_schema(message) + allow(connection).to receive(:execute) do |sql| + raise ActiveRecord::StatementInvalid, message if sql.include?('CREATE SCHEMA') + + executed << sql + end + end + + it 'tolerates another instance having created the schema concurrently' do + raise_on_create_schema('ERROR: schema "forest" already exists') + + expect { migrator.run }.not_to raise_error + expect(executed).to include('SELECT pg_advisory_xact_lock(17999, 21076)') + end + + it 'still reports a schema creation that failed for another reason' do + raise_on_create_schema('ERROR: permission denied for database') + + expect { migrator.run }.to raise_error(ActiveRecord::StatementInvalid, /permission denied/) + end + + it 'skips migrations already applied to this table' do + migrator.run + + expect(connection).not_to have_received(:create_table).with('forest.audit_logs', any_args) + end + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb index 838153a54..93102e137 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb @@ -40,6 +40,46 @@ def audited_ids(attributes) expect(audited_ids({ ids: ['4'], all_records: true, all_records_ids_excluded: ['9'] })).to eq([]) end + describe 'auditing the run' do + let(:store) { double('store', append: nil) } + let(:context) { double('context', collection: collection, caller: build_caller) } + let(:args) { { params: { data: { attributes: { ids: %w[4 7] } } } } } + + before do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: store, redact: { 'orders' => ['reason'] } } }) + end + + def execute(result: 'ok', &raising) + allow(collection).to receive(:execute, &(raising || proc { result })) + route.send(:execute_and_audit, context, args, { 'amount' => 30, 'reason' => 'damaged' }, nil) + end + + it 'records one row per selected record and returns the action result' do + expect(execute).to eq('ok') + + expect(store).to have_received(:append).twice + expect(store).to have_received(:append).with( + having_attributes(operation: 'action', collection: 'orders', record_id: '4', + new_values: { 'amount' => 30, 'reason' => '[redacted]' }) + ) + end + + it 'records the attempt and re-raises when the action fails' do + expect { execute { raise StandardError, 'boom' } }.to raise_error(StandardError, 'boom') + + expect(store).to have_received(:append) + .with(having_attributes(operation: 'action_failed', record_id: '4')).once + end + + it 'records nothing when no audit database is configured' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache).and_return({}) + + expect(execute).to eq('ok') + expect(store).not_to have_received(:append) + end + end + context 'with a global action' do let(:action_scope) { ForestAdminDatasourceCustomizer::Decorators::Action::Types::ActionScope::GLOBAL } diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb index 66f5e9e7b..e215f7bec 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb @@ -70,7 +70,8 @@ def route_with_store(history: []) entry = { operation: 'update', record_id: '2', new_values: { 'first_name' => 'Jo' } } route = route_with_store(history: [double('entry', to_h: entry)]) - result = route.handle_history( + # Through the registered closure rather than the handler, so the wiring is covered too. + result = route.routes['forest_audit_trail_correlation'][:closure].call( { headers: {}, params: { 'collection' => 'books', 'recordId' => '2', 'correlation_key' => 'req-1' } } ) @@ -86,7 +87,7 @@ def route_with_store(history: []) it 'reads a batch history from comma-separated query keys (GET)' do route = route_with_store(history: [double('entry', to_h: { operation: 'update' })]) - route.handle_batch( + route.routes['forest_audit_trail_correlations'][:closure].call( { headers: {}, params: { 'collection' => 'books', 'recordId' => '2', 'correlationKeys' => 'a, b' } } ) @@ -98,7 +99,7 @@ def route_with_store(history: []) it 'reads a batch history from a body array (POST)' do route = route_with_store - route.handle_batch( + route.routes['forest_audit_trail_correlations_batch'][:closure].call( { headers: {}, params: { 'collection' => 'books', 'recordId' => '2', 'correlationKeys' => %w[a b] } } ) @@ -116,6 +117,40 @@ def route_with_store(history: []) expect(result[:content]).to eq({ data: [] }) end + it 'answers 404 for a collection the datasource does not know' do + route = route_with_store + datasource = double('datasource') + allow(datasource).to receive(:get_collection).with('ghosts') + .and_raise(ForestAdminDatasourceToolkit::Exceptions::ForestException, + "Collection 'ghosts' not found") + allow(route).to receive(:build).and_return( + double('context', datasource: datasource, caller: build_caller, permissions: permissions) + ) + + expect do + route.handle_history( + { headers: {}, params: { 'collection' => 'ghosts', 'recordId' => '2', 'correlation_key' => 'req-1' } } + ) + end.to raise_error(Http::Exceptions::NotFoundError, /not found/) + end + + it 'passes through a datasource error that is not a missing collection' do + route = route_with_store + datasource = double('datasource') + allow(datasource).to receive(:get_collection).with('books') + .and_raise(ForestAdminDatasourceToolkit::Exceptions::ForestException, + 'connection lost') + allow(route).to receive(:build).and_return( + double('context', datasource: datasource, caller: build_caller, permissions: permissions) + ) + + expect do + route.handle_history( + { headers: {}, params: { 'collection' => 'books', 'recordId' => '2', 'correlation_key' => 'req-1' } } + ) + end.to raise_error(ForestAdminDatasourceToolkit::Exceptions::ForestException, /connection lost/) + end + it 'rejects a missing collection' do route = route_with_store diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb index 9ade67934..194d7df67 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -52,7 +52,10 @@ def route_with_store(records: []) entry = { operation: 'update', record_id: '4', previous_values: { 'first_name' => 'Jo' } } route = route_with_store(records: [double('entry', to_h: entry)]) - result = route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) + # Through the registered closure rather than the handler, so the wiring is covered too. + result = route.routes['forest_audit_trail'][:closure].call( + { headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } } + ) expect(store).to have_received(:list_by_record).with( collection: 'projects', record_id: '4', skip: 0, limit: 20, order: 'desc' @@ -155,6 +158,42 @@ def route_with_store(records: []) expect(store).to have_received(:list_by_record).with(hash_including(start_timestamp: '2026-01-02T05:00:00.000Z')) end + it 'reads a wall-clock datetime, completing a minutes-only end boundary to :59.999' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'startDate' => '2026-01-02T08:30', 'endDate' => '2026-01-02 09:30' } }) + + expect(store).to have_received(:list_by_record).with( + hash_including(start_timestamp: '2026-01-02T08:30:00.000Z', + end_timestamp: '2026-01-02T09:30:59.999Z') + ) + end + + it 'keeps explicit seconds as given on both bounds' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'startDate' => '2026-01-02T08:30:15', + 'endDate' => '2026-01-02T09:30:45' } }) + + expect(store).to have_received(:list_by_record).with( + hash_including(start_timestamp: '2026-01-02T08:30:15.000Z', + end_timestamp: '2026-01-02T09:30:45.000Z') + ) + end + + # Right shape, impossible instant: the regex accepts it, the zone refuses to parse it. + it 'rejects a well-formed datetime that is out of range' do + route = route_with_store + + expect do + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'startDate' => '2026-01-02T99:00' } }) + end.to raise_error(Http::Exceptions::ValidationError, /Invalid date/) + end + it 'rejects an unparsable date' do route = route_with_store From d9948978427e9840f26298b9fbb10b27c77f766c Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Wed, 12 Aug 2026 11:00:33 +0200 Subject: [PATCH 15/35] docs(audit_trail): state that a concurrent overwrite can stale previous_values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hooks bracket the write as separate calls and the data layer exposes no lock or transaction primitive — deliberately, since it spans ActiveRecord, Mongoid, HTTP APIs and more. Two writes racing on one record therefore snapshot the same state, and the second records a previous_values that was already overwritten. Nothing to fix inside the capture layer: reading the snapshot in the write's transaction is not something a hook can do, and after the write the before-state is gone. Documented in the code and in AUDIT_TRAIL.md, next to the ORM-write limitation it belongs with. Co-Authored-By: Claude Opus 5 (1M context) --- packages/forest_admin_agent/AUDIT_TRAIL.md | 12 ++++++++++++ .../lib/forest_admin_agent/audit_trail/capture.rb | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index dab148e89..f5005c9d3 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -183,6 +183,18 @@ The `correlation_key` is the agent's per-request id (`caller.request_id`), gener echoed back to the client in the `X-Forest-Correlation-Id` response header — so every change made in one request shares a key, and the caller can tie it to its own activity log. +## Concurrent writes to one record + +The before/after values are captured around the write, not inside it: the customizer hooks bracket the +write as separate calls, and the data layer deliberately exposes no lock or transaction primitive since +it spans ActiveRecord, Mongoid, HTTP APIs and more. + +So when two writes race on the same record, both snapshot the same state and the one that lands second +records a `previousValues` that had already been overwritten. `newValues` is always exact — it is the +patch that was written — and no row is ever lost; only the prior state of an overlapping write can be +stale. Exact before-images under concurrency need the database itself (triggers, or CDC), not an agent +hook. + ## Schema migrations & concurrency The table is created/evolved through an ordered, append-only migration list tracked in a dedicated diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb index 1bcc23118..4d30a3ad1 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb @@ -88,6 +88,11 @@ def add_delete_hooks(collection_customizer, columns, primary_keys, name, project end end + # Read outside the write's transaction — hooks bracket the write as separate calls and the data layer + # exposes no lock, on purpose, since it spans ActiveRecord, Mongoid, HTTP APIs... So two updates + # racing on the same record both snapshot the same state, and the one that lands second records a + # `previous_values` that was already overwritten. `new_values` stays exact (it is the patch). + # # An empty snapshot on failure rather than no snapshot at all: the after hook pops unconditionally, # so skipping the push would pair it with an unrelated entry. def snapshot(context, projection) From 7858c43d6eb46d97ba7613f989c5457b9d610ace Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Wed, 12 Aug 2026 11:07:51 +0200 Subject: [PATCH 16/35] fix(audit_trail): record the write even when another after hook raises MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Hooks#execute_after` stops at the first exception, and the capture layer is installed last, so any customization raising in its own after hook dropped the audit row of a write that had already happened — and stranded its snapshot. `add_hook` takes a `prepend` flag now (optional, defaults to appending as before), and the capture layer prepends its after hooks so nothing can preempt them. Its before hooks stay appended, so the snapshot still reads the filter and patch every other customization has had its say on. Co-Authored-By: Claude Opus 5 (1M context) --- packages/forest_admin_agent/AUDIT_TRAIL.md | 4 +++ .../forest_admin_agent/audit_trail/capture.rb | 10 ++++--- .../audit_trail/capture_spec.rb | 15 ++++++++++- .../collection_customizer.rb | 6 +++-- .../hook/hook_collection_decorator.rb | 4 +-- .../decorators/hook/hooks.rb | 9 +++++-- .../decorators/hook/hooks_spec.rb | 26 +++++++++++++++++++ 7 files changed, 64 insertions(+), 10 deletions(-) diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index f5005c9d3..acb870047 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -183,6 +183,10 @@ The `correlation_key` is the agent's per-request id (`caller.request_id`), gener echoed back to the client in the `X-Forest-Correlation-Id` response header — so every change made in one request shares a key, and the caller can tie it to its own activity log. +The capture layer registers its **after** hooks ahead of any other customization's (`prepend: true`, +since `execute_after` stops at the first exception, and by then the write has already happened) and its +**before** hooks after them, so the snapshot sees the filter and patch everyone else has had their say on. + ## Concurrent writes to one record The before/after values are captured around the write, not inside it: the customizer hooks bracket the diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb index 4d30a3ad1..55d397ef4 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb @@ -40,8 +40,12 @@ def instrument(collection_customizer) add_delete_hooks(collection_customizer, columns, primary_keys, name, projection) end + # The "after" hooks are prepended: `execute_after` stops at the first exception, so a customization + # raising in its own after hook would otherwise drop the record of a write that already happened. + # The "before" hooks stay appended, so they read the filter and patch every other customization + # has had its say on. def add_create_hook(collection_customizer, columns, primary_keys, name) - collection_customizer.add_hook('After', 'Create') do |context| + collection_customizer.add_hook('After', 'Create', prepend: true) do |context| emit( context.caller, 'create', name, record_id(context.record, primary_keys), {}, pick(context.record, columns) @@ -56,7 +60,7 @@ def add_update_hooks(collection_customizer, columns, primary_keys, name, project push_snapshot(records: snapshot(context, projection), patch: context.patch) end - collection_customizer.add_hook('After', 'Update') do |context| + collection_customizer.add_hook('After', 'Update', prepend: true) do |context| snapshot = snapshots.pop snapshot&.fetch(:records)&.each do |record| @@ -76,7 +80,7 @@ def add_delete_hooks(collection_customizer, columns, primary_keys, name, project push_snapshot(records: snapshot(context, projection)) end - collection_customizer.add_hook('After', 'Delete') do |context| + collection_customizer.add_hook('After', 'Delete', prepend: true) do |context| snapshot = snapshots.pop snapshot&.fetch(:records)&.each do |record| diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb index 8276b617b..841707bb1 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb @@ -37,9 +37,13 @@ def append(record) instance_double(ForestAdminDatasourceCustomizer::Context::RelaxedWrappers::RelaxedCollection) end + let(:registrations) { [] } let(:collection_customizer) do customizer = double('CollectionCustomizer', name: 'companies', collection: collection) - allow(customizer).to receive(:add_hook) { |position, type, &block| hooks["#{position}_#{type}"] = block } + allow(customizer).to receive(:add_hook) do |position, type, prepend: false, &block| + registrations << [position, type, prepend] + hooks["#{position}_#{type}"] = block + end customizer end @@ -53,6 +57,15 @@ def append(record) described_class.new.run(datasource_customizer, nil, store: store) end + # execute_after stops at the first exception, so being last would mean losing the record of a write + # that already happened whenever another customization raises in its own after hook. + it 'registers its after hooks ahead of the ones already there, and its before hooks after them' do + expect(registrations).to contain_exactly( + ['After', 'Create', true], ['Before', 'Update', false], ['After', 'Update', true], + ['Before', 'Delete', false], ['After', 'Delete', true] + ) + end + it 'records a create with only the writable columns' do record = { 'id' => 1, 'name' => 'Acme', 'address' => { 'city' => 'Paris' } } diff --git a/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/collection_customizer.rb b/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/collection_customizer.rb index c4d3ee68d..7cdab3d46 100644 --- a/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/collection_customizer.rb +++ b/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/collection_customizer.rb @@ -263,8 +263,10 @@ def add_chart(name, &definition) # .add_hook('before', 'list') do |context| # # Do something before the list action # end - def add_hook(position, type, &handler) - push_customization { @stack.hook.get_collection(@name).add_hook(position, type, handler) } + def add_hook(position, type, prepend: false, &handler) + push_customization do + @stack.hook.get_collection(@name).add_hook(position, type, handler, prepend: prepend) + end end # Add a new segment on the collection. diff --git a/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hook_collection_decorator.rb b/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hook_collection_decorator.rb index f12455a27..9e47ebd9e 100644 --- a/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hook_collection_decorator.rb +++ b/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hook_collection_decorator.rb @@ -18,8 +18,8 @@ def initialize(child_collection, datasource) } end - def add_hook(position, type, hook) - @hooks[type].add_handler(position, hook) + def add_hook(position, type, hook, prepend: false) + @hooks[type].add_handler(position, hook, prepend: prepend) end def create(caller, data) diff --git a/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hooks.rb b/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hooks.rb index fc644bb5a..a249bf48e 100644 --- a/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hooks.rb +++ b/packages/forest_admin_datasource_customizer/lib/forest_admin_datasource_customizer/decorators/hook/hooks.rb @@ -17,8 +17,13 @@ def execute_after(context) @after.each { |hook| hook.call(context) } end - def add_handler(position, hook) - position == 'After' ? @after << hook : @before << hook + # `prepend` puts the handler ahead of the ones already registered. `execute_after` stops at the + # first exception, so a handler that must run whatever a sibling does (the audit trail recording a + # write that already happened) cannot afford to be last. + def add_handler(position, hook, prepend: false) + handlers = position == 'After' ? @after : @before + + prepend ? handlers.unshift(hook) : handlers.push(hook) end end end diff --git a/packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/decorators/hook/hooks_spec.rb b/packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/decorators/hook/hooks_spec.rb index a053293e4..d3f0af571 100644 --- a/packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/decorators/hook/hooks_spec.rb +++ b/packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/decorators/hook/hooks_spec.rb @@ -80,6 +80,32 @@ def initialize end end + describe 'add_handler with prepend' do + it 'runs the prepended handler before the ones already registered' do + calls = [] + + hooks = described_class.new + hooks.add_handler('After', proc { calls << :existing }) + hooks.add_handler('After', proc { calls << :prepended }, prepend: true) + hooks.execute_after(fake_hook_context.new) + + expect(calls).to eq(%i[prepended existing]) + end + + # execute_after stops at the first exception: a handler that must run whatever a sibling does + # cannot afford to be last. + it 'runs the prepended handler even when a later one raises' do + calls = [] + + hooks = described_class.new + hooks.add_handler('After', proc { raise 'boom' }) + hooks.add_handler('After', proc { calls << :prepended }, prepend: true) + + expect { hooks.execute_after(fake_hook_context.new) }.to raise_error('boom') + expect(calls).to eq([:prepended]) + end + end + describe 'execute_after' do describe 'when multiple after hooks are defined' do it 'call all of them' do From cee3d218c9fe6cd97dd1096b63aed74ef86df193 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Thu, 13 Aug 2026 10:04:38 +0200 Subject: [PATCH 17/35] feat(audit_trail): reconstruct a record's state, filter history by field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parity with the Node agent, plus the two gaps found on the way. - `GET /_audit-trail/:collection/:id/state?timestamp=` rebuilds a record by undoing every entry recorded strictly after the instant (an entry stamped exactly at it belongs to that state). A create means the record did not exist then; a delete restores the row it recorded and the walk carries on into an earlier life of the same id. - The diff now leaves a key out of the side where it does not exist instead of writing nil, so "key added" and "key holding nil" stay tellable apart — `Diff.revert` needs that, and a round-trip property spec pins it. No sentinel value reaches the database. - `fields` filter on the history route, with the JSON key test per adapter (Postgres / SQLite / MySQL, raising on anything else) in its own FieldFilter class. Field names are quoted as whole keys, so a name holding a dot is not read as a path, and both sides of the diff are searched since an added key exists on the new side only. - The correlation-id middleware goes ahead of ShowExceptions: a Rack middleware can only add a header to a response it sees returned, so sitting under the exception handlers meant the 500s they build never carried the id. - The schema-already-exists rescue checks SQLSTATE (42P06 / 23505) where the adapter exposes one, and RecordNotUnique by class, instead of matching "already exists" anywhere in a message. Co-Authored-By: Claude Opus 5 (1M context) --- .rubocop.yml | 1 + packages/forest_admin_agent/AUDIT_TRAIL.md | 26 +++++ .../forest_admin_agent/audit_trail/diff.rb | 68 ++++++++++-- .../audit_trail/record_state.rb | 42 ++++++++ .../audit_trail/sql/field_filter.rb | 58 ++++++++++ .../audit_trail/sql/migrator.rb | 27 ++++- .../forest_admin_agent/audit_trail/store.rb | 28 +++-- .../routes/resources/audit_trail.rb | 58 +++++++++- .../routes/resources/audit_trail_route.rb | 12 +++ .../audit_trail/diff_spec.rb | 78 +++++++++++++- .../audit_trail/record_state_spec.rb | 100 ++++++++++++++++++ .../audit_trail/sql/migrator_spec.rb | 38 +++++++ .../audit_trail/store_spec.rb | 64 +++++++++++ .../routes/resources/audit_trail_spec.rb | 99 +++++++++++++++++ .../lib/forest_admin_rails/engine.rb | 14 ++- .../lib/forest_admin_rails/engine_spec.rb | 33 ++++++ 16 files changed, 722 insertions(+), 24 deletions(-) create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/record_state.rb create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/field_filter.rb create mode 100644 packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/record_state_spec.rb diff --git a/.rubocop.yml b/.rubocop.yml index 0fe2fd95f..e0e1dfb6d 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -359,6 +359,7 @@ Metrics/BlockLength: Metrics/ClassLength: Exclude: + - 'packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/collection.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/introspection/introspector.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/introspection/schema_converter.rb' diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index acb870047..f75fb46f4 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -79,6 +79,12 @@ independent of the page. Optional filters (all combine with `AND`; omit them for | `userIds` | comma-separated integers `12,45` | keep only entries whose `user_id` is in the list | | `startDate` | `YYYY-MM-DD` or datetime (incl.) | keep entries from this lower bound onward | | `endDate` | `YYYY-MM-DD` or datetime (incl.) | keep entries up to this upper bound | +| `fields` | comma-separated field names | keep only entries whose diff touched one of them | + +`fields` matches whole keys, never paths, so a name holding a dot (`address.city`) is quoted before it +reaches SQL. Both sides of the diff are searched, since a field the change added exists in `newValues` +only and one it removed in `previousValues` only. The JSON test is per adapter (Postgres, SQLite, MySQL / +MariaDB); on any other adapter the filter raises rather than silently returning everything. `startDate` / `endDate` are read as **local wall-clock time** in the request `timezone` query param (e.g. `Europe/Paris`, default `UTC`) and converted to a UTC instant before querying, so filtering @@ -106,6 +112,26 @@ A record that no longer exists keeps its history: only a record that still exist caller's permission scope is refused (404). Inspecting what was deleted is much of the point of an audit trail, and the delete event itself is the last thing recorded. +### State route + +`GET /forest/_audit-trail/{collection}/{recordId}/state?timestamp=…` returns the record as it stood at +that instant, rebuilt by taking the record as it stands now and undoing every entry recorded **strictly +after** the timestamp — an entry stamped exactly at it counts as part of that state: + +```json +{ "data": { "status": "paid", "address": { "city": "Paris" } }, "meta": { "timestamp": "…", "reverted": 3 } } +``` + +`timestamp` accepts an ISO-8601 instant, or the same wall-clock forms as the filters above read in the +request `timezone`; it is required (**400** otherwise). `data` is `null` when the record did not exist at +that instant — either created later, or deleted and never recreated. + +Walking back stops being able to help where the trail stops: only audited (writable) columns are +reconstructed, and a `create` means the record did not exist before it, while a `delete` restores the whole +row it recorded and the walk carries on into any earlier life of the same id. + +> The exact response shape is not yet pinned against a consumer — no frontend reads this endpoint today. + ### Correlation route `GET /forest/_audit-trail/correlation/{correlationKey}` returns `{ "data": [...] }` — the diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb index 8d8ed7a1a..cf92fdfb2 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb @@ -8,6 +8,11 @@ module AuditTrail # Ruby's `==` already performs deep, key-order-independent equality on Hash and (ordered) equality # on Array, so it is used directly as the equality primitive. module Diff + # A key that does not exist on one side of the diff. It is never stored: the key is simply left out + # of that side's hash, so `{"flag" => nil}` (a key holding nil) and `{}` (no key) stay tellable + # apart in the database — which a revert needs and a string sentinel would only fake. + ABSENT = Object.new.freeze + module_function # @return [Hash{Symbol=>Object}, nil] { previous:, next: } of the changed leaves, or nil when equal. @@ -55,19 +60,22 @@ def diff_hashes(before, after) sub = diff_at(before, after, key) next unless sub - previous[key] = sub[:previous] - next_values[key] = sub[:next] + previous[key] = sub[:previous] unless sub[:previous].equal?(ABSENT) + next_values[key] = sub[:next] unless sub[:next].equal?(ABSENT) end { previous: previous, next: next_values } end - # A key held with a nil value is not the same thing as a missing key: recursing on the values - # alone reads both as nil and reports no change at all. + # A key held with a nil value is not the same thing as a missing key: recursing on the values alone + # reads both as nil and reports no change at all. def diff_at(before, after, key) return diff(before[key], after[key]) if before.key?(key) == after.key?(key) - { previous: before[key], next: after[key] } + { + previous: before.key?(key) ? before[key] : ABSENT, + next: after.key?(key) ? after[key] : ABSENT + } end def diff_object_arrays(before, after) @@ -85,7 +93,55 @@ def diff_object_arrays(before, after) { previous: previous, next: next_values } end - private_class_method :diff_hashes, :diff_at, :diff_object_arrays + # Undo one recorded change: given the value as it stands now and the two sides of the diff that + # produced it, return the value as it was before. Nested hashes are walked so untouched keys keep + # their current value; a key missing from `previous` was added by the change, so it goes away. + # + # @param current [Object] the value as it stands now (may be nil when the record is gone) + # @param previous [Object] the `previous_values` side of the recorded diff + # @param changed [Object] the `new_values` side of the recorded diff + def revert(current, previous, changed) + return revert_array(current, previous, changed) if current.is_a?(Array) && partial?(previous, changed) + return revert_hash(current, previous, changed) if current.is_a?(Hash) && partial?(previous, changed) + + previous + end + + # Both sides being hashes is what `diff` emits for a structural diff; anything else replaced the + # value as a whole. + def partial?(previous, changed) + previous.is_a?(Hash) && changed.is_a?(Hash) + end + + def revert_hash(current, previous, changed) + (previous.keys | changed.keys).each_with_object(current.dup) do |key, result| + if previous.key?(key) + result[key] = revert(current[key], previous[key], changed[key]) + else + # Only the change introduced this key, so before the change there was none. + result.delete(key) + end + end + end + + # Arrays of objects are diffed index by index, and JSON turns those indexes into strings on the way + # back out. An element the change appended is dropped, which can only ever shorten the tail. + def revert_array(current, previous, changed) + result = current.dup + + (previous.keys | changed.keys).map(&:to_i).sort.reverse_each do |index| + key = index.to_s + if previous.key?(key) || previous.key?(index) + result[index] = revert(current[index], previous[key] || previous[index], changed[key] || changed[index]) + else + result.delete_at(index) + end + end + + result + end + + private_class_method :diff_hashes, :diff_at, :diff_object_arrays, :partial?, :revert_hash, :revert_array end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/record_state.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/record_state.rb new file mode 100644 index 000000000..accb43386 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/record_state.rb @@ -0,0 +1,42 @@ +module ForestAdminAgent + module AuditTrail + # Rebuilds a record as it stood at a given instant by walking its history backwards from the record as + # it stands now, undoing every entry recorded after that instant (newest first). + # + # Only audited columns are reconstructed — read-only, computed and DB-managed fields are never recorded, + # so they cannot be restored. + module RecordState + module_function + + # @param current [Hash, nil] the record as it stands now, nil when it no longer exists + # @param entries [Array] entries recorded after the instant, newest first + # @return [Hash, nil] the record at that instant, nil when it did not exist then + def at(current, entries) + entries.reduce(current) { |state, entry| undo(state, entry) } + end + + def undo(state, entry) + case entry.operation + when 'create' + # Created after the instant, so it did not exist then. An older entry can still bring a previous + # life of the same id back — the walk carries on. + nil + when 'delete' + # The delete recorded the whole record, which is exactly the state it was deleted from. + entry.previous_values + when 'update' + revert_columns(state, entry) + else + # Action rows carry no field change. + state + end + end + + def revert_columns(state, entry) + entry.previous_values.each_with_object((state || {}).dup) do |(column, previous), result| + result[column] = Diff.revert(result[column], previous, entry.new_values[column]) + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/field_filter.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/field_filter.rb new file mode 100644 index 000000000..a29737e9f --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/field_filter.rb @@ -0,0 +1,58 @@ +module ForestAdminAgent + module AuditTrail + module Sql + # SQL keeping only the audit entries whose diff touched one of the given fields. Both JSON columns are + # searched: a field the change added exists in `new_values` only, one it removed in `previous_values` + # only. + # + # The test is per adapter, and a field name is always a whole key — never a path — so a name holding a + # dot (`address.city`) has to be quoted or the database reads it as a traversal. + class FieldFilter + COLUMNS = %w[previous_values new_values].freeze + + def initialize(connection) + @connection = connection + end + + def condition(fields) + adapter = @connection.adapter_name.downcase + + case adapter + when /postgres/ then COLUMNS.map { |column| postgres_has_key(column, fields) }.join(' OR ') + when /sqlite/ then json_paths(fields) { |column, path| "json_type(#{column}, #{path}) IS NOT NULL" } + when /mysql|maria/ then mysql_has_keys(fields) + else + raise ForestAdminDatasourceToolkit::Exceptions::ForestException, + "Filtering the audit trail by field is not supported on #{adapter}" + end + end + + private + + # `jsonb_object_keys` rather than the `?|` operator: `?` is a bind placeholder for ActiveRecord and + # the function form needs no escaping. The column is `json`, hence the cast. + def postgres_has_key(column, fields) + list = fields.map { |field| @connection.quote(field) }.join(', ') + + "EXISTS (SELECT 1 FROM jsonb_object_keys(#{column}::jsonb) AS key WHERE key IN (#{list}))" + end + + # `json_type` and not `json_extract`: a key holding a JSON null extracts as SQL NULL, which would + # read as "no such key". + def json_paths(fields) + COLUMNS.flat_map { |column| fields.map { |field| yield(column, json_path(field)) } }.join(' OR ') + end + + def mysql_has_keys(fields) + paths = fields.map { |field| json_path(field) }.join(', ') + + COLUMNS.map { |column| "JSON_CONTAINS_PATH(#{column}, 'one', #{paths})" }.join(' OR ') + end + + def json_path(field) + @connection.quote(%($."#{field.to_s.gsub(/["\\]/) { |char| "\\#{char}" }}")) + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb index d05419afb..294151703 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb @@ -12,6 +12,8 @@ class Migrator MIGRATIONS_TABLE = 'audit_migrations'.freeze # Arbitrary but stable key pair identifying the audit-trail migration critical section. ADVISORY_LOCK = [0x464f, 0x5254].freeze # "FO", "RT" + # duplicate_schema, and the unique violation on pg_namespace the same race can raise instead. + DUPLICATE_SCHEMA_STATES = %w[42P06 23505].freeze MIGRATIONS = [ { @@ -80,9 +82,30 @@ def ensure_schema return unless schema? @connection.execute("CREATE SCHEMA IF NOT EXISTS #{@connection.quote_schema_name(@schema)}") + rescue ActiveRecord::RecordNotUnique + # 23505 on pg_namespace, already mapped to its own class by ActiveRecord: another instance + # created the schema between our IF NOT EXISTS check and the create itself. + nil rescue ActiveRecord::StatementInvalid => e - # Another instance created it concurrently — CREATE SCHEMA is not fully race-free. - raise unless /already exists|duplicate/i.match?(e.message) + raise unless duplicate_schema?(e) + end + + # By SQLSTATE where the adapter exposes one, so an unrelated failure (no permission to create a + # schema, say) is not read as a lost race just because its message says "exists". + def duplicate_schema?(error) + state = sql_state(error) + return DUPLICATE_SCHEMA_STATES.include?(state) if state + + /already exists|duplicate/i.match?(error.message) + end + + def sql_state(error) + cause = error.cause + return nil unless cause.respond_to?(:result) && defined?(PG::Result) + + cause.result.error_field(PG::Result::PG_DIAG_SQLSTATE) + rescue StandardError + nil end def apply_pending diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb index 8c5d9b4e7..92be245ae 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb @@ -22,11 +22,11 @@ def append(record) model.create!(to_row(record)) end - def list_by_record(collection:, record_id:, skip: 0, limit: nil, - user_ids: nil, start_timestamp: nil, end_timestamp: nil, order: 'asc') + def list_by_record(collection:, record_id:, skip: 0, limit: nil, user_ids: nil, + start_timestamp: nil, end_timestamp: nil, fields: nil, order: 'asc') # `id` (insertion order) breaks ties on equal timestamps in both directions, keeping pages # deterministic and stable. - relation = scope(collection, record_id, user_ids, start_timestamp, end_timestamp) + relation = scope(collection, record_id, user_ids, start_timestamp, end_timestamp, fields) .order(timestamp: order.to_s == 'desc' ? :desc : :asc, id: :asc) .offset(skip || 0) relation = relation.limit(limit) unless limit.nil? @@ -34,16 +34,25 @@ def list_by_record(collection:, record_id:, skip: 0, limit: nil, relation.map { |row| from_row(row) } end - def count_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil, end_timestamp: nil) - scope(collection, record_id, user_ids, start_timestamp, end_timestamp).count + def count_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil, + end_timestamp: nil, fields: nil) + scope(collection, record_id, user_ids, start_timestamp, end_timestamp, fields).count end - def list_by_correlation(collection:, record_id:, correlation_key:) - model.where(collection: collection, record_id: record_id, correlation_key: correlation_key) - .order(:timestamp, :id) + # Entries recorded strictly after `timestamp`, newest first: what a state reconstruction has to undo. + # Strictly after, so an entry stamped exactly at the requested instant counts as part of that state + # instead of being reverted out of it. + def list_since(collection:, record_id:, timestamp:) + model.where(collection: collection, record_id: record_id) + .where('timestamp > ?', as_time(timestamp)) + .order(timestamp: :desc, id: :desc) .map { |row| from_row(row) } end + def list_by_correlation(collection:, record_id:, correlation_key:) + list_by_correlations(collection: collection, record_id: record_id, correlation_keys: [correlation_key]) + end + def list_by_correlations(collection:, record_id:, correlation_keys:) return [] if correlation_keys.empty? @@ -54,9 +63,10 @@ def list_by_correlations(collection:, record_id:, correlation_keys:) private - def scope(collection, record_id, user_ids, start_timestamp, end_timestamp) + def scope(collection, record_id, user_ids, start_timestamp, end_timestamp, fields = nil) relation = model.where(collection: collection, record_id: record_id) relation = relation.where(user_id: user_ids) if user_ids + relation = relation.where(Sql::FieldFilter.new(model.connection).condition(fields)) if fields&.any? # Compare as Time so ActiveRecord casts the bound to the datetime column's storage format # (raw ISO strings with a `Z` would compare lexically against the cast rows and never match). relation = relation.where('timestamp >= ?', as_time(start_timestamp)) if start_timestamp diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb index 1c0453270..ca0f1055e 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb @@ -26,6 +26,12 @@ def setup_routes '/_audit-trail/:collection_name/:id', ->(args) { handle_request(args) } ) + add_route( + 'forest_audit_trail_state', + 'get', + '/_audit-trail/:collection_name/:id/state', + ->(args) { handle_state(args) } + ) self end @@ -53,8 +59,42 @@ def handle_request(args = {}) } end + # Record as it stood at `timestamp`: the current record with every later entry undone. `data` is + # null when the record did not exist yet (or not any more) at that instant. + def handle_state(args = {}) + context = build(args) + context.permissions.can?(:read, context.collection) + assert_record_in_scope(context, context.collection, args[:params]['id']) + + timestamp = parse_state_timestamp(args) + entries = store.list_since( + collection: context.collection.name, record_id: args[:params]['id'], timestamp: timestamp + ) + # Fully qualified: inside this class, `AuditTrail` is the route itself. + current = current_record(context, context.collection, args[:params]['id']) + state = ::ForestAdminAgent::AuditTrail::RecordState.at(current, entries) + + { + name: args[:params]['collection_name'], + content: { data: state, meta: { timestamp: timestamp, reverted: entries.size } } + } + end + private + # An ISO-8601 instant, or the same wall-clock forms the history filters accept, read in the request + # timezone. + def parse_state_timestamp(args) + raw = args.dig(:params, 'timestamp').to_s + raise Http::Exceptions::ValidationError, 'Missing timestamp' if raw.empty? + + begin + Time.iso8601(raw).utc.iso8601(3) + rescue ArgumentError + parse_date_boundary(raw, request_timezone(args), :start) + end + end + # JSON:API `sort`: `timestamp` → oldest first, anything else (absent/unsupported) → newest first. def parse_sort(args) args.dig(:params, 'sort').to_s == 'timestamp' ? 'asc' : 'desc' @@ -77,17 +117,31 @@ def parse_pagination(args) [(number - 1) * size, size] end - def parse_filters(args) + def request_timezone(args) timezone = args.dig(:params, 'timezone').to_s - timezone = 'UTC' if timezone.empty? + + timezone.empty? ? 'UTC' : timezone + end + + def parse_filters(args) + timezone = request_timezone(args) { user_ids: parse_user_ids(args.dig(:params, 'userIds')), + fields: parse_fields(args.dig(:params, 'fields')), start_timestamp: parse_date_boundary(args.dig(:params, 'startDate'), timezone, :start), end_timestamp: parse_date_boundary(args.dig(:params, 'endDate'), timezone, :end) }.compact end + # Comma-separated field names, kept verbatim (a name may hold a dot). Empty after parsing → no filter. + def parse_fields(raw) + return nil if raw.nil? + + names = (raw.is_a?(Array) ? raw : raw.to_s.split(',')).map { |name| name.to_s.strip }.reject(&:empty?) + names.empty? ? nil : names + end + # Comma-separated integer ids; non-numeric tokens are dropped. Empty after parsing → no filter. def parse_user_ids(raw) return nil if raw.nil? || raw.to_s.empty? diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb index 631c0c5ee..3e37f9278 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb @@ -35,6 +35,18 @@ def store ::ForestAdminAgent::AuditTrail.store end + # The audited columns as they stand now — the starting point of a state reconstruction. No scope in + # the filter: the caller was already authorized against it, and a record that is simply gone has to + # come back as nil so the history can restore it. + def current_record(context, collection, packed_id) + columns = collection.schema[:fields].select { |_name, field| field.type == 'Column' }.keys + condition = ConditionTree::ConditionTreeFactory.match_records( + collection, [Utils::Id.unpack_id(collection, packed_id, with_key: true)] + ) + + collection.list(context.caller, Filter.new(condition_tree: condition), Projection.new(columns)).first + end + def any_record?(context, collection, condition_tree) collection.list( context.caller, diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb index 10405b2eb..cc3f0f0f1 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb @@ -18,16 +18,18 @@ module AuditTrail ) end - it 'reports a key that disappeared, even when its value was nil' do + # A key holding nil and a missing key have to stay tellable apart, so the side where the key does + # not exist simply does not carry it. Nothing sentinel-shaped reaches the database. + it 'reports a key that disappeared by leaving it out of the new values' do expect(described_class.diff({ 'flag' => nil }, {})).to eq( previous: { 'flag' => nil }, - next: { 'flag' => nil } + next: {} ) end - it 'reports a key that appeared holding nil' do + it 'reports a key that appeared by leaving it out of the previous values' do expect(described_class.diff({}, { 'flag' => nil })).to eq( - previous: { 'flag' => nil }, + previous: {}, next: { 'flag' => nil } ) end @@ -52,6 +54,74 @@ module AuditTrail end end + describe '.revert' do + it 'restores a scalar' do + expect(described_class.revert('Lyon', 'Paris', 'Lyon')).to eq('Paris') + end + + it 'restores only the touched keys, leaving the rest of the object alone' do + current = { 'city' => 'Lyon', 'zip' => '69001', 'country' => 'FR' } + + expect(described_class.revert(current, { 'city' => 'Paris' }, { 'city' => 'Lyon' })).to eq( + { 'city' => 'Paris', 'zip' => '69001', 'country' => 'FR' } + ) + end + + it 'removes a key the change had added' do + expect(described_class.revert({ 'city' => 'Lyon', 'zip' => '69001' }, {}, { 'zip' => '69001' })).to eq( + { 'city' => 'Lyon' } + ) + end + + it 'puts back a key the change had removed, nil value included' do + expect(described_class.revert({ 'city' => 'Lyon' }, { 'zip' => nil }, {})).to eq( + { 'city' => 'Lyon', 'zip' => nil } + ) + end + + it 'walks nested objects' do + current = { 'address' => { 'city' => 'Lyon', 'zip' => '69001' }, 'name' => 'Acme' } + previous = { 'address' => { 'city' => 'Paris' } } + changed = { 'address' => { 'city' => 'Lyon' } } + + expect(described_class.revert(current, previous, changed)).to eq( + { 'address' => { 'city' => 'Paris', 'zip' => '69001' }, 'name' => 'Acme' } + ) + end + + it 'restores an element of an array of objects, JSON string indexes included' do + current = [{ 'name' => 'a' }, { 'name' => 'c' }] + + expect(described_class.revert(current, { '1' => { 'name' => 'b' } }, { '1' => { 'name' => 'c' } })).to eq( + [{ 'name' => 'a' }, { 'name' => 'b' }] + ) + end + + it 'drops an element the change had appended' do + current = [{ 'name' => 'a' }, { 'name' => 'b' }] + + expect(described_class.revert(current, {}, { '1' => { 'name' => 'b' } })).to eq([{ 'name' => 'a' }]) + end + + it 'replaces the value as a whole when the change was not structural' do + expect(described_class.revert({ 'city' => 'Lyon' }, nil, { 'city' => 'Lyon' })).to be_nil + end + + # A round trip is the property that matters: diff then revert gives the original back. + it 'undoes any diff it is given' do + [ + [{ 'a' => 1, 'b' => { 'c' => 2, 'd' => nil } }, { 'a' => 1, 'b' => { 'c' => 3 } }], + [{ 'a' => nil }, { 'a' => 'set' }], + [{ 'list' => [{ 'x' => 1 }, { 'x' => 2 }] }, { 'list' => [{ 'x' => 1 }, { 'x' => 9 }] }], + ['plain', 'changed'] + ].each do |before, after| + delta = described_class.diff(before, after) + + expect(described_class.revert(after, delta[:previous], delta[:next])).to eq(before) + end + end + end + describe '.changed_values' do it 'only records writable columns present in the patch that actually changed' do before = { 'status' => 'open', 'name' => 'Acme', 'ignored' => 1 } diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/record_state_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/record_state_spec.rb new file mode 100644 index 000000000..e4633766f --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/record_state_spec.rb @@ -0,0 +1,100 @@ +require 'spec_helper' + +module ForestAdminAgent + module AuditTrail + describe RecordState do + def entry(operation, previous_values = {}, new_values = {}) + AuditRecord.new(operation: operation, collection: 'orders', record_id: '1', + previous_values: previous_values, new_values: new_values) + end + + it 'gives the record back untouched when nothing happened after the instant' do + current = { 'id' => 1, 'status' => 'shipped' } + + expect(described_class.at(current, [])).to eq(current) + end + + it 'undoes updates from newest to oldest' do + current = { 'id' => 1, 'status' => 'shipped' } + entries = [ + entry('update', { 'status' => 'paid' }, { 'status' => 'shipped' }), + entry('update', { 'status' => 'draft' }, { 'status' => 'paid' }) + ] + + expect(described_class.at(current, entries)).to eq({ 'id' => 1, 'status' => 'draft' }) + end + + it 'leaves columns the entries never touched alone' do + current = { 'id' => 1, 'status' => 'shipped', 'note' => 'keep me' } + entries = [entry('update', { 'status' => 'paid' }, { 'status' => 'shipped' })] + + expect(described_class.at(current, entries)['note']).to eq('keep me') + end + + it 'reports no record at all when it was created after the instant' do + current = { 'id' => 1, 'status' => 'draft' } + entries = [entry('create', {}, { 'status' => 'draft' })] + + expect(described_class.at(current, entries)).to be_nil + end + + # The record is gone now, so the walk starts from nothing and the delete brings the whole row back. + it 'restores a deleted record from the snapshot the delete recorded' do + entries = [entry('delete', { 'status' => 'shipped', 'note' => 'bye' }, {})] + + expect(described_class.at(nil, entries)).to eq({ 'status' => 'shipped', 'note' => 'bye' }) + end + + it 'keeps undoing older entries after restoring a delete' do + entries = [ + entry('delete', { 'status' => 'shipped' }, {}), + entry('update', { 'status' => 'paid' }, { 'status' => 'shipped' }) + ] + + expect(described_class.at(nil, entries)).to eq({ 'status' => 'paid' }) + end + + # A create can sit on top of an older life of the same id: nil for the create, then the older + # delete brings that life back. + it 'walks past a re-created id into its previous life' do + entries = [ + entry('create', {}, { 'status' => 'draft' }), + entry('delete', { 'status' => 'archived' }, {}) + ] + + expect(described_class.at({ 'status' => 'draft' }, entries)).to eq({ 'status' => 'archived' }) + end + + it 'ignores smart-action rows, which carry no field change' do + current = { 'status' => 'shipped' } + + expect(described_class.at(current, [entry('action', {}, { 'amount' => 30 })])).to eq(current) + end + + it 'undoes a nested change without disturbing the rest of the object' do + current = { 'address' => { 'city' => 'Lyon', 'zip' => '69001' } } + entries = [entry('update', { 'address' => { 'city' => 'Paris' } }, { 'address' => { 'city' => 'Lyon' } })] + + expect(described_class.at(current, entries)).to eq( + { 'address' => { 'city' => 'Paris', 'zip' => '69001' } } + ) + end + + # The whole point of the absent-vs-nil encoding: a key the change added has to disappear again, + # rather than come back holding nil. + it 'removes a nested key that the change had introduced' do + current = { 'address' => { 'city' => 'Lyon', 'zip' => '69001' } } + entries = [entry('update', { 'address' => {} }, { 'address' => { 'zip' => '69001' } })] + + expect(described_class.at(current, entries)).to eq({ 'address' => { 'city' => 'Lyon' } }) + end + + it 'puts back a nested key holding nil that the change had removed' do + current = { 'address' => { 'city' => 'Lyon' } } + entries = [entry('update', { 'address' => { 'zip' => nil } }, { 'address' => {} })] + + expect(described_class.at(current, entries)).to eq({ 'address' => { 'city' => 'Lyon', 'zip' => nil } }) + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb index 9f5de88b7..89e37d143 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb @@ -65,6 +65,44 @@ def raise_on_create_schema(message) expect { migrator.run }.to raise_error(ActiveRecord::StatementInvalid, /permission denied/) end + it 'treats the unique violation on pg_namespace as a lost race' do + allow(connection).to receive(:execute) do |sql| + raise ActiveRecord::RecordNotUnique, 'duplicate key value' if sql.include?('CREATE SCHEMA') + + executed << sql + end + + expect { migrator.run }.not_to raise_error + end + + # A message match alone would read "role ... already exists" as a lost race; the SQLSTATE does not. + context 'when the adapter exposes a SQLSTATE' do + def raise_with_sql_state(state, message) + stub_const('PG::Result', Class.new { const_set(:PG_DIAG_SQLSTATE, 67) }) unless defined?(PG::Result) + cause = double('PG::Error', result: double('result', error_field: state)) + error = ActiveRecord::StatementInvalid.new(message) + allow(error).to receive(:cause).and_return(cause) + + allow(connection).to receive(:execute) do |sql| + raise error if sql.include?('CREATE SCHEMA') + + executed << sql + end + end + + it 'accepts duplicate_schema' do + raise_with_sql_state('42P06', 'ERROR: schema "forest" already exists') + + expect { migrator.run }.not_to raise_error + end + + it 'reports anything else, however its message reads' do + raise_with_sql_state('42501', 'ERROR: permission denied, object already exists elsewhere') + + expect { migrator.run }.to raise_error(ActiveRecord::StatementInvalid) + end + end + it 'skips migrations already applied to this table' do migrator.run diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb index c9c9d153b..c9f32eea5 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb @@ -20,6 +20,10 @@ def record(over = {}) ) end + def matching_fields(fields) + store.list_by_record(collection: 'accounts', record_id: '1', fields: fields).map(&:new_values) + end + def connection Sql::AuditConnectionBase.connection end @@ -152,6 +156,66 @@ def connection .to eq(%w[main other]) end + it 'lists only entries whose diff touched one of the given fields' do + store.append(record(previous_values: { 'status' => 'open' }, new_values: { 'status' => 'closed' })) + store.append(record(previous_values: { 'note' => 'a' }, new_values: { 'note' => 'b' })) + # Added key: it exists on the new side only, so both sides have to be searched. + store.append(record(previous_values: {}, new_values: { 'tags' => ['x'] })) + + expect(matching_fields(%w[status])).to eq([{ 'status' => 'closed' }]) + expect(matching_fields(%w[tags])).to eq([{ 'tags' => ['x'] }]) + expect(matching_fields(%w[status note]).size).to eq(2) + expect(matching_fields(%w[missing])).to eq([]) + end + + it 'treats a field name holding a dot as a whole key, not a path' do + store.append(record(previous_values: { 'address.city' => 'Paris' }, + new_values: { 'address.city' => 'Lyon' })) + store.append(record(previous_values: { 'address' => { 'city' => 'Paris' } }, + new_values: { 'address' => { 'city' => 'Lyon' } })) + + expect(matching_fields(['address.city'])).to eq([{ 'address.city' => 'Lyon' }]) + expect(matching_fields(['address'])).to eq([{ 'address' => { 'city' => 'Lyon' } }]) + end + + it 'counts with the field filter applied' do + store.append(record(previous_values: { 'status' => 'open' }, new_values: { 'status' => 'closed' })) + store.append(record(previous_values: { 'note' => 'a' }, new_values: { 'note' => 'b' })) + + expect(store.count_by_record(collection: 'accounts', record_id: '1', fields: %w[status])).to eq(1) + end + + it 'refuses to filter by field on an adapter it has no JSON test for' do + store.append(record) + allow(store.send(:model).connection).to receive(:adapter_name).and_return('Informix') + + expect { store.list_by_record(collection: 'accounts', record_id: '1', fields: %w[status]) } + .to raise_error(ForestAdminDatasourceToolkit::Exceptions::ForestException, /not supported on informix/) + end + + describe '#list_since' do + it 'returns entries strictly newer than the instant, newest first' do + store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'older')) + store.append(record(timestamp: '2026-01-02T03:04:06.000Z', correlation_key: 'at')) + store.append(record(timestamp: '2026-01-02T03:04:07.000Z', correlation_key: 'newer')) + + history = store.list_since(collection: 'accounts', record_id: '1', + timestamp: '2026-01-02T03:04:06.000Z') + + expect(history.map(&:correlation_key)).to eq(['newer']) + end + + it 'breaks ties on equal timestamps by reverse insertion order' do + store.append(record(timestamp: '2026-01-02T03:04:07.000Z', correlation_key: 'first')) + store.append(record(timestamp: '2026-01-02T03:04:07.000Z', correlation_key: 'second')) + + history = store.list_since(collection: 'accounts', record_id: '1', + timestamp: '2026-01-02T03:04:06.000Z') + + expect(history.map(&:correlation_key)).to eq(%w[second first]) + end + end + it 'indexes record_id, correlation_key and user_id' do store.append(record) diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb index 194d7df67..cf6670916 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -35,6 +35,87 @@ def route_with_store(records: []) route end + describe 'state reconstruction' do + def state_route(entries: [], record: { 'id' => 4, 'status' => 'shipped' }) + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: store } }) + allow(store).to receive(:list_since).and_return(entries) + allow(collection).to receive(:list).and_return([{ 'id' => 4 }], [record].compact) + + route = described_class.new + context = double('context', collection: collection, caller: build_caller, permissions: permissions) + allow(route).to receive(:build).and_return(context) + route + end + + def entry(operation, previous_values = {}, new_values = {}) + ForestAdminAgent::AuditTrail::AuditRecord.new( + operation: operation, collection: 'projects', record_id: '4', + previous_values: previous_values, new_values: new_values + ) + end + + def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) + route.handle_state( + { headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'timestamp' => timestamp }.merge(extra) } + ) + end + + it 'registers the state route' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: Object.new } }) + + expect(described_class.new.routes).to include('forest_audit_trail_state') + end + + it 'returns the record with every later entry undone' do + route = state_route(entries: [entry('update', { 'status' => 'paid' }, { 'status' => 'shipped' })]) + + result = get_state(route) + + expect(result[:content][:data]).to eq({ 'id' => 4, 'status' => 'paid' }) + expect(result[:content][:meta]).to eq({ timestamp: '2026-01-02T10:00:00.000Z', reverted: 1 }) + end + + # Strictly after the requested instant: an entry stamped exactly at it belongs to that state. + it 'asks the store for entries strictly newer than the instant' do + route = state_route + get_state(route) + + expect(store).to have_received(:list_since).with( + collection: 'projects', record_id: '4', timestamp: '2026-01-02T10:00:00.000Z' + ) + end + + it 'returns no data when the record did not exist yet' do + route = state_route(entries: [entry('create', {}, { 'status' => 'draft' })]) + + expect(get_state(route)[:content][:data]).to be_nil + end + + it 'reads a wall-clock instant in the request timezone' do + route = state_route + get_state(route, timestamp: '2026-01-02T08:30', extra: { 'timezone' => 'America/New_York' }) + + expect(store).to have_received(:list_since).with(hash_including(timestamp: '2026-01-02T13:30:00.000Z')) + end + + it 'rejects a missing timestamp' do + route = state_route + + expect { get_state(route, timestamp: '') }.to raise_error( + Http::Exceptions::ValidationError, /Missing timestamp/ + ) + end + + it 'rejects an unparsable timestamp' do + route = state_route + + expect { get_state(route, timestamp: 'yesterday') }.to raise_error(Http::Exceptions::ValidationError) + end + end + it 'registers the record-history route when an audit_trail store is configured' do allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) .and_return({ audit_trail: { store: Object.new } }) @@ -128,6 +209,24 @@ def route_with_store(records: []) expect(store).to have_received(:list_by_record).with(hash_including(skip: 0, limit: 20)) end + it 'passes a fields filter through, keeping names that hold a dot' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'fields' => 'status, address.city ,' } }) + + expect(store).to have_received(:list_by_record).with(hash_including(fields: ['status', 'address.city'])) + expect(store).to have_received(:count_by_record).with(hash_including(fields: ['status', 'address.city'])) + end + + it 'sends no fields filter when the param is absent or empty' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'fields' => ' , ' } }) + + expect(store).to have_received(:list_by_record).with(hash_excluding(:fields)) + end + it 'parses userIds, dropping non-numeric tokens' do route = route_with_store route.handle_request({ headers: {}, diff --git a/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb b/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb index 5d82dfa42..e113e7fdc 100644 --- a/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb +++ b/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb @@ -40,8 +40,20 @@ class Engine < ::Rails::Engine # Echo the agent-generated correlation id on every response (mirrors the Node agent's # router.use(correlationIdMiddleware)); CORS exposure of the header is handled in load_cors. + # + # Ahead of ShowExceptions rather than at the end of the stack: a Rack middleware can only add a header + # to a response it sees returned, so sitting under the exception handlers meant error responses they + # build on the way out never carried the id. def load_correlation_id - config.middleware.use ForestAdminAgent::Http::CorrelationIdMiddleware + middleware = ForestAdminAgent::Http::CorrelationIdMiddleware + + begin + config.middleware.insert_before ActionDispatch::ShowExceptions, middleware + rescue StandardError + # No ShowExceptions in this stack (or already gone): the id still reaches every response the app + # returns normally. + config.middleware.use middleware + end end def load_configuration diff --git a/packages/forest_admin_rails/spec/lib/forest_admin_rails/engine_spec.rb b/packages/forest_admin_rails/spec/lib/forest_admin_rails/engine_spec.rb index 33b0906ce..ea06fd133 100644 --- a/packages/forest_admin_rails/spec/lib/forest_admin_rails/engine_spec.rb +++ b/packages/forest_admin_rails/spec/lib/forest_admin_rails/engine_spec.rb @@ -272,6 +272,39 @@ def self.setup!; end end end + RSpec.describe 'load_correlation_id' do + let(:engine_class) { ForestAdminRails::Engine } + let(:engine_instance) { engine_class.allocate } + # rubocop:disable RSpec/VerifiedDoubles + let(:config) { double('config') } + let(:middleware) { double('middleware') } + # rubocop:enable RSpec/VerifiedDoubles + + before do + allow(engine_instance).to receive(:config).and_return(config) + allow(config).to receive(:middleware).and_return(middleware) + end + + # Under the exception handlers, the 500 they build on the way out would never carry the header. + it 'inserts the middleware ahead of the exception handlers' do + allow(middleware).to receive(:insert_before) + + engine_instance.load_correlation_id + + expect(middleware).to have_received(:insert_before) + .with(ActionDispatch::ShowExceptions, ForestAdminAgent::Http::CorrelationIdMiddleware) + end + + it 'falls back to appending when the stack has no exception handler to insert before' do + allow(middleware).to receive(:insert_before).and_raise(RuntimeError, 'no such middleware') + allow(middleware).to receive(:use) + + engine_instance.load_correlation_id + + expect(middleware).to have_received(:use).with(ForestAdminAgent::Http::CorrelationIdMiddleware) + end + end + RSpec.describe 'Engine autoload behaviour' do it 'does not register an initializer that adds the host lib/ to autoload_paths' do initializer_names = ForestAdminRails::Engine.initializers.map(&:name) From de72e58e25d775e698439bb4ec2b0b85c66109d4 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Thu, 13 Aug 2026 10:06:52 +0200 Subject: [PATCH 18/35] test(audit_trail): pin the SQL of the adapters no local database exercises MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The specs run on SQLite, so the Postgres and MySQL field filters were never executed. FieldFilter only builds a condition string, so a fake connection is enough to pin all three — including that neither emits a `?` (a bind placeholder for ActiveRecord) nor `json_extract` (which cannot tell a key holding null from a missing one). Also covers the SQLSTATE read failing, and dispatches the state route through its registered closure so the wiring is exercised like the other three. Co-Authored-By: Claude Opus 5 (1M context) --- .../audit_trail/sql/field_filter_spec.rb | 79 +++++++++++++++++++ .../audit_trail/sql/migrator_spec.rb | 15 ++++ .../routes/resources/audit_trail_spec.rb | 3 +- 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/field_filter_spec.rb diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/field_filter_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/field_filter_spec.rb new file mode 100644 index 000000000..07f92b6d6 --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/field_filter_spec.rb @@ -0,0 +1,79 @@ +require 'spec_helper' + +module ForestAdminAgent + module AuditTrail + module Sql + # The store specs run on SQLite, so the SQL for the other adapters is pinned here — a fake connection + # is enough, the class only builds a condition string. + describe FieldFilter do + def filter_for(adapter) + # rubocop:disable RSpec/VerifiedDoubles + connection = double(adapter.to_s, adapter_name: adapter) + # rubocop:enable RSpec/VerifiedDoubles + allow(connection).to receive(:quote) { |value| "'#{value}'" } + + described_class.new(connection) + end + + it 'asks Postgres for the JSON keys of both sides' do + condition = filter_for('PostgreSQL').condition(%w[status note]) + + expect(condition).to eq( + 'EXISTS (SELECT 1 FROM jsonb_object_keys(previous_values::jsonb) AS key ' \ + "WHERE key IN ('status', 'note')) OR " \ + "EXISTS (SELECT 1 FROM jsonb_object_keys(new_values::jsonb) AS key WHERE key IN ('status', 'note'))" + ) + end + + # `?|` would read as a bind placeholder, and `json_extract` cannot tell a key holding null from a + # missing one. + it 'never emits a bind placeholder or json_extract' do + %w[PostgreSQL SQLite Mysql2].each do |adapter| + condition = filter_for(adapter).condition(%w[status]) + + expect(condition).not_to include('?') + expect(condition).not_to include('json_extract') + end + end + + it 'asks SQLite for the type at a quoted path on both sides' do + condition = filter_for('SQLite').condition(%w[status]) + + expect(condition).to eq( + %(json_type(previous_values, '$."status"') IS NOT NULL OR ) + + %(json_type(new_values, '$."status"') IS NOT NULL) + ) + end + + it 'asks MySQL for any of the paths on both sides' do + condition = filter_for('Mysql2').condition(%w[status note]) + + expect(condition).to eq( + %(JSON_CONTAINS_PATH(previous_values, 'one', '$."status"', '$."note"') OR ) + + %(JSON_CONTAINS_PATH(new_values, 'one', '$."status"', '$."note"')) + ) + end + + it 'treats MariaDB like MySQL' do + expect(filter_for('Mariadb').condition(%w[status])).to include('JSON_CONTAINS_PATH') + end + + # A dot in a field name is part of the name, not a traversal into the object. + it 'quotes a field name holding a dot as a single key' do + expect(filter_for('SQLite').condition(['address.city'])).to include(%('$."address.city"')) + expect(filter_for('Mysql2').condition(['address.city'])).to include(%('$."address.city"')) + end + + it 'escapes a quote inside a field name' do + expect(filter_for('SQLite').condition(['we"ird'])).to include(%q($."we\"ird")) + end + + it 'refuses an adapter it has no JSON test for' do + expect { filter_for('Informix').condition(%w[status]) }.to raise_error( + ForestAdminDatasourceToolkit::Exceptions::ForestException, /not supported on informix/ + ) + end + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb index 89e37d143..1632c9932 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb @@ -96,6 +96,21 @@ def raise_with_sql_state(state, message) expect { migrator.run }.not_to raise_error end + it 'falls back to the message when reading the SQLSTATE itself blows up' do + stub_const('PG::Result', Class.new { const_set(:PG_DIAG_SQLSTATE, 67) }) unless defined?(PG::Result) + cause = double('PG::Error') + allow(cause).to receive(:result).and_raise(StandardError, 'connection already closed') + error = ActiveRecord::StatementInvalid.new('ERROR: schema "forest" already exists') + allow(error).to receive(:cause).and_return(cause) + allow(connection).to receive(:execute) do |sql| + raise error if sql.include?('CREATE SCHEMA') + + executed << sql + end + + expect { migrator.run }.not_to raise_error + end + it 'reports anything else, however its message reads' do raise_with_sql_state('42501', 'ERROR: permission denied, object already exists elsewhere') diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb index cf6670916..482ba4e2d 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -55,8 +55,9 @@ def entry(operation, previous_values = {}, new_values = {}) ) end + # Through the registered closure rather than the handler, so the wiring is covered too. def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) - route.handle_state( + route.routes['forest_audit_trail_state'][:closure].call( { headers: {}, params: { 'collection_name' => 'projects', 'id' => '4', 'timestamp' => timestamp }.merge(extra) } ) From c535e153af38bc333f7798cdd716309a9cf12412 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Thu, 13 Aug 2026 10:14:40 +0200 Subject: [PATCH 19/35] fix(audit_trail): drop an appended array element on revert, honour the timezone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings, both hidden by specs that asserted an encoding by hand instead of the one the diff actually emits. - `diff_object_arrays` recorded `previous[index] = nil` for an index only the new side reaches, so reverting an append left a nil hole instead of shortening the array. Indexes now follow the same rule as hash keys — left out of the side that does not reach them — and the round-trip spec carries length changes so an append and a drop are both covered. - `parse_state_timestamp` handed `2026-01-02T08:30:15` to Time.iso8601, which parses it happily in the server's timezone and ignores the request's. Wall-clock values (with or without seconds, and bare days) go through the request timezone; Time.iso8601 is left for values carrying an offset or Z. Co-Authored-By: Claude Opus 5 (1M context) --- .../forest_admin_agent/audit_trail/diff.rb | 29 +++++++++++------- .../routes/resources/audit_trail.rb | 7 +++++ .../audit_trail/diff_spec.rb | 30 +++++++++++++++++-- .../routes/resources/audit_trail_spec.rb | 22 ++++++++++++++ 4 files changed, 76 insertions(+), 12 deletions(-) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb index cf92fdfb2..7ec8fd0c2 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/diff.rb @@ -86,8 +86,10 @@ def diff_object_arrays(before, after) sub = diff(before[index], after[index]) next unless sub - previous[index] = sub[:previous] - next_values[index] = sub[:next] + # Same rule as for hash keys: an index one side does not reach is left out of that side, so a + # revert can tell an appended element (drop it) from one whose value became nil (keep it). + previous[index] = sub[:previous] if index < before.length + next_values[index] = sub[:next] if index < after.length end { previous: previous, next: next_values } @@ -125,23 +127,30 @@ def revert_hash(current, previous, changed) end # Arrays of objects are diffed index by index, and JSON turns those indexes into strings on the way - # back out. An element the change appended is dropped, which can only ever shorten the tail. + # back out. Highest index first, so dropping an element the change appended leaves the lower ones + # where the diff expects them. def revert_array(current, previous, changed) - result = current.dup + indexes = (previous.keys | changed.keys).map(&:to_i).sort.reverse - (previous.keys | changed.keys).map(&:to_i).sort.reverse_each do |index| - key = index.to_s - if previous.key?(key) || previous.key?(index) - result[index] = revert(current[index], previous[key] || previous[index], changed[key] || changed[index]) + indexes.each_with_object(current.dup) do |index, result| + if index?(previous, index) + result[index] = revert(current[index], at(previous, index), at(changed, index)) else result.delete_at(index) end end + end + + def index?(hash, index) + hash.key?(index) || hash.key?(index.to_s) + end - result + def at(hash, index) + hash.key?(index) ? hash[index] : hash[index.to_s] end - private_class_method :diff_hashes, :diff_at, :diff_object_arrays, :partial?, :revert_hash, :revert_array + private_class_method :diff_hashes, :diff_at, :diff_object_arrays, :partial?, :revert_hash, + :revert_array, :index?, :at end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb index ca0f1055e..696c2dc6a 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb @@ -87,6 +87,9 @@ def handle_state(args = {}) def parse_state_timestamp(args) raw = args.dig(:params, 'timestamp').to_s raise Http::Exceptions::ValidationError, 'Missing timestamp' if raw.empty? + # A wall-clock value carries no offset, so it belongs to the request timezone. Handing it to + # Time.iso8601 would read it in the server's instead — silently, since it parses just fine. + return parse_date_boundary(raw, request_timezone(args), :start) if wall_clock?(raw) begin Time.iso8601(raw).utc.iso8601(3) @@ -95,6 +98,10 @@ def parse_state_timestamp(args) end end + def wall_clock?(raw) + DATE_ONLY.match?(raw) || DATE_TIME.match?(raw) + end + # JSON:API `sort`: `timestamp` → oldest first, anything else (absent/unsupported) → newest first. def parse_sort(args) args.dig(:params, 'sort').to_s == 'timestamp' ? 'asc' : 'desc' diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb index cc3f0f0f1..caad7989d 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/diff_spec.rb @@ -34,6 +34,20 @@ module AuditTrail ) end + it 'leaves an appended index out of the previous side' do + expect(described_class.diff([{ 'x' => 1 }], [{ 'x' => 1 }, { 'y' => 2 }])).to eq( + previous: {}, + next: { 1 => { 'y' => 2 } } + ) + end + + it 'leaves a dropped index out of the new side' do + expect(described_class.diff([{ 'x' => 1 }, { 'y' => 2 }], [{ 'x' => 1 }])).to eq( + previous: { 1 => { 'y' => 2 } }, + next: {} + ) + end + it 'diffs an array of objects index by index' do before = [{ 'name' => 'a' }, { 'name' => 'b' }] after = [{ 'name' => 'a' }, { 'name' => 'c' }] @@ -97,22 +111,34 @@ module AuditTrail ) end - it 'drops an element the change had appended' do + it 'drops an element the change had appended, rather than leaving a nil behind' do current = [{ 'name' => 'a' }, { 'name' => 'b' }] expect(described_class.revert(current, {}, { '1' => { 'name' => 'b' } })).to eq([{ 'name' => 'a' }]) end + it 'brings back an element the change had dropped' do + expect(described_class.revert([{ 'name' => 'a' }], { '1' => { 'name' => 'b' } }, {})).to eq( + [{ 'name' => 'a' }, { 'name' => 'b' }] + ) + end + it 'replaces the value as a whole when the change was not structural' do expect(described_class.revert({ 'city' => 'Lyon' }, nil, { 'city' => 'Lyon' })).to be_nil end - # A round trip is the property that matters: diff then revert gives the original back. + # A round trip is the property that matters: diff then revert gives the original back. The array + # cases carry a length change on purpose — an appended element has to disappear again rather than + # revert to nil, and a dropped one has to come back. it 'undoes any diff it is given' do [ [{ 'a' => 1, 'b' => { 'c' => 2, 'd' => nil } }, { 'a' => 1, 'b' => { 'c' => 3 } }], [{ 'a' => nil }, { 'a' => 'set' }], [{ 'list' => [{ 'x' => 1 }, { 'x' => 2 }] }, { 'list' => [{ 'x' => 1 }, { 'x' => 9 }] }], + [[{ 'x' => 1 }], [{ 'x' => 1 }, { 'y' => 2 }]], + [[{ 'x' => 1 }, { 'y' => 2 }], [{ 'x' => 1 }]], + [[{ 'x' => 1 }, { 'y' => 2 }, { 'z' => 3 }], [{ 'x' => 9 }]], + [{ 'list' => [{ 'x' => 1 }] }, { 'list' => [{ 'x' => 1 }, { 'y' => 2 }] }], ['plain', 'changed'] ].each do |before, after| delta = described_class.diff(before, after) diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb index 482ba4e2d..bf6e42918 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -102,6 +102,28 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) expect(store).to have_received(:list_since).with(hash_including(timestamp: '2026-01-02T13:30:00.000Z')) end + # Seconds make it parse as ISO-8601, which would silently read it in the server's timezone. + it 'reads a wall-clock instant carrying seconds in the request timezone too' do + route = state_route + get_state(route, timestamp: '2026-01-02T08:30:15', extra: { 'timezone' => 'America/New_York' }) + + expect(store).to have_received(:list_since).with(hash_including(timestamp: '2026-01-02T13:30:15.000Z')) + end + + it 'reads a bare day in the request timezone' do + route = state_route + get_state(route, timestamp: '2026-01-02', extra: { 'timezone' => 'America/New_York' }) + + expect(store).to have_received(:list_since).with(hash_including(timestamp: '2026-01-02T05:00:00.000Z')) + end + + it 'honours an explicit offset instead of the request timezone' do + route = state_route + get_state(route, timestamp: '2026-01-02T08:30:15+02:00', extra: { 'timezone' => 'America/New_York' }) + + expect(store).to have_received(:list_since).with(hash_including(timestamp: '2026-01-02T06:30:15.000Z')) + end + it 'rejects a missing timestamp' do route = state_route From 3a5d3cba131dafb95e81116e454134fb086b7a86 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Thu, 13 Aug 2026 10:29:47 +0200 Subject: [PATCH 20/35] fix(audit_trail): read the record for /state through the caller's scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `handle_state` authorized the record with a scoped query and then read its columns with an unscoped one, so the row it returned was not the row the check covered. Authorization and read are now the same query: `scoped_record` returns the record matched against the caller's scope (nil when it is simply gone, 404 when it exists outside that scope), and `assert_record_in_scope` is that same call with its result discarded. Nothing reads the row around the scope any more, and /state does one query instead of two. The spec that was meant to cover this passed for the wrong reason — it consumed the primary-key stub as the record, so the full read was never exercised. It now asserts the scope is in the filter, that every column is projected and that a single query happens, plus the two branches around it: a record living outside the scope, and a deleted one rebuilt from its history. Co-Authored-By: Claude Opus 5 (1M context) --- .../routes/resources/audit_trail.rb | 6 ++- .../routes/resources/audit_trail_route.rb | 48 ++++++++++--------- .../routes/resources/audit_trail_spec.rb | 41 +++++++++++++++- 3 files changed, 69 insertions(+), 26 deletions(-) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb index 696c2dc6a..95ddaa181 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb @@ -64,14 +64,16 @@ def handle_request(args = {}) def handle_state(args = {}) context = build(args) context.permissions.can?(:read, context.collection) - assert_record_in_scope(context, context.collection, args[:params]['id']) + # Authorizes and reads in one query: the record it hands back is the one the scope covered. + current = scoped_record( + context, context.collection, args[:params]['id'], column_projection(context.collection) + ) timestamp = parse_state_timestamp(args) entries = store.list_since( collection: context.collection.name, record_id: args[:params]['id'], timestamp: timestamp ) # Fully qualified: inside this class, `AuditTrail` is the route itself. - current = current_record(context, context.collection, args[:params]['id']) state = ::ForestAdminAgent::AuditTrail::RecordState.at(current, entries) { diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb index 3e37f9278..516a53e5b 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb @@ -10,18 +10,29 @@ module AuditTrailRoute include ForestAdminDatasourceToolkit::Components::Query def assert_record_in_scope(context, collection, packed_id) + scoped_record(context, collection, packed_id) + + nil + end + + # The record as it stands, read through the caller's permission scope. nil when it no longer exists + # — a deleted record keeps its history readable, which is much of the point of an audit trail — and + # a 404 when it does exist outside that scope. + # + # Authorizing and reading are the same query on purpose: a scoped check followed by an unscoped read + # would hand back a row the check never covered, the moment the two drifted apart. + def scoped_record(context, collection, packed_id, projection = nil) condition = ConditionTree::ConditionTreeFactory.match_records( collection, [Utils::Id.unpack_id(collection, packed_id, with_key: true)] ) scope = context.permissions.get_scope(collection) + in_scope = ConditionTree::ConditionTreeFactory.intersect([condition, scope]) + record = first_record(context, collection, in_scope, projection || key_projection(collection)) - return if any_record?(context, collection, ConditionTree::ConditionTreeFactory.intersect([condition, scope])) - - # Nothing in scope means the record is either outside it or gone for good. A deleted record - # keeps its history readable — inspecting what was deleted is much of the point of an audit - # trail — so only a record that still exists outside the scope is refused. Without a scope - # the first query already answered the question. - return if scope.nil? || !any_record?(context, collection, condition) + return record if record + # Nothing in scope: either gone for good, or someone else's record. Without a scope the query + # above already answered the question. + return nil if scope.nil? || first_record(context, collection, condition, key_projection(collection)).nil? raise Http::Exceptions::NotFoundError, 'Record does not exists' end @@ -35,24 +46,17 @@ def store ::ForestAdminAgent::AuditTrail.store end - # The audited columns as they stand now — the starting point of a state reconstruction. No scope in - # the filter: the caller was already authorized against it, and a record that is simply gone has to - # come back as nil so the history can restore it. - def current_record(context, collection, packed_id) - columns = collection.schema[:fields].select { |_name, field| field.type == 'Column' }.keys - condition = ConditionTree::ConditionTreeFactory.match_records( - collection, [Utils::Id.unpack_id(collection, packed_id, with_key: true)] - ) + # Every column, the starting point of a state reconstruction. + def column_projection(collection) + Projection.new(collection.schema[:fields].select { |_name, field| field.type == 'Column' }.keys) + end - collection.list(context.caller, Filter.new(condition_tree: condition), Projection.new(columns)).first + def key_projection(collection) + Projection.new(ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(collection)) end - def any_record?(context, collection, condition_tree) - collection.list( - context.caller, - Filter.new(condition_tree: condition_tree), - Projection.new(ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(collection)) - ).any? + def first_record(context, collection, condition_tree, projection) + collection.list(context.caller, Filter.new(condition_tree: condition_tree), projection).first end end end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb index bf6e42918..91f49dab0 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -17,7 +17,8 @@ module Resources 'id' => ColumnSchema.new( column_type: 'Number', is_primary_key: true, filter_operators: [Operators::IN, Operators::EQUAL] - ) + ), + 'status' => ColumnSchema.new(column_type: 'String') } }, list: [{ 'id' => 4 }] @@ -40,7 +41,7 @@ def state_route(entries: [], record: { 'id' => 4, 'status' => 'shipped' }) allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) .and_return({ audit_trail: { store: store } }) allow(store).to receive(:list_since).and_return(entries) - allow(collection).to receive(:list).and_return([{ 'id' => 4 }], [record].compact) + allow(collection).to receive(:list).and_return([record].compact) route = described_class.new context = double('context', collection: collection, caller: build_caller, permissions: permissions) @@ -79,6 +80,42 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) expect(result[:content][:meta]).to eq({ timestamp: '2026-01-02T10:00:00.000Z', reverted: 1 }) end + # Authorization and read are one query: a scoped check followed by an unscoped read would hand + # back a row the check never covered. + it 'reads the record through the caller scope, in a single query' do + scope = Nodes::ConditionTreeLeaf.new('id', Operators::EQUAL, 4) + allow(permissions).to receive(:get_scope).and_return(scope) + route = state_route + + get_state(route) + + expect(collection).to have_received(:list).once + expect(collection).to have_received(:list) do |_caller, filter, projection| + expect(filter.condition_tree.conditions).to include(scope) + expect(projection).to include('status') + end + end + + it 'refuses a record that exists outside the caller scope' do + allow(permissions).to receive(:get_scope).and_return(Nodes::ConditionTreeLeaf.new('id', + Operators::EQUAL, 9)) + route = state_route(record: nil) + # Nothing in scope, but the record does exist without it: someone else's. + allow(collection).to receive(:list).and_return([], [{ 'id' => 4 }]) + + expect { get_state(route) }.to raise_error(Http::Exceptions::NotFoundError) + expect(store).not_to have_received(:list_since) + end + + it 'rebuilds a deleted record from its history' do + allow(permissions).to receive(:get_scope).and_return(Nodes::ConditionTreeLeaf.new('id', + Operators::EQUAL, 4)) + route = state_route(entries: [entry('delete', { 'status' => 'shipped' }, {})], record: nil) + allow(collection).to receive(:list).and_return([], []) + + expect(get_state(route)[:content][:data]).to eq({ 'status' => 'shipped' }) + end + # Strictly after the requested instant: an entry stamped exactly at it belongs to that state. it 'asks the store for entries strictly newer than the instant' do route = state_route From d044bfc16e12af423720287a708773003c3072ca Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Thu, 13 Aug 2026 10:31:38 +0200 Subject: [PATCH 21/35] refactor(audit_trail): return only data from /state, matching the Node agent Node's handleStateAt answers with `{ data }` and nothing else, so the timestamp and reverted-entry count go: they were ours to invent and nothing reads them. `timestamp` and `entries` are still needed to query and rebuild, only the meta hash is gone. The spec now pins the whole payload rather than asserting on the two keys, so a stray meta key fails instead of passing unnoticed, and AUDIT_TRAIL.md drops the note about the shape being unconfirmed. Co-Authored-By: Claude Opus 5 (1M context) --- packages/forest_admin_agent/AUDIT_TRAIL.md | 4 +--- .../forest_admin_agent/routes/resources/audit_trail.rb | 8 +++----- .../routes/resources/audit_trail_spec.rb | 3 +-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index f75fb46f4..e3fa1c7e4 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -119,7 +119,7 @@ that instant, rebuilt by taking the record as it stands now and undoing every en after** the timestamp — an entry stamped exactly at it counts as part of that state: ```json -{ "data": { "status": "paid", "address": { "city": "Paris" } }, "meta": { "timestamp": "…", "reverted": 3 } } +{ "data": { "status": "paid", "address": { "city": "Paris" } } } ``` `timestamp` accepts an ISO-8601 instant, or the same wall-clock forms as the filters above read in the @@ -130,8 +130,6 @@ Walking back stops being able to help where the trail stops: only audited (writa reconstructed, and a `create` means the record did not exist before it, while a `delete` restores the whole row it recorded and the walk carries on into any earlier life of the same id. -> The exact response shape is not yet pinned against a consumer — no frontend reads this endpoint today. - ### Correlation route `GET /forest/_audit-trail/correlation/{correlationKey}` returns `{ "data": [...] }` — the diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb index 95ddaa181..36a47b36a 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb @@ -60,7 +60,8 @@ def handle_request(args = {}) end # Record as it stood at `timestamp`: the current record with every later entry undone. `data` is - # null when the record did not exist yet (or not any more) at that instant. + # null when the record did not exist yet (or not any more) at that instant. Shape matches the Node + # agent's handleStateAt — `data` and nothing else. def handle_state(args = {}) context = build(args) context.permissions.can?(:read, context.collection) @@ -76,10 +77,7 @@ def handle_state(args = {}) # Fully qualified: inside this class, `AuditTrail` is the route itself. state = ::ForestAdminAgent::AuditTrail::RecordState.at(current, entries) - { - name: args[:params]['collection_name'], - content: { data: state, meta: { timestamp: timestamp, reverted: entries.size } } - } + { name: args[:params]['collection_name'], content: { data: state } } end private diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb index 91f49dab0..2865a6e48 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -76,8 +76,7 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) result = get_state(route) - expect(result[:content][:data]).to eq({ 'id' => 4, 'status' => 'paid' }) - expect(result[:content][:meta]).to eq({ timestamp: '2026-01-02T10:00:00.000Z', reverted: 1 }) + expect(result[:content]).to eq({ data: { 'id' => 4, 'status' => 'paid' } }) end # Authorization and read are one query: a scoped check followed by an unscoped read would hand From 1a2a56ed2d20ab6b5904272a3e56e37de47c22fc Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Thu, 13 Aug 2026 10:46:27 +0200 Subject: [PATCH 22/35] fix(audit_trail): insert the correlation middleware into the application stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An insert_before recorded on the engine's own middleware proxy does not see ShowExceptions and raises when the stack is applied — too late for the rescue here, which only guards the recording. It goes to Rails.application's stack instead, where the exception handlers actually live. The spec followed the engine's proxy, so it was updated to the application's, and gained the case where there is no application to reach at all (the rescue covers it by appending to the engine stack, as before). Co-Authored-By: Claude Opus 5 (1M context) --- .../lib/forest_admin_rails/engine.rb | 4 ++- .../lib/forest_admin_rails/engine_spec.rb | 34 +++++++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb b/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb index e113e7fdc..463c50baf 100644 --- a/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb +++ b/packages/forest_admin_rails/lib/forest_admin_rails/engine.rb @@ -48,7 +48,9 @@ def load_correlation_id middleware = ForestAdminAgent::Http::CorrelationIdMiddleware begin - config.middleware.insert_before ActionDispatch::ShowExceptions, middleware + # The application's stack, not the engine's: ShowExceptions lives there, and an + # insert_before recorded on the engine's own (empty) proxy raises when it is applied. + Rails.application.config.middleware.insert_before ActionDispatch::ShowExceptions, middleware rescue StandardError # No ShowExceptions in this stack (or already gone): the id still reaches every response the app # returns normally. diff --git a/packages/forest_admin_rails/spec/lib/forest_admin_rails/engine_spec.rb b/packages/forest_admin_rails/spec/lib/forest_admin_rails/engine_spec.rb index ea06fd133..a048fb76e 100644 --- a/packages/forest_admin_rails/spec/lib/forest_admin_rails/engine_spec.rb +++ b/packages/forest_admin_rails/spec/lib/forest_admin_rails/engine_spec.rb @@ -276,32 +276,44 @@ def self.setup!; end let(:engine_class) { ForestAdminRails::Engine } let(:engine_instance) { engine_class.allocate } # rubocop:disable RSpec/VerifiedDoubles - let(:config) { double('config') } - let(:middleware) { double('middleware') } + let(:engine_middleware) { double('engine middleware') } + let(:app_middleware) { double('application middleware') } + let(:application) { double('application', config: double('app config', middleware: app_middleware)) } + let(:engine_config) { double('engine config', middleware: engine_middleware) } # rubocop:enable RSpec/VerifiedDoubles before do - allow(engine_instance).to receive(:config).and_return(config) - allow(config).to receive(:middleware).and_return(middleware) + allow(engine_instance).to receive(:config).and_return(engine_config) + allow(Rails).to receive(:application).and_return(application) end - # Under the exception handlers, the 500 they build on the way out would never carry the header. - it 'inserts the middleware ahead of the exception handlers' do - allow(middleware).to receive(:insert_before) + # The application's stack, not the engine's: ShowExceptions lives there, and under the exception + # handlers the 500s they build would never carry the header. + it 'inserts the middleware ahead of the exception handlers of the application stack' do + allow(app_middleware).to receive(:insert_before) engine_instance.load_correlation_id - expect(middleware).to have_received(:insert_before) + expect(app_middleware).to have_received(:insert_before) .with(ActionDispatch::ShowExceptions, ForestAdminAgent::Http::CorrelationIdMiddleware) end it 'falls back to appending when the stack has no exception handler to insert before' do - allow(middleware).to receive(:insert_before).and_raise(RuntimeError, 'no such middleware') - allow(middleware).to receive(:use) + allow(app_middleware).to receive(:insert_before).and_raise(RuntimeError, 'no such middleware') + allow(engine_middleware).to receive(:use) + + engine_instance.load_correlation_id + + expect(engine_middleware).to have_received(:use).with(ForestAdminAgent::Http::CorrelationIdMiddleware) + end + + it 'falls back to appending when there is no application to reach' do + allow(Rails).to receive(:application).and_return(nil) + allow(engine_middleware).to receive(:use) engine_instance.load_correlation_id - expect(middleware).to have_received(:use).with(ForestAdminAgent::Http::CorrelationIdMiddleware) + expect(engine_middleware).to have_received(:use).with(ForestAdminAgent::Http::CorrelationIdMiddleware) end end From 52a5c920ffea087b02e2421e8c7e54e33f6b3eca Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Thu, 13 Aug 2026 11:04:22 +0200 Subject: [PATCH 23/35] fix(audit_trail): count an Error result as a failed action run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An action reporting failure through `result_builder.error` never raises, so the run was recorded as `action`. It answers with `{ type: 'Error', ... }`, which is as failed as a raised one — same rule as the Node agent's `result.type === 'Error'` — and the result is still returned to the caller untouched. The result-type read is guarded, since it happens before ActionResult.parse and an action need not answer with a hash at all. Co-Authored-By: Claude Opus 5 (1M context) --- packages/forest_admin_agent/AUDIT_TRAIL.md | 2 +- .../routes/action/actions.rb | 10 +++++++-- .../routes/action/actions_spec.rb | 22 ++++++++++++++++--- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index e3fa1c7e4..d7b253d68 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -166,7 +166,7 @@ Running a smart action writes one row per selected record, in the same table: | column | value | | ----------- | -------------------------------------------------------------------------- | -| `operation` | `action` when it ran, `action_failed` when it raised | +| `operation` | `action` when it went through, `action_failed` when it raised or answered with an `Error` result | | `newValues` | the submitted form values (redacted with the same `redact` config) | | `recordId` | each selected record — empty for a global action or a select-all selection | diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb index fdb6e1bc2..75dd437ec 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb @@ -117,10 +117,12 @@ def handle_hook_request(args = {}) private - # A failed run is worth recording too: "who tried to run this" is usually the interesting part. + # A failed run is worth recording too: "who tried to run this" is usually the interesting part. An + # action that answers with an Error result failed just as much as one that raised — it simply chose + # to say so through `result_builder.error` instead. def execute_and_audit(context, args, data, filter) result = context.collection.execute(context.caller, @action_name, data, filter) - audit_action(context, args, data) + audit_action(context, args, data, failed: error_result?(result)) result rescue StandardError @@ -128,6 +130,10 @@ def execute_and_audit(context, args, data, filter) raise end + def error_result?(result) + result.is_a?(Hash) && result[:type] == 'Error' + end + def audit_action(context, args, data, failed: false) capture = ForestAdminAgent::AuditTrail::ActionCapture.new( ForestAdminAgent::AuditTrail.store, ForestAdminAgent::AuditTrail.options[:redact] diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb index 93102e137..fce92144b 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb @@ -50,13 +50,13 @@ def audited_ids(attributes) .and_return({ audit_trail: { store: store, redact: { 'orders' => ['reason'] } } }) end - def execute(result: 'ok', &raising) + def execute(result: { type: 'Success', message: 'done' }, &raising) allow(collection).to receive(:execute, &(raising || proc { result })) route.send(:execute_and_audit, context, args, { 'amount' => 30, 'reason' => 'damaged' }, nil) end it 'records one row per selected record and returns the action result' do - expect(execute).to eq('ok') + expect(execute).to eq({ type: 'Success', message: 'done' }) expect(store).to have_received(:append).twice expect(store).to have_received(:append).with( @@ -75,9 +75,25 @@ def execute(result: 'ok', &raising) it 'records nothing when no audit database is configured' do allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache).and_return({}) - expect(execute).to eq('ok') + expect(execute).to eq({ type: 'Success', message: 'done' }) expect(store).not_to have_received(:append) end + + # An action reporting failure through result_builder.error never raises, so only the result says so. + it 'records an Error result as a failed run, and still returns it' do + result = execute(result: { type: 'Error', message: 'not allowed' }) + + expect(result).to eq({ type: 'Error', message: 'not allowed' }) + expect(store).to have_received(:append) + .with(having_attributes(operation: 'action_failed', record_id: '4')).once + end + + it 'records any other result type as a run that went through' do + execute(result: { type: 'Webhook', url: 'https://example.test' }) + + expect(store).to have_received(:append) + .with(having_attributes(operation: 'action', record_id: '4')).once + end end context 'with a global action' do From 110d2a240310aa6a98fb72036355dbc255b0fe50 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Fri, 14 Aug 2026 11:13:20 +0200 Subject: [PATCH 24/35] feat(audit_trail): record what an action answered, next to what was submitted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An action row now uses both value columns for what they mean on that kind of row: `previous_values` is the submitted form, `new_values` what the action answered. No new column, no migration. The answer is an allowlist — type, message, name, mime_type, method, url, path — because a result also carries the file's contents, a webhook's body and headers and arbitrary response headers. File bytes have no business in an audit table, the other two routinely hold credentials, and an allowlist means a field added to a result later is not stored until someone decides it should be. `html` is left out as operator-facing markup the message already summarises. An action that raised stores no answer at all. This makes the state walk-back's handling of action rows load-bearing: their value columns are not a record's before and after, so applying either would corrupt a rebuild. The comment says so and the spec now feeds it a realistic action row. Also folds the row mapping onto the column list — an AuditRecord is a Struct and a row answers to `[]`, so the two 10-line field-by-field hashes were one list — and moves the migration list beside the runner that applies it. Co-Authored-By: Claude Opus 5 (1M context) --- .rubocop.yml | 1 + packages/forest_admin_agent/AUDIT_TRAIL.md | 20 ++++++-- .../audit_trail/action_capture.rb | 20 ++++++-- .../audit_trail/record_state.rb | 3 +- .../audit_trail/sql/migrations.rb | 39 +++++++++++++++ .../audit_trail/sql/migrator.rb | 34 ++----------- .../forest_admin_agent/audit_trail/store.rb | 30 ++++-------- .../routes/action/actions.rb | 5 +- .../audit_trail/action_capture_spec.rb | 49 +++++++++++++++++-- .../audit_trail/record_state_spec.rb | 10 +++- .../audit_trail/store_spec.rb | 9 ++-- .../routes/action/actions_spec.rb | 16 +++++- 12 files changed, 165 insertions(+), 71 deletions(-) create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb diff --git a/.rubocop.yml b/.rubocop.yml index e0e1dfb6d..24c0e14e2 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -257,6 +257,7 @@ Naming/PredicatePrefix: Metrics/ParameterLists: Exclude: + - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/configuration.rb' diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index d7b253d68..5ad2d4d1a 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -164,11 +164,21 @@ the store. A missing `collection` or `recordId` returns **HTTP 400** (`Validatio Running a smart action writes one row per selected record, in the same table: -| column | value | -| ----------- | -------------------------------------------------------------------------- | -| `operation` | `action` when it went through, `action_failed` when it raised or answered with an `Error` result | -| `newValues` | the submitted form values (redacted with the same `redact` config) | -| `recordId` | each selected record — empty for a global action or a select-all selection | +| column | value | +| ---------------- | -------------------------------------------------------------------------- | +| `operation` | `action` when it went through, `action_failed` when it raised or answered with an `Error` result | +| `previousValues` | the submitted form values (redacted with the same `redact` config) | +| `newValues` | what the action answered — an allowlist of the result: `type`, `message`, `name`, `mimeType`, `method`, `url`, `path`. Empty when it raised | +| `recordId` | each selected record — empty for a global action or a select-all selection | + +The two value columns carry what went in and what came back, rather than a record's before and after. A +result also holds the file's contents, a webhook's body and headers, and arbitrary response headers: file +bytes have no business in an audit table and the other two routinely hold credentials, so the stored answer +is an allowlist — a field added to a result later is not recorded until someone decides it should be. `html` +is left out too, being operator-facing markup the message already summarises. + +Because these are the same columns the field filter searches, filtering by a field named like a result key +(`message`, `type`) also matches action rows. **Which** action ran is not stored: the Forest activity logs already record it, and `correlationKey` is the join between the two. diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb index f05d36210..e594e6a4b 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb @@ -12,6 +12,12 @@ class ActionCapture EXECUTED = 'action'.freeze FAILED = 'action_failed'.freeze + # What of the action's answer is worth keeping — an allowlist, not a denylist: a result also carries + # the file's contents, a webhook's body and headers and arbitrary response headers. File bytes have no + # business in an audit table and the other two routinely hold credentials, and an allowlist means a + # field added to a result later is not stored until someone decides it should be. `html` is left out + # too: operator-facing markup, sometimes large, and the message already says what happened. + RESULT_FIELDS = %i[type message name mime_type method url path].freeze # Bulk runs over a select-all selection, and global actions, target no id we can name without # querying the whole selection: they get one row attached to no record rather than none at all. NO_RECORD = ''.freeze @@ -22,7 +28,7 @@ def initialize(store, redact = {}) end # No-op unless an audit database was configured, and best-effort: the action has already run. - def record(caller:, collection:, form_values:, record_ids:, failed: false) + def record(caller:, collection:, form_values:, record_ids:, result: nil, failed: false) return unless @store audit_safely do @@ -40,13 +46,21 @@ def record(caller:, collection:, form_values:, record_ids:, failed: false) record_id: record_id, user_id: caller&.id, correlation_key: correlation_key, - previous_values: {}, - new_values: values + previous_values: values, + new_values: summarize(result) || {} ) ) end end end + + private + + def summarize(result) + return nil unless result.is_a?(Hash) + + result.slice(*RESULT_FIELDS).compact.transform_keys(&:to_s) + end end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/record_state.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/record_state.rb index accb43386..6af292c2d 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/record_state.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/record_state.rb @@ -27,7 +27,8 @@ def undo(state, entry) when 'update' revert_columns(state, entry) else - # Action rows carry no field change. + # Action rows carry no field change: their two value columns hold what was submitted to the action + # and what it answered, so applying either as a column change would corrupt the rebuild. state end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb new file mode 100644 index 000000000..243192c1e --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb @@ -0,0 +1,39 @@ +module ForestAdminAgent + module AuditTrail + module Sql + # The audit table's schema, as an ordered, append-only list. Never edit an entry that has shipped: + # add another one, since a database out there has already recorded the earlier ones as applied. Every + # statement is written to tolerate being replayed (`if_not_exists`), because a store that predates the + # per-table tracking key replays them once on upgrade. + module Migrations + ALL = [ + { + name: '001-create-audit-logs', + up: lambda do |connection, table| + # if_not_exists: a non-PG race (no advisory lock) can let two instances both reach here. + connection.create_table(table, if_not_exists: true) do |t| + t.datetime :timestamp, null: false + t.string :operation, null: false + t.string :collection, null: false + t.string :record_id, null: false + t.integer :user_id + t.string :correlation_key + t.json :previous_values + t.json :new_values + end + end + }, + { + name: '002-index-record-and-correlation', + up: lambda do |connection, table| + base = table.split('.').last + connection.add_index(table, :record_id, name: "#{base}_record_id", if_not_exists: true) + connection.add_index(table, :correlation_key, name: "#{base}_correlation_key", if_not_exists: true) + connection.add_index(table, :user_id, name: "#{base}_user_id", if_not_exists: true) + end + } + ].freeze + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb index 294151703..c0963ee4c 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb @@ -1,8 +1,8 @@ module ForestAdminAgent module AuditTrail module Sql - # Creates and evolves the audit table through an ordered, append-only list of migrations, tracked - # in a dedicated `audit_migrations` table (namespaced in the `forest` schema on Postgres). + # Applies {Migrations::ALL} to the audit table, tracking what has run in a dedicated + # `audit_migrations` table (namespaced in the `forest` schema on Postgres). # # On Postgres the migrations run inside a transaction-scoped advisory lock, so several agent # instances booting at once apply them one after another instead of racing on the same DDL. The @@ -15,34 +15,6 @@ class Migrator # duplicate_schema, and the unique violation on pg_namespace the same race can raise instead. DUPLICATE_SCHEMA_STATES = %w[42P06 23505].freeze - MIGRATIONS = [ - { - name: '001-create-audit-logs', - up: lambda do |connection, table| - # if_not_exists: a non-PG race (no advisory lock) can let two instances both reach here. - connection.create_table(table, if_not_exists: true) do |t| - t.datetime :timestamp, null: false - t.string :operation, null: false - t.string :collection, null: false - t.string :record_id, null: false - t.integer :user_id - t.string :correlation_key - t.json :previous_values - t.json :new_values - end - end - }, - { - name: '002-index-record-and-correlation', - up: lambda do |connection, table| - base = table.split('.').last - connection.add_index(table, :record_id, name: "#{base}_record_id", if_not_exists: true) - connection.add_index(table, :correlation_key, name: "#{base}_correlation_key", if_not_exists: true) - connection.add_index(table, :user_id, name: "#{base}_user_id", if_not_exists: true) - end - } - ].freeze - def initialize(connection, schema:, table_name:) @connection = connection @schema = schema # nil on adapters without schema support @@ -112,7 +84,7 @@ def apply_pending done = applied_migrations table = qualified(@table_name) - MIGRATIONS.each do |migration| + Migrations::ALL.each do |migration| # The tracking table is shared by every audit table of the database/schema, so the target # table belongs in the key: a store configured with another `table_name` must still get its # own table created instead of reading someone else's migration as done. Rows written by diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb index 92be245ae..728b0df96 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb @@ -9,6 +9,8 @@ module AuditTrail class Store DEFAULT_SCHEMA = 'forest'.freeze DEFAULT_TABLE = 'audit_logs'.freeze + COLUMNS = %i[timestamp operation collection record_id user_id correlation_key + previous_values new_values].freeze def initialize(database:, schema: DEFAULT_SCHEMA, table_name: DEFAULT_TABLE) @database = database @@ -113,30 +115,18 @@ def qualified(connection) schema ? "#{schema}.#{@table_name}" : @table_name end + # An AuditRecord is a Struct and a row answers to `[]` too, so the mapping is the column list itself. def to_row(record) - { - timestamp: record.timestamp, - operation: record.operation, - collection: record.collection, - record_id: record.record_id, - user_id: record.user_id, - correlation_key: record.correlation_key, - previous_values: record.previous_values, - new_values: record.new_values - } + COLUMNS.to_h { |column| [column, record[column]] } end def from_row(row) - AuditRecord.new( - timestamp: row.timestamp.respond_to?(:iso8601) ? row.timestamp.iso8601(3) : row.timestamp.to_s, - operation: row.operation, - collection: row.collection, - record_id: row.record_id, - user_id: row.user_id, - correlation_key: row.correlation_key, - previous_values: row.previous_values || {}, - new_values: row.new_values || {} - ) + values = COLUMNS.to_h { |column| [column, row[column]] } + values[:timestamp] = row.timestamp.respond_to?(:iso8601) ? row.timestamp.iso8601(3) : row.timestamp.to_s + values[:previous_values] ||= {} + values[:new_values] ||= {} + + AuditRecord.new(**values) end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb index 75dd437ec..0d99f6dfc 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb @@ -122,7 +122,7 @@ def handle_hook_request(args = {}) # to say so through `result_builder.error` instead. def execute_and_audit(context, args, data, filter) result = context.collection.execute(context.caller, @action_name, data, filter) - audit_action(context, args, data, failed: error_result?(result)) + audit_action(context, args, data, result: result, failed: error_result?(result)) result rescue StandardError @@ -134,7 +134,7 @@ def error_result?(result) result.is_a?(Hash) && result[:type] == 'Error' end - def audit_action(context, args, data, failed: false) + def audit_action(context, args, data, result: nil, failed: false) capture = ForestAdminAgent::AuditTrail::ActionCapture.new( ForestAdminAgent::AuditTrail.store, ForestAdminAgent::AuditTrail.options[:redact] ) @@ -144,6 +144,7 @@ def audit_action(context, args, data, failed: false) collection: context.collection.name, form_values: data, record_ids: audited_record_ids(args, context), + result: result, failed: failed ) end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb index d20756c9d..66563f7c2 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb @@ -36,8 +36,8 @@ def record(capture: described_class.new(store), **over) expect(audit.record_id).to eq('4') expect(audit.user_id).to eq(42) expect(audit.correlation_key).to eq('req-xyz') - expect(audit.previous_values).to eq({}) - expect(audit.new_values).to eq({ 'amount' => 30, 'reason' => 'damaged' }) + expect(audit.previous_values).to eq({ 'amount' => 30, 'reason' => 'damaged' }) + expect(audit.new_values).to eq({}) end it 'writes one row per selected record, sharing timestamp and correlation key' do @@ -65,7 +65,50 @@ def record(capture: described_class.new(store), **over) audit = record(capture: capture).last - expect(audit.new_values).to eq({ 'amount' => 30, 'reason' => Recording::REDACTED }) + expect(audit.previous_values).to eq({ 'amount' => 30, 'reason' => Recording::REDACTED }) + end + + describe 'the action answer' do + it 'keeps the type and the message of a success' do + audit = record(result: { response_headers: {}, type: 'Success', message: 'Refunded', invalidated: [] }).last + + expect(audit.new_values).to eq({ 'type' => 'Success', 'message' => 'Refunded' }) + end + + it 'keeps the type and the message of an error' do + audit = record(result: { type: 'Error', message: 'not allowed', html: nil }, failed: true).last + + expect(audit.new_values).to eq({ 'type' => 'Error', 'message' => 'not allowed' }) + end + + # A file result carries the whole file; a webhook carries its body and headers, which is where + # credentials live. None of that belongs in an audit table. + it 'never keeps a file stream, nor a webhook body or headers' do + file = record(result: { type: 'File', name: 'refunds.csv', mime_type: 'text/csv', + stream: 'id,amount\n1,30' }).last + webhook = record(result: { type: 'Webhook', url: 'https://pay.test/refund', method: 'POST', + body: { 'secret' => 'x' }, headers: { 'Authorization' => 'Bearer t' } }).last + + expect(file.new_values).to eq({ 'type' => 'File', 'name' => 'refunds.csv', 'mime_type' => 'text/csv' }) + expect(webhook.new_values).to eq({ 'type' => 'Webhook', 'url' => 'https://pay.test/refund', + 'method' => 'POST' }) + end + + it 'drops the response headers of any result' do + audit = record(result: { response_headers: { 'Set-Cookie' => 'session=x' }, type: 'Success' }).last + + expect(audit.new_values).to eq({ 'type' => 'Success' }) + end + + it 'keeps where a redirect pointed' do + audit = record(result: { type: 'Redirect', path: '/orders/4' }).last + + expect(audit.new_values).to eq({ 'type' => 'Redirect', 'path' => '/orders/4' }) + end + + it 'stores no answer at all when the action raised' do + expect(record(failed: true).last.new_values).to eq({}) + end end it 'does nothing without a configured store' do diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/record_state_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/record_state_spec.rb index e4633766f..d5b93aa81 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/record_state_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/record_state_spec.rb @@ -65,10 +65,16 @@ def entry(operation, previous_values = {}, new_values = {}) expect(described_class.at({ 'status' => 'draft' }, entries)).to eq({ 'status' => 'archived' }) end - it 'ignores smart-action rows, which carry no field change' do + # An action row's two value columns hold the submitted form and the action's answer, not a record's + # before and after: applying either would corrupt the rebuild. + it 'ignores smart-action rows, whichever way they went' do current = { 'status' => 'shipped' } + entries = [ + entry('action', { 'status' => 'submitted value' }, { 'type' => 'Success' }), + entry('action_failed', { 'status' => 'submitted value' }, {}) + ] - expect(described_class.at(current, [entry('action', {}, { 'amount' => 30 })])).to eq(current) + expect(described_class.at(current, entries)).to eq(current) end it 'undoes a nested change without disturbing the rest of the object' do diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb index c9f32eea5..ba31c2925 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb @@ -36,12 +36,15 @@ def connection ) end - it 'persists a smart-action row, form values included' do - store.append(record(operation: 'action', previous_values: {}, new_values: { 'amount' => 30 })) + # An action row uses the two sides for what went in and what came back. + it 'persists a smart-action row, submitted form and answer included' do + store.append(record(operation: 'action', previous_values: { 'amount' => 30 }, + new_values: { 'type' => 'Success', 'message' => 'Refunded' })) audit = store.list_by_record(collection: 'accounts', record_id: '1').first expect(audit.operation).to eq('action') - expect(audit.new_values).to eq({ 'amount' => 30 }) + expect(audit.previous_values).to eq({ 'amount' => 30 }) + expect(audit.new_values).to eq({ 'type' => 'Success', 'message' => 'Refunded' }) end it 'persists and reads back a record, decoding the JSON columns' do diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb index fce92144b..620411e70 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb @@ -61,7 +61,7 @@ def execute(result: { type: 'Success', message: 'done' }, &raising) expect(store).to have_received(:append).twice expect(store).to have_received(:append).with( having_attributes(operation: 'action', collection: 'orders', record_id: '4', - new_values: { 'amount' => 30, 'reason' => '[redacted]' }) + previous_values: { 'amount' => 30, 'reason' => '[redacted]' }) ) end @@ -80,6 +80,20 @@ def execute(result: { type: 'Success', message: 'done' }, &raising) end # An action reporting failure through result_builder.error never raises, so only the result says so. + it 'records what the action answered' do + execute(result: { type: 'Success', message: 'done', response_headers: { 'X-Token' => 'x' } }) + + expect(store).to have_received(:append).with( + having_attributes(operation: 'action', new_values: { 'type' => 'Success', 'message' => 'done' }) + ).twice + end + + it 'records no answer when the action raised' do + expect { execute { raise StandardError, 'boom' } }.to raise_error(StandardError) + + expect(store).to have_received(:append).with(having_attributes(new_values: {})).twice + end + it 'records an Error result as a failed run, and still returns it' do result = execute(result: { type: 'Error', message: 'not allowed' }) From d95e805d92915d52b32c264aca00826cc51b597a Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Mon, 17 Aug 2026 16:54:50 +0200 Subject: [PATCH 25/35] feat(audit_trail): pending/confirm protocol, denormalised identity, richer rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared contract with the Node agent, plus the hardening that came with it. Schema, as new append-only migrations: the caller's first name, last name and email denormalised onto every row (who acted then, not who holds that id today); `action_name`; a not-null `status` backfilled to `done`; and `record_id` widened to TEXT and made nullable, since a create's pending row has no id yet and a packed composite id outgrows a varchar. Widening it rebuilds the indexes 002 created — sqlite drops them all with the table, and MySQL cannot index unbounded TEXT without a length prefix. Write protocol: a pending row before the write, confirmed after, one code path in both modes so `status` always means something. `config.audit_trail[:critical]` (default false) decides what a failed pending insert costs: the operation, or the row. What it buys is no unaudited write, not exact after-values everywhere — a row left pending says the write may or may not have landed, and that residue is evidence. A write that changed nothing has its row discarded instead. Also, per the review: - connect and migrate at boot rather than on first write, so a bad database stops the agent starting instead of recording nothing while looking healthy; and the class-level connection now refuses a second, different database rather than silently clobbering the first pool - one operation audits at most 500 records, batched, with `N audited, M skipped` logged — a "delete all" no longer materialises every row, twice - action results: url and path sanitised (userinfo, query, fragment), answer keys camelCased on write, and the targeted ids read back through the caller's filter instead of trusted from the request - the action's selection read and pending insert both sit inside the gate, so neither runs without an audit database nor 500s an action that already ran - /state projects primary keys plus writable columns only, matching what is actually recorded - the update diff is taken against the persisted record, and the row filed under the id the record ended up with - payload exposes the row `id`, and the history route carries `meta.availableUsers` on the first fetch Co-Authored-By: Claude Opus 5 (1M context) --- .rubocop.yml | 2 + packages/forest_admin_agent/AUDIT_TRAIL.md | 60 +++- .../lib/forest_admin_agent/audit_trail.rb | 47 ++- .../audit_trail/action_capture.rb | 103 ++++--- .../audit_trail/audit_record.rb | 13 +- .../forest_admin_agent/audit_trail/capture.rb | 217 ++++++++----- .../audit_trail/recording.rb | 26 +- .../audit_trail/snapshots.rb | 60 ++++ .../audit_trail/sql/audit_connection_base.rb | 22 ++ .../audit_trail/sql/migrations.rb | 44 +++ .../forest_admin_agent/audit_trail/store.rb | 60 +++- .../builder/agent_factory.rb | 9 +- .../routes/action/actions.rb | 82 +++-- .../routes/resources/audit_trail.rb | 26 +- .../routes/resources/audit_trail_route.rb | 16 +- .../audit_trail/action_capture_spec.rb | 196 +++++++----- .../audit_trail/capture_spec.rb | 286 +++++++++++------- .../audit_trail/sql/migrator_spec.rb | 2 + .../audit_trail/store_spec.rb | 95 +++++- .../routes/action/actions_spec.rb | 139 +++++---- .../routes/resources/audit_trail_spec.rb | 49 ++- .../forest_admin_agent/spec/spec_helper.rb | 4 + 22 files changed, 1138 insertions(+), 420 deletions(-) create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb diff --git a/.rubocop.yml b/.rubocop.yml index 24c0e14e2..c6985f27d 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -360,6 +360,8 @@ Metrics/BlockLength: Metrics/ClassLength: Exclude: + - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb' + - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb' - 'packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/collection.rb' - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/introspection/introspector.rb' diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index 5ad2d4d1a..3a514255d 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -52,10 +52,32 @@ ForestAdminAgent::Builder::AgentFactory.instance.setup( | `schema` | Postgres schema holding the table (default `forest`; ignored on other adapters) | | `table_name` | default `audit_logs` | | `redact` | `{ 'collection_name' => ['field', ...] }` — values masked while recording the change | +| `critical` | default `false`. `true` refuses an operation the audit trail cannot record — see below | -On the first write or read the store ensures the schema exists and runs any pending migrations; every -create / update / delete performed through Forest then writes one row per record, and the **Historic** -tab in the UI reads from the same table. +The store connects and migrates **at boot**, not on the first write: an audit database the agent cannot +reach stops it starting, rather than leaving it looking healthy while recording nothing. Every create / +update / delete performed through Forest then writes one row per record, and the **Historic** tab in the UI +reads from the same table. + +## The write protocol, and `critical` + +Every operation is recorded twice: a `pending` row **before** the write, confirmed `done` **after** it. One +code path either way, so `status` always means the same thing. + +| `critical` | a pending row that cannot be written | +| ---------- | ------------------------------------------------------------------------------------------ | +| `false` | is logged and dropped; the operation goes ahead unaudited (the default, today's behaviour) | +| `true` | **refuses the operation**. Nothing was written, so there is nothing to repair and no compensating write ever happens | + +What this buys is **no unaudited write** — not that every row holds exact after-values. A row left `pending` +means the write may or may not have landed: that residue is evidence, and it is the point. Everything after +the pending insert stays best-effort in both modes, because by then the write has happened and raising would +report a failure for an operation that succeeded. + +Two consequences worth knowing. A write that turns out to change nothing has its pending row **discarded** +rather than confirmed, so no-op updates leave no trace. And one operation audits at most 500 records: a +wider selection is truncated, with `N records audited, M skipped` logged at `Warn` — an incomplete audit +somebody knows about beats an OOM, and beats silence. ## Routes @@ -71,8 +93,11 @@ by default) together with the filtered total: { "data": [ /* current page rows */ ], "meta": { "count": 137 } } ``` -`meta.count` is the number of rows matching the active filters (not the absolute total) and is -independent of the page. Optional filters (all combine with `AND`; omit them for the full history): +`meta.count` is the number of rows matching the active filters (not the absolute total) and is independent of +the page. `meta.availableUsers` rides along **on the first fetch only** — the front keeps the list it saw — +and holds the distinct authors of the entries the current filters match, as `{ id, firstName, lastName, +email }`, whatever page was asked for. The identity comes from the rows themselves, so someone since renamed +or removed still reads as they were when they acted. Optional filters (all combine with `AND`; omit them for the full history): | query param | format | effect | | ----------- | -------------------------------- | ----------------------------------------------- | @@ -104,9 +129,13 @@ erroring. Sorting follows JSON:API `sort` on `timestamp`: `sort=-timestamp` (or is newest first, `sort=timestamp` is oldest first. Ties on equal timestamps fall back to insertion order (the auto-increment `id`), so paging is deterministic in either direction. -All three routes serialize audit records the same way: top-level keys are camelCased -(`recordId`, `userId`, `correlationKey`, `previousValues`, `new_values` → `newValues`), while the -`previousValues` / `newValues` hashes keep the audited record's own column names. +All routes serialize audit records the same way: top-level keys are camelCased — `id`, `recordId`, `userId`, +`userFirstName`, `userLastName`, `userEmail`, `actionName`, `status`, `correlationKey`, `previousValues`, +`newValues`. The row `id` is exposed because both agents order by `(timestamp, id)` and the front uses it as +the merge tiebreaker. + +Inside the value objects: a record's column names pass through untouched, while an action answer's keys are +Forest's own and so are camelCase (`mimeType`, not `mime_type`) — the agent transforms them on write. A record that no longer exists keeps its history: only a record that still exists *outside* the caller's permission scope is refused (404). Inspecting what was deleted is much of the point of an @@ -188,6 +217,14 @@ A **global** action targets no record, and a **select-all** selection only tells attached to no record. Recording is best-effort — a failing audit database logs an error rather than breaking the action. +The targeted records are read back through the caller's own filter rather than taken from the request: the ids +a client sends are a claim, and in a compliance record asserting that an operator acted on a record their +scope excludes is worse than a missing row. + +`url` and `path` are sanitised before storage — credentials in the userinfo and anything in a query string or +fragment come off, since either can carry a signed one-time token that would otherwise sit permanently in the +one table nobody deletes from. + > **What an action changes is only audited when it goes through Forest.** `context.collection.update(...)` > passes through the same hooks as any other write, so it produces the usual field-level rows sharing the > action's `correlationKey`. A direct ORM write (`Customer.find(id).update!(...)`) is invisible to the @@ -199,12 +236,15 @@ breaking the action. | column | description | | ----------------- | ----------------------------------------------------------- | -| `id` | auto-increment primary key | +| `id` | auto-increment primary key, exposed in the payload | +| `status` | `pending` before the write, `done` once confirmed | | `timestamp` | when the change happened | | `operation` | `create` / `update` / `delete` | | `collection` | audited collection name | -| `record_id` | packed record id (primary keys joined by `\|`) | +| `record_id` | packed record id (primary keys joined by `\|`), TEXT and nullable — a create's pending row has none yet, and a composite id outgrows a varchar | | `user_id` | the Forest user who made the change | +| `user_first_name`, `user_last_name`, `user_email` | denormalised from the caller at write time: who acted then, not whoever holds that id today | +| `action_name` | smart-action rows only | | `correlation_key` | per-request id; groups every change made within one request | | `previous_values` | values before the change (JSON) | | `new_values` | values after the change (JSON) | diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb index 56e3d2994..6ef79960f 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb @@ -1,8 +1,24 @@ module ForestAdminAgent - # The audit trail is inert unless `config.audit_trail[:database]` was set: the agent factory builds - # the store during setup and stores it in the config, and everything (capture layers and routes) - # resolves it from here. + # The audit trail is inert unless `config.audit_trail[:database]` was set: the agent factory builds the + # store during setup — connecting and migrating there rather than on first write — and everything (capture + # layers and routes) resolves it from here. module AuditTrail + # One operation must not materialise an unbounded number of records: a "delete all" would otherwise read + # every matched row and, with the pending/confirm protocol, write each of them twice. Truncation is logged, + # never silent. + # ponytail: 500 covers any hand-made bulk edit; raise it if a real workflow needs more. + MAX_RECORDS_PER_OPERATION = 500 + + def self.log_truncation(kept, total) + skipped = total ? total - kept : 'further' + + Facades::Container.logger.log( + 'Warn', + "[ForestAdmin] Audit trail: #{kept} records audited, #{skipped} skipped " \ + "(cap #{MAX_RECORDS_PER_OPERATION} per operation)" + ) + end + def self.options config = Facades::Container.config_from_cache @@ -12,5 +28,30 @@ def self.options def self.store options[:store] end + + # `critical: true` makes the pending insert a precondition of the write: if the audit trail cannot record + # that an operation is about to happen, the operation is refused. Nothing was written, so there is nothing + # to repair and no compensating write ever happens. Default false keeps today's behaviour, where a broken + # audit database costs rows rather than writes. + def self.critical? + options[:critical] == true + end + + def self.log_failure(error) + Facades::Container.logger.log('Error', "[ForestAdmin] Audit trail unavailable, skipping: #{error.message}") + end + + # Runs the pending insert under the configured policy: refusing the operation when critical, logging and + # carrying on otherwise. + def self.gate + return yield if critical? + + begin + yield + rescue StandardError => e + log_failure(e) + nil + end + end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb index e594e6a4b..412d77b23 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb @@ -1,65 +1,96 @@ +require 'uri' + module ForestAdminAgent module AuditTrail - # Records smart-action invocations into the same table as the field-level history. + # Records smart-action runs into the same table as the field-level history: the submitted form on the + # `previous_values` side, what the action answered on the `new_values` side. + # + # {Capture} cannot see them — the customizer has no `Execute` hook — and an action's writes are only + # audited when they go through the Forest data layer, so a direct ORM write stays invisible. What lands + # here is the run itself: who ran which action, on which records, with which form, and how it ended. # - # {Capture} cannot see them: the customizer has no `Execute` hook, and an action's writes are only - # audited when it goes through the Forest data layer — a direct ORM write (`Customer.find(id).update!`) - # is invisible to the agent and is therefore not recorded at all. What lands here is the invocation - # itself: on which records, with which form values, and whether it raised. Which action it was lives in - # the Forest activity logs — `correlation_key` is the join. + # Same protocol as a write: {#pending} before the action, {#confirm} after. The route owns the gate, so a + # pending insert that fails refuses the run under `critical: true`. class ActionCapture include Recording EXECUTED = 'action'.freeze FAILED = 'action_failed'.freeze - # What of the action's answer is worth keeping — an allowlist, not a denylist: a result also carries - # the file's contents, a webhook's body and headers and arbitrary response headers. File bytes have no - # business in an audit table and the other two routinely hold credentials, and an allowlist means a - # field added to a result later is not stored until someone decides it should be. `html` is left out - # too: operator-facing markup, sometimes large, and the message already says what happened. - RESULT_FIELDS = %i[type message name mime_type method url path].freeze - # Bulk runs over a select-all selection, and global actions, target no id we can name without - # querying the whole selection: they get one row attached to no record rather than none at all. + # A global action, and a bulk run over a selection wider than the cap, name no single target: they get + # one row attached to no record rather than none at all. NO_RECORD = ''.freeze + # What of the action's answer is worth keeping — an allowlist, not a denylist: a result also carries the + # file's contents, a webhook's body and headers and arbitrary response headers. File bytes have no + # business in an audit table and the other two routinely hold credentials, and an allowlist means a field + # added to a result later is not stored until someone decides it should be. `html` is left out too: + # operator-facing markup, sometimes large, and the message already says what happened. + RESULT_FIELDS = %i[type message name mime_type method url path].freeze + # Either can carry userinfo credentials or a signed one-time token, which would then sit permanently in + # the one table nobody deletes from. + URL_FIELDS = %i[url path].freeze def initialize(store, redact = {}) @store = store @redact = redact || {} end - # No-op unless an audit database was configured, and best-effort: the action has already run. - def record(caller:, collection:, form_values:, record_ids:, result: nil, failed: false) - return unless @store + # One row per targeted record, provisionally an `action` — {#confirm} settles which it really was. Returns + # the row ids to confirm. + def pending(caller:, collection:, action_name:, form_values:, record_ids:) + return [] unless @store - audit_safely do - timestamp = now - correlation_key = correlation_key_for(caller) - values = redact(form_values || {}, @redact[collection] || []) - ids = record_ids.empty? ? [NO_RECORD] : record_ids + timestamp = now + correlation_key = correlation_key_for(caller) + identity = identity_of(caller) + submitted = redact(form_values || {}, @redact[collection] || []) + ids = record_ids.empty? ? [NO_RECORD] : record_ids - ids.each do |record_id| - @store.append( - AuditRecord.new( - timestamp: timestamp, - operation: failed ? FAILED : EXECUTED, - collection: collection, - record_id: record_id, - user_id: caller&.id, - correlation_key: correlation_key, - previous_values: values, - new_values: summarize(result) || {} - ) + @store.append_all( + ids.map do |record_id| + AuditRecord.new( + timestamp: timestamp, operation: EXECUTED, collection: collection, record_id: record_id, + status: PENDING, action_name: action_name, correlation_key: correlation_key, + previous_values: submitted, new_values: {}, **identity ) end + ) + end + + # Best-effort: the action has already run, so a failure here loses the answer, never the run. + def confirm(ids, result: nil, failed: false) + return if ids.nil? || ids.empty? + + audit_safely do + answer = summarize(result) + + ids.each { |id| @store.confirm(id, operation: failed ? FAILED : EXECUTED, new_values: answer) } end end private + # Keys of an action's answer are Forest's own, so they are camelCase on the wire — unlike a record's + # column names, which pass through untouched. def summarize(result) - return nil unless result.is_a?(Hash) + return {} unless result.is_a?(Hash) + + result.slice(*RESULT_FIELDS).compact.to_h do |field, value| + [field.to_s.camelize(:lower), URL_FIELDS.include?(field) ? sanitize_url(value) : value] + end + end + + # `userinfo = nil` is a no-op on URI, so the credentials come off textually; the parser then takes care + # of the query and fragment. + def sanitize_url(value) + bare = value.to_s.sub(%r{\A([a-z][a-z0-9+.-]*://)[^/@]*@}i, '\1') + uri = URI.parse(bare) + uri.query = nil + uri.fragment = nil - result.slice(*RESULT_FIELDS).compact.transform_keys(&:to_s) + uri.to_s + rescue StandardError + # Not something the parser accepts: keep the shape, drop everything that can carry a secret. + bare.to_s.split(/[?#]/).first.to_s end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb index 011d4238a..63750812b 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb @@ -1,10 +1,15 @@ module ForestAdminAgent module AuditTrail - # One audited change. Mirrors the columns of `forest.audit_logs`; only the actor's `user_id` is - # stored, the rest of the actor identity is correlated elsewhere through `correlation_key`. + # One audited change. Mirrors the columns of `forest.audit_logs`. + # + # The actor's name and email are denormalised from the caller at write time: the row says who acted then, + # not whoever holds that user id today. `action_name` is set on smart-action rows only, and `status` + # follows the write protocol — inserted as {Recording::PENDING} before the write and confirmed + # {Recording::DONE} after, so a row left pending means the write may or may not have landed. AuditRecord = Struct.new( - :timestamp, :operation, :collection, :record_id, :user_id, :correlation_key, - :previous_values, :new_values, + :id, :timestamp, :operation, :collection, :record_id, :status, + :user_id, :user_first_name, :user_last_name, :user_email, :action_name, + :correlation_key, :previous_values, :new_values, keyword_init: true ) end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb index 55d397ef4..1833f3d9b 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb @@ -1,19 +1,21 @@ module ForestAdminAgent module AuditTrail - # Datasource-agnostic capture layer, installed by the agent factory as soon as an audit-trail - # database is configured. It instruments every collection through the Forest customizer hooks, so - # it behaves the same whatever the audited datasource is (ActiveRecord, Mongoid, ...), computes the - # minimal before/after diff for each change and appends an {AuditRecord} to the store. + # Datasource-agnostic capture layer, installed by the agent factory as soon as an audit-trail database is + # configured. It instruments every collection through the Forest customizer hooks, so it behaves the same + # whatever the audited datasource is (ActiveRecord, Mongoid, ...). + # + # Every operation is recorded twice: a PENDING row before the write, confirmed DONE after it. One code + # path in both `critical` modes, so `status` always means the same thing — and what the protocol buys is + # that no write goes unaudited, not that every row holds exact after-values. A row left pending says the + # write may or may not have landed, which is evidence rather than a defect. class Capture include Recording - # ponytail: 16 deep is far past any legitimate nesting; raise it if one ever gets that far. - MAX_SNAPSHOTS = 16 - # Signature imposed by DatasourceCustomizer#use; the agent always instruments the whole datasource. def run(datasource_customizer, _collection_customizer = nil, options = {}) @store = options[:store] @redact = options[:redact] || {} + @snapshots = Snapshots.new datasource_customizer.collections.each_value { |collection| instrument(collection) } end @@ -33,108 +35,175 @@ def instrument(collection_customizer) projection = ForestAdminDatasourceToolkit::Components::Query::Projection.new( (primary_keys + columns).uniq ) - name = collection_customizer.name + target = { columns: columns, primary_keys: primary_keys, projection: projection, + name: collection_customizer.name } - add_create_hook(collection_customizer, columns, primary_keys, name) - add_update_hooks(collection_customizer, columns, primary_keys, name, projection) - add_delete_hooks(collection_customizer, columns, primary_keys, name, projection) + add_create_hooks(collection_customizer, target) + add_update_hooks(collection_customizer, target) + add_delete_hooks(collection_customizer, target) end # The "after" hooks are prepended: `execute_after` stops at the first exception, so a customization # raising in its own after hook would otherwise drop the record of a write that already happened. - # The "before" hooks stay appended, so they read the filter and patch every other customization + # The "before" hooks stay appended, so they read the data, filter and patch every other customization # has had its say on. - def add_create_hook(collection_customizer, columns, primary_keys, name) + def add_create_hooks(collection_customizer, target) + collection_customizer.add_hook('Before', 'Create') do |context| + # No record id yet — that is what the column being nullable is for. + @snapshots.push(ids: pending_rows(context.caller, 'create', target[:name], + [{ record_id: nil, new_values: pick(context.data, target[:columns]) }])) + end + collection_customizer.add_hook('After', 'Create', prepend: true) do |context| - emit( - context.caller, 'create', name, record_id(context.record, primary_keys), - {}, pick(context.record, columns) - ) + pending = @snapshots.pop + next unless pending + + confirm(pending[:ids].first, + record_id: record_id(context.record, target[:primary_keys]), + new_values: redacted(target[:name], pick(context.record, target[:columns]))) end end - def add_update_hooks(collection_customizer, columns, primary_keys, name, projection) + def add_update_hooks(collection_customizer, target) collection_customizer.add_hook('Before', 'Update') do |context| - # Snapshot the patch here: installed last, this hook sees what actually gets written, while - # the after-context is handed the original patch the caller sent. - push_snapshot(records: snapshot(context, projection), patch: context.patch) + records = @snapshots.take(context, target[:projection]) + @snapshots.push( + records: records, + patch: context.patch, + ids: pending_rows( + context.caller, 'update', target[:name], + records.map do |record| + { record_id: record_id(record, target[:primary_keys]), + previous_values: pick(record, context.patch.keys & target[:columns]), + new_values: pick(context.patch, target[:columns]) } + end + ) + ) end collection_customizer.add_hook('After', 'Update', prepend: true) do |context| - snapshot = snapshots.pop - - snapshot&.fetch(:records)&.each do |record| - delta = Diff.changed_values(record, snapshot[:patch], columns) - next if delta[:new_values].empty? + pending = @snapshots.pop + next unless pending - emit( - context.caller, 'update', name, record_id(record, primary_keys), - delta[:previous_values], delta[:new_values] - ) - end + confirm_updates(context, pending, target) end end - def add_delete_hooks(collection_customizer, columns, primary_keys, name, projection) + def add_delete_hooks(collection_customizer, target) collection_customizer.add_hook('Before', 'Delete') do |context| - push_snapshot(records: snapshot(context, projection)) + records = @snapshots.take(context, target[:projection]) + @snapshots.push( + records: records, + ids: pending_rows( + context.caller, 'delete', target[:name], + records.map do |record| + { record_id: record_id(record, target[:primary_keys]), + previous_values: pick(record, target[:columns]) } + end + ) + ) end - collection_customizer.add_hook('After', 'Delete', prepend: true) do |context| - snapshot = snapshots.pop + collection_customizer.add_hook('After', 'Delete', prepend: true) do |_context| + pending = @snapshots.pop + next unless pending - snapshot&.fetch(:records)&.each do |record| - emit( - context.caller, 'delete', name, record_id(record, primary_keys), - pick(record, columns), {} - ) - end + pending[:ids].each { |id| confirm(id) } end end - # Read outside the write's transaction — hooks bracket the write as separate calls and the data layer - # exposes no lock, on purpose, since it spans ActiveRecord, Mongoid, HTTP APIs... So two updates - # racing on the same record both snapshot the same state, and the one that lands second records a - # `previous_values` that was already overwritten. `new_values` stays exact (it is the patch). - # - # An empty snapshot on failure rather than no snapshot at all: the after hook pops unconditionally, - # so skipping the push would pair it with an unrelated entry. - def snapshot(context, projection) - audit_safely { context.collection.list(context.filter, projection) } || [] - end + # The diff is taken against the record as persisted, not the patch that was requested, so a value the + # datasource normalised or a decorator rewrote is recorded as what actually landed. The id is packed from + # those same values, so an update that changed a primary key files the row under the id the history will + # be queried by. + def confirm_updates(context, pending, target) + pks = target[:primary_keys] + persisted = reread(context, pending[:records], pending[:patch], target) + empty = [] + + pending[:records].each_with_index do |record, index| + expected = record.merge(pending[:patch]) + after = persisted[record_id(expected, pks)] || expected + delta = Diff.changed_values(record, after, target[:columns]) + + if delta[:new_values].empty? + empty << pending[:ids][index] + else + confirm(pending[:ids][index], + record_id: record_id(after, pks), + previous_values: redacted(target[:name], delta[:previous_values]), + new_values: redacted(target[:name], delta[:new_values])) + end + end - # Snapshots taken in a "before" hook and consumed in the matching "after" hook. Both bracket one - # operation on one thread, so a LIFO stack pairs them without relying on the filter object - # reaching both hooks unchanged. An operation raising in between strands its entry, hence the cap. - def push_snapshot(snapshot) - stack = snapshots - stack.shift while stack.size >= MAX_SNAPSHOTS - stack.push(snapshot) + audit_safely { @store.discard(empty.compact) } if empty.any? end - def snapshots - Thread.current[:forest_audit_trail_snapshots] ||= [] + # Reads the updated records back in one query, keyed by their own id, so each snapshot can find what + # actually landed — including when the patch moved a primary key. + def reread(context, records, patch, target) + pks = target[:primary_keys] + return {} if records.empty? + + audit_safely do + condition = ids_condition(pks, records.map { |record| record.merge(patch).slice(*pks) }) + filter = ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: condition) + + context.collection.list(filter, target[:projection]).to_h { |row| [record_id(row, pks), row] } + end || {} end - def emit(caller, operation, collection, record_id, previous_values, new_values) - redacted = @redact[collection] || [] + # Built from the primary keys the instrumentation already resolved, rather than through + # ConditionTreeFactory, which would need the underlying collection a hook context does not hand out. + def ids_condition(primary_keys, ids) + leaf = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Nodes::ConditionTreeLeaf + factory = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::ConditionTreeFactory - audit_safely do - @store.append( - AuditRecord.new( - timestamp: now, - operation: operation, - collection: collection, - record_id: record_id, - user_id: caller&.id, - correlation_key: correlation_key_for(caller), - previous_values: redact(previous_values, redacted), - new_values: redact(new_values, redacted) - ) + operators = ForestAdminDatasourceToolkit::Components::Query::ConditionTree::Operators + + if primary_keys.size == 1 + key = primary_keys.first + + leaf.new(key, operators::IN, ids.map { |id| id[key] }.uniq) + else + factory.union( + ids.map { |id| factory.intersect(id.map { |key, value| leaf.new(key, operators::EQUAL, value) }) } ) end end + # The one place the audit trail may refuse an operation, and only under `critical: true`: if we cannot + # record that a write is about to happen, the write does not happen. + def pending_rows(caller, operation, collection, rows) + timestamp = now + correlation_key = correlation_key_for(caller) + identity = identity_of(caller) + + AuditTrail.gate do + @store.append_all( + rows.map do |row| + AuditRecord.new( + timestamp: timestamp, operation: operation, collection: collection, status: PENDING, + correlation_key: correlation_key, record_id: row[:record_id], + previous_values: redacted(collection, row[:previous_values] || {}), + new_values: redacted(collection, row[:new_values] || {}), + **identity + ) + end + ) + end || [] + end + + def confirm(id, attributes = {}) + return unless id + + audit_safely { @store.confirm(id, attributes) } + end + + def redacted(collection, values) + redact(values, @redact[collection] || []) + end + def record_id(record, primary_keys) primary_keys.map { |pk| record[pk].to_s }.join('|') end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb index 76bcb39e6..614263c0a 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb @@ -4,9 +4,23 @@ module ForestAdminAgent module AuditTrail # Shared by the two capture layers: {Capture} for the changes Forest writes, {ActionCapture} for the - # smart actions it runs. + # smart actions it runs. Holds the write protocol's vocabulary and its failure policy. module Recording REDACTED = '[redacted]'.freeze + # A row is inserted before the write and confirmed after it. One left PENDING means the write may or + # may not have landed — that residue is evidence, and it is the point. + PENDING = 'pending'.freeze + DONE = 'done'.freeze + + IDENTITY = { user_id: :id, user_first_name: :first_name, + user_last_name: :last_name, user_email: :email }.freeze + + # Denormalised at write time, so the row says who acted then rather than whoever holds that id today. + # Read defensively: a caller built by another code path need not carry a full identity, and a + # NoMethodError here would refuse the write outright under `critical: true`. + def identity_of(caller) + IDENTITY.transform_values { |reader| caller.respond_to?(reader) ? caller.public_send(reader) : nil } + end # Same id for every change made within one request — set on the caller by the agent (see # CallerParser), mirroring the Node agent's caller.requestId. @@ -26,14 +40,14 @@ def now Time.now.utc.iso8601(3) end - # Auditing must never take the request down with it. By the time a change is recorded the write has - # already happened, so raising would report a failure for an operation that succeeded (and invite a - # retry that duplicates it); a snapshot read failing must not block the write either. Losing the row - # is the lesser evil, so it is logged and dropped. + # Everything after the pending insert is best-effort: by then the write has happened, so raising would + # report a failure for an operation that succeeded (and invite a retry that duplicates it). Losing the + # row is the lesser evil, so it is logged and dropped. Only the pending insert itself can refuse an + # operation, and only under `critical: true` — see {AuditTrail.critical?}. def audit_safely yield rescue StandardError => e - Facades::Container.logger.log('Error', "[ForestAdmin] Audit trail unavailable, skipping: #{e.message}") + AuditTrail.log_failure(e) nil end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb new file mode 100644 index 000000000..68c93950f --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb @@ -0,0 +1,60 @@ +module ForestAdminAgent + module AuditTrail + # What a "before" hook leaves for the matching "after" hook: the records as they stood, the patch, and the + # ids of the pending rows to confirm. + # + # Both hooks bracket one operation on one thread, so a LIFO stack pairs them without relying on the filter + # object reaching both unchanged. An operation raising in between strands its entry, hence the cap. + class Snapshots + include Recording + + # ponytail: 16 deep is far past any legitimate nesting; raise it if one ever gets that far. + MAX_PENDING = 16 + + def push(snapshot) + stack = pending + stack.shift while stack.size >= MAX_PENDING + stack.push(snapshot) + end + + def pop + pending.pop + end + + # The records an operation is about to touch, capped. Reading "delete all" unbounded would materialise + # every matched row — and the pending/confirm protocol writes each of them twice. Truncation is logged + # rather than silent: an incomplete audit somebody knows about beats an OOM. + # + # Read outside the write's transaction — hooks bracket the write as separate calls and the data layer + # exposes no lock, on purpose, since it spans ActiveRecord, Mongoid, HTTP APIs. So two updates racing on + # one record both snapshot the same state, and the one that lands second records a `previous_values` that + # was already overwritten. + # + # An empty list on failure rather than no snapshot at all: the after hook pops unconditionally, so + # skipping the push would pair it with an unrelated entry. + def take(context, projection) + cap = AuditTrail::MAX_RECORDS_PER_OPERATION + page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 0, limit: cap + 1) + records = audit_safely { context.collection.list(context.filter.override(page: page), projection) } || [] + return records if records.size <= cap + + kept = records.first(cap) + AuditTrail.log_truncation(kept.size, audit_safely { count_matching(context) }) + + kept + end + + private + + def count_matching(context) + aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') + + context.collection.aggregate(context.filter, aggregation).first&.fetch('value', nil) + end + + def pending + Thread.current[:forest_audit_trail_snapshots] ||= [] + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb index 9005f1c40..061ea9047 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb @@ -13,6 +13,28 @@ module Sql # eagerly and blows up before any connection is established. class AuditConnectionBase < ActiveRecord::Base self.abstract_class = true + + class << self + # `establish_connection` is class-level, so two stores pointed at different databases would silently + # clobber each other's pool and both end up writing to whichever connected last. One audit database + # per agent is the supported shape; a second, different one is a configuration mistake worth hearing + # about at boot. + def connect_to(database) + if @database && @database != database + raise ForestAdminDatasourceToolkit::Exceptions::ForestException, + 'The audit trail is already connected to another database. One agent, one audit database.' + end + return if @database + + establish_connection(database) + @database = database + end + + def disconnect! + @database = nil + remove_connection + end + end end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb index 243192c1e..1a09a7c14 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb @@ -31,6 +31,50 @@ module Migrations connection.add_index(table, :correlation_key, name: "#{base}_correlation_key", if_not_exists: true) connection.add_index(table, :user_id, name: "#{base}_user_id", if_not_exists: true) end + }, + { + # Denormalised from the caller at write time: who acted then, not whoever holds that id today. + name: '003-add-user-identity', + up: lambda do |connection, table| + connection.add_column(table, :user_first_name, :text, if_not_exists: true) + connection.add_column(table, :user_last_name, :text, if_not_exists: true) + connection.add_column(table, :user_email, :text, if_not_exists: true) + end + }, + { + name: '004-add-action-name', + up: lambda do |connection, table| + connection.add_column(table, :action_name, :text, if_not_exists: true) + end + }, + { + # The default is what backfills the rows written before the pending/confirm protocol existed. + name: '005-add-status', + up: lambda do |connection, table| + connection.add_column(table, :status, :string, null: false, default: 'done', if_not_exists: true) + end + }, + { + # A create's pending row has no id yet, and a packed composite id outgrows a VARCHAR(255). + name: '006-record-id-nullable-text', + up: lambda do |connection, table| + base = table.split('.').last + mysql = connection.adapter_name.downcase.match?(/mysql|maria/) + + # MySQL refuses to widen an indexed column to TEXT, so the index goes first. SQLite rebuilds + # the whole table to change a column, dropping every index 002 created along with it — which is + # why all three are (re)created below rather than just the one dropped here. + connection.remove_index(table, name: "#{base}_record_id", if_exists: true) + connection.change_column(table, :record_id, :text, null: true) + + # ... and MySQL cannot index unbounded TEXT at all: that one needs a length prefix. + record_id_index = { name: "#{base}_record_id", if_not_exists: true } + record_id_index[:length] = 255 if mysql + + connection.add_index(table, :record_id, **record_id_index) + connection.add_index(table, :correlation_key, name: "#{base}_correlation_key", if_not_exists: true) + connection.add_index(table, :user_id, name: "#{base}_user_id", if_not_exists: true) + end } ].freeze end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb index 728b0df96..7e1c07382 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb @@ -4,13 +4,16 @@ module ForestAdminAgent module AuditTrail # SQL-backed storage that both writes every audited change and reads the per-record history back. # - # Construction is cheap; the connection is opened lazily on the first append or read, at which - # point the `forest` schema and the `audit_logs` table are created/evolved through migrations. + # {#connect!} opens the connection and migrates; the agent factory calls it at boot rather than leaving it + # to the first write, so a database the agent cannot reach is a startup failure instead of an agent that + # looks healthy while recording nothing — and, under `critical: true`, instead of one that refuses every + # write the moment somebody first tries to save something. class Store DEFAULT_SCHEMA = 'forest'.freeze DEFAULT_TABLE = 'audit_logs'.freeze - COLUMNS = %i[timestamp operation collection record_id user_id correlation_key - previous_values new_values].freeze + COLUMNS = %i[timestamp operation collection record_id status user_id user_first_name user_last_name + user_email action_name correlation_key previous_values new_values].freeze + AUTHOR_COLUMNS = %i[user_id user_first_name user_last_name user_email].freeze def initialize(database:, schema: DEFAULT_SCHEMA, table_name: DEFAULT_TABLE) @database = database @@ -20,8 +23,39 @@ def initialize(database:, schema: DEFAULT_SCHEMA, table_name: DEFAULT_TABLE) @ready = false end + def connect! + ensure_ready + + self + end + def append(record) - model.create!(to_row(record)) + append_all([record]).first + end + + # Inserts rows and returns their ids, in the order given. Batched, because a "delete all" snapshot can + # be thousands of records and the pending/confirm protocol writes each of them twice. + def append_all(records) + return [] if records.empty? + + rows = records.map { |record| to_row(record) } + + return model.insert_all(rows, returning: %w[id]).rows.flatten if model.connection.supports_insert_returning? + + # MySQL has no RETURNING: fall back to one insert per row, which does hand the id back. + rows.map { |row| model.create!(row).id } + end + + def confirm(id, attributes) + row = model.find_by(id: id) + + row&.update!(**attributes, status: Recording::DONE) + end + + # A write that turned out to change nothing leaves no trace: the pending row goes rather than sitting + # there implying the write is unaccounted for. + def discard(ids) + model.where(id: ids).delete_all unless ids.empty? end def list_by_record(collection:, record_id:, skip: 0, limit: nil, user_ids: nil, @@ -41,6 +75,19 @@ def count_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil scope(collection, record_id, user_ids, start_timestamp, end_timestamp, fields).count end + # The distinct authors of the entries the current filters match, whatever page is being asked for. The + # identity comes from the rows themselves, so a user who has since been renamed or removed still reads + # as they were when they acted. + def authors_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil, + end_timestamp: nil, fields: nil) + scope(collection, record_id, user_ids, start_timestamp, end_timestamp, fields) + .where.not(user_id: nil) + .distinct + .pluck(*AUTHOR_COLUMNS) + .map { |values| AUTHOR_COLUMNS.zip(values).to_h } + .uniq { |author| author[:user_id] } + end + # Entries recorded strictly after `timestamp`, newest first: what a state reconstruction has to undo. # Strictly after, so an entry stamped exactly at the requested instant counts as part of that state # instead of being reverted out of it. @@ -91,7 +138,7 @@ def ensure_ready @mutex.synchronize do return if @ready - Sql::AuditConnectionBase.establish_connection(@database) + Sql::AuditConnectionBase.connect_to(@database) connection = Sql::AuditConnectionBase.connection Sql::Migrator.new(connection, schema: schema_for(connection), table_name: @table_name).run @model = build_model(qualified(connection)) @@ -122,6 +169,7 @@ def to_row(record) def from_row(row) values = COLUMNS.to_h { |column| [column, row[column]] } + values[:id] = row.id values[:timestamp] = row.timestamp.respond_to?(:iso8601) ? row.timestamp.iso8601(3) : row.timestamp.to_s values[:previous_values] ||= {} values[:new_values] ||= {} diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/builder/agent_factory.rb b/packages/forest_admin_agent/lib/forest_admin_agent/builder/agent_factory.rb index 8099232f4..4fca40d36 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/builder/agent_factory.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/builder/agent_factory.rb @@ -297,14 +297,15 @@ def build_cache configure_rpc_polling_pool if @options[:rpc_max_polling_threads] end - # The audit trail switches on as soon as a database is configured. The store is built here - # (cheap: the connection only opens on the first write) and shared through the config, so the - # record-history routes read the very table the capture layer writes to. + # The audit trail switches on as soon as a database is configured. The store connects and migrates here, + # at boot, rather than on the first write: an audit database the agent cannot reach should stop it + # starting, not leave it looking healthy while recording nothing — and under `critical: true` it would + # otherwise refuse every write from the moment somebody first tried to save something. def build_audit_trail_store options = @options[:audit_trail] return unless options && options[:database] - options[:store] = AuditTrail::Store.new(**options.slice(:database, :schema, :table_name).compact) + options[:store] = AuditTrail::Store.new(**options.slice(:database, :schema, :table_name).compact).connect! end def install_audit_trail diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb index 0d99f6dfc..0942c34d6 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb @@ -117,48 +117,74 @@ def handle_hook_request(args = {}) private - # A failed run is worth recording too: "who tried to run this" is usually the interesting part. An - # action that answers with an Error result failed just as much as one that raised — it simply chose - # to say so through `result_builder.error` instead. - def execute_and_audit(context, args, data, filter) - result = context.collection.execute(context.caller, @action_name, data, filter) - audit_action(context, args, data, result: result, failed: error_result?(result)) + # Recorded as pending before the action runs and confirmed after, so an action that takes the process + # down with it still leaves evidence that it started. A failed run is worth recording too — "who tried + # to run this" is usually the interesting part — and an action answering with an Error result failed + # just as much as one that raised, it simply said so through `result_builder.error`. + def execute_and_audit(context, _args, data, filter) + pending = audit_pending(context, data, filter) + + begin + result = context.collection.execute(context.caller, @action_name, data, filter) + rescue StandardError + audit_confirm(pending, failed: true) + raise + end + + audit_confirm(pending, result: result, failed: error_result?(result)) result - rescue StandardError - audit_action(context, args, data, failed: true) - raise end def error_result?(result) result.is_a?(Hash) && result[:type] == 'Error' end - def audit_action(context, args, data, result: nil, failed: false) - capture = ForestAdminAgent::AuditTrail::ActionCapture.new( - ForestAdminAgent::AuditTrail.store, ForestAdminAgent::AuditTrail.options[:redact] - ) + # Everything audit-related sits inside the gate, the record selection included: without an audit + # database none of it runs at all, and a failure refuses the action only under `critical: true` — which + # is safe here, since the action has not run yet. + def audit_pending(context, data, filter) + store = ForestAdminAgent::AuditTrail.store + return [] unless store + + ForestAdminAgent::AuditTrail.gate do + action_capture(store).pending( + caller: context.caller, + collection: context.collection.name, + action_name: @action_name, + form_values: data, + record_ids: audited_record_ids(context, filter) + ) + end || [] + end - capture.record( - caller: context.caller, - collection: context.collection.name, - form_values: data, - record_ids: audited_record_ids(args, context), - result: result, - failed: failed - ) + def audit_confirm(pending, result: nil, failed: false) + store = ForestAdminAgent::AuditTrail.store + + action_capture(store).confirm(pending, result: result, failed: failed) if store && pending.any? + end + + def action_capture(store) + ForestAdminAgent::AuditTrail::ActionCapture.new(store, ForestAdminAgent::AuditTrail.options[:redact]) end - # Packed ids, the form the audit store keys on. A global action targets nothing, and a select-all - # selection only tells us which ids were *excluded*: naming the targets would mean querying the - # whole selection, so those runs are recorded once, attached to no record. - def audited_record_ids(args, context) + # Packed ids, the form the audit store keys on — read back through the caller's own filter rather than + # taken from the request. The ids a client sends are a claim: in a compliance record, asserting that an + # operator acted on a record their scope excludes is worse than a missing row. A global action targets + # no record, and a selection wider than the cap is recorded as one row attached to none. + def audited_record_ids(context, filter) return [] if context.collection.schema[:actions][@action_name].scope == Types::ActionScope::GLOBAL - selection = Utils::Id.parse_selection_ids(context.collection, args[:params], with_key: true) - return [] if selection[:are_excluded] + cap = ForestAdminAgent::AuditTrail::MAX_RECORDS_PER_OPERATION + primary_keys = ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(context.collection) + records = context.collection.list( + context.caller, filter.override(page: Page.new(offset: 0, limit: cap + 1)), Projection.new(primary_keys) + ) + + return records.map { |record| Utils::Id.pack_id(context.collection, record) } if records.size <= cap - selection[:ids].map { |record| Utils::Id.pack_id(context.collection, record) } + ForestAdminAgent::AuditTrail.log_truncation(0, nil) + [] end def middleware_custom_action_approval_request_data(args) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb index 36a47b36a..dafa4d137 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb @@ -55,7 +55,7 @@ def handle_request(args = {}) { name: args[:params]['collection_name'], - content: { data: history.map { |record| serialize_record(record) }, meta: { count: count } } + content: { data: history.map { |record| serialize_record(record) }, meta: meta(args, filters, count) } } end @@ -67,7 +67,7 @@ def handle_state(args = {}) context.permissions.can?(:read, context.collection) # Authorizes and reads in one query: the record it hands back is the one the scope covered. current = scoped_record( - context, context.collection, args[:params]['id'], column_projection(context.collection) + context, context.collection, args[:params]['id'], audited_projection(context.collection) ) timestamp = parse_state_timestamp(args) @@ -82,6 +82,28 @@ def handle_state(args = {}) private + # `availableUsers` rides along on the first fetch only — the front keeps the list it saw — and lists the + # distinct authors of the entries the current filters match, whatever page was asked for. The identity + # comes from the rows, so someone since renamed or removed still reads as they were when they acted. + def meta(args, filters, count) + return { count: count } unless first_fetch?(args) + + { count: count, availableUsers: available_users(filters) } + end + + def first_fetch?(args) + page = args.dig(:params, 'page') + + (page.is_a?(Hash) ? page['number'].to_i : 0) <= 1 + end + + def available_users(filters) + store.authors_by_record(**filters).map do |author| + { id: author[:user_id], firstName: author[:user_first_name], + lastName: author[:user_last_name], email: author[:user_email] } + end + end + # An ISO-8601 instant, or the same wall-clock forms the history filters accept, read in the request # timezone. def parse_state_timestamp(args) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb index 516a53e5b..6e09af5e9 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb @@ -37,7 +37,9 @@ def scoped_record(context, collection, packed_id, projection = nil) raise Http::Exceptions::NotFoundError, 'Record does not exists' end - # Camelize only the top-level keys; previous/new value hashes keep the record's own column names. + # Camelize only the top-level keys — the row `id` included, which the front uses as the tiebreaker when + # merging pages ordered by (timestamp, id). Value hashes keep the keys they were stored with: a record's + # own column names, or an action answer's camelCase Forest names. def serialize_record(record) record.to_h.transform_keys { |key| key.to_s.camelize(:lower) } end @@ -46,9 +48,15 @@ def store ::ForestAdminAgent::AuditTrail.store end - # Every column, the starting point of a state reconstruction. - def column_projection(collection) - Projection.new(collection.schema[:fields].select { |_name, field| field.type == 'Column' }.keys) + # What the audit trail actually records: primary keys, so a state can be identified, plus the writable + # columns. Reading read-only ones would hand them back at their present value inside an answer that + # claims to describe a past instant. + def audited_projection(collection) + writable = collection.schema[:fields].select do |_name, field| + field.type == 'Column' && !field.is_read_only + end.keys + + Projection.new((ForestAdminDatasourceToolkit::Utils::Schema.primary_keys(collection) + writable).uniq) end def key_projection(collection) diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb index 66563f7c2..ad74c8612 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb @@ -3,6 +3,7 @@ module ForestAdminAgent module AuditTrail describe ActionCapture do + # Rows keyed by position, which stands in for the row id the store hands back. let(:store) do Class.new do attr_reader :records @@ -11,125 +12,162 @@ def initialize @records = [] end - def append(record) - @records << record + def append_all(rows) + first = @records.size + @records.concat(rows) + + (first...@records.size).to_a + end + + def confirm(id, attributes) + attributes.each { |key, value| @records[id][key] = value } + @records[id][:status] = ForestAdminAgent::AuditTrail::Recording::DONE end end.new end - let(:caller_double) { double('caller', id: 42, request_id: 'req-xyz') } + let(:caller_double) { double('caller', id: 42, first_name: 'Ada', last_name: 'L', email: 'ada@test') } + let(:capture) { described_class.new(store) } + + def invocation(**over) + { caller: caller_double, collection: 'orders', action_name: 'Refund', + form_values: { 'amount' => 30, 'reason' => 'damaged' }, record_ids: ['4'] }.merge(over) + end - def record(capture: described_class.new(store), **over) - invocation = { caller: caller_double, collection: 'orders', - form_values: { 'amount' => 30, 'reason' => 'damaged' }, record_ids: ['4'] } - invocation.update(over) + def run(capture: described_class.new(store), result: { type: 'Success', message: 'Refunded' }, + failed: false, **over) + ids = capture.pending(**invocation(**over)) + capture.confirm(ids, result: result, failed: failed) - capture.record(**invocation) store.records end - it 'records the invocation against the selected record' do - audit = record.last + describe '#pending' do + it 'records the run before it happens, with the form on the previous side' do + capture.pending(**invocation) + + row = store.records.last + expect(row.status).to eq(Recording::PENDING) + expect(row.operation).to eq(described_class::EXECUTED) + expect(row.action_name).to eq('Refund') + expect(row.collection).to eq('orders') + expect(row.record_id).to eq('4') + expect(row.previous_values).to eq({ 'amount' => 30, 'reason' => 'damaged' }) + expect(row.new_values).to eq({}) + end - expect(audit.operation).to eq('action') - expect(audit.collection).to eq('orders') - expect(audit.record_id).to eq('4') - expect(audit.user_id).to eq(42) - expect(audit.correlation_key).to eq('req-xyz') - expect(audit.previous_values).to eq({ 'amount' => 30, 'reason' => 'damaged' }) - expect(audit.new_values).to eq({}) - end + it 'denormalises who acted, so a later rename does not rewrite history' do + capture.pending(**invocation) - it 'writes one row per selected record, sharing timestamp and correlation key' do - records = record(record_ids: %w[4 7 9]) + row = store.records.last + expect(row.user_id).to eq(42) + expect(row.user_first_name).to eq('Ada') + expect(row.user_last_name).to eq('L') + expect(row.user_email).to eq('ada@test') + end - expect(records.map(&:record_id)).to eq(%w[4 7 9]) - expect(records.map(&:timestamp).uniq.size).to eq(1) - expect(records.map(&:correlation_key).uniq).to eq(['req-xyz']) - end + it 'copes with a caller carrying no identity at all' do + capture.pending(**invocation(caller: double('caller'))) - # Global actions and select-all selections name no record, but the run still has to leave a trace. - it 'records a run attached to no record when the selection is unknown' do - records = record(record_ids: []) + expect(store.records.last.user_email).to be_nil + end - expect(records.size).to eq(1) - expect(records.last.record_id).to eq('') - end + it 'writes one row per targeted record, sharing timestamp and correlation key' do + ids = capture.pending(**invocation(record_ids: %w[4 7 9])) - it 'marks a failed run with its own operation' do - expect(record(failed: true).last.operation).to eq('action_failed') - end + expect(ids.size).to eq(3) + expect(store.records.map(&:record_id)).to eq(%w[4 7 9]) + expect(store.records.map(&:timestamp).uniq.size).to eq(1) + end - it 'masks redacted fields of the form, keeping the rest' do - capture = described_class.new(store, { 'orders' => ['reason'] }) + it 'records a run attached to no record when no target can be named' do + capture.pending(**invocation(record_ids: [])) - audit = record(capture: capture).last + expect(store.records.map(&:record_id)).to eq([described_class::NO_RECORD]) + end - expect(audit.previous_values).to eq({ 'amount' => 30, 'reason' => Recording::REDACTED }) - end + it 'masks redacted form fields, keeping the rest' do + described_class.new(store, { 'orders' => ['reason'] }).pending(**invocation) - describe 'the action answer' do - it 'keeps the type and the message of a success' do - audit = record(result: { response_headers: {}, type: 'Success', message: 'Refunded', invalidated: [] }).last + expect(store.records.last.previous_values).to eq({ 'amount' => 30, 'reason' => Recording::REDACTED }) + end - expect(audit.new_values).to eq({ 'type' => 'Success', 'message' => 'Refunded' }) + it 'does nothing without a configured store' do + expect(described_class.new(nil).pending(**invocation)).to eq([]) end + end - it 'keeps the type and the message of an error' do - audit = record(result: { type: 'Error', message: 'not allowed', html: nil }, failed: true).last + describe '#confirm' do + it 'settles the row and keeps what the action answered' do + row = run.last - expect(audit.new_values).to eq({ 'type' => 'Error', 'message' => 'not allowed' }) + expect(row.status).to eq(Recording::DONE) + expect(row.operation).to eq(described_class::EXECUTED) + expect(row.new_values).to eq({ 'type' => 'Success', 'message' => 'Refunded' }) end - # A file result carries the whole file; a webhook carries its body and headers, which is where - # credentials live. None of that belongs in an audit table. - it 'never keeps a file stream, nor a webhook body or headers' do - file = record(result: { type: 'File', name: 'refunds.csv', mime_type: 'text/csv', - stream: 'id,amount\n1,30' }).last - webhook = record(result: { type: 'Webhook', url: 'https://pay.test/refund', method: 'POST', - body: { 'secret' => 'x' }, headers: { 'Authorization' => 'Bearer t' } }).last + it 'marks a failed run with its own operation' do + row = run(result: { type: 'Error', message: 'not allowed' }, failed: true).last - expect(file.new_values).to eq({ 'type' => 'File', 'name' => 'refunds.csv', 'mime_type' => 'text/csv' }) - expect(webhook.new_values).to eq({ 'type' => 'Webhook', 'url' => 'https://pay.test/refund', - 'method' => 'POST' }) + expect(row.operation).to eq(described_class::FAILED) + expect(row.new_values).to eq({ 'type' => 'Error', 'message' => 'not allowed' }) + end + + it 'leaves the answer empty when the action raised' do + expect(run(result: nil, failed: true).last.new_values).to eq({}) end - it 'drops the response headers of any result' do - audit = record(result: { response_headers: { 'Set-Cookie' => 'session=x' }, type: 'Success' }).last + # The row already says an action started; losing its answer beats reporting a failure for a run that + # went through. + it 'logs and swallows a store failure' do + logger = instance_spy(Services::LoggerService) + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) + ids = capture.pending(**invocation) + allow(store).to receive(:confirm).and_raise(StandardError, 'audit db is down') - expect(audit.new_values).to eq({ 'type' => 'Success' }) + expect { capture.confirm(ids, result: {}) }.not_to raise_error + expect(logger).to have_received(:log).with('Error', /audit db is down/) end + end - it 'keeps where a redirect pointed' do - audit = record(result: { type: 'Redirect', path: '/orders/4' }).last + describe 'the answer it keeps' do + # Forest's own keys, so camelCase on the wire — unlike a record's column names, which pass through. + it 'camelCases the keys it keeps' do + row = run(result: { type: 'File', name: 'refunds.csv', mime_type: 'text/csv' }).last - expect(audit.new_values).to eq({ 'type' => 'Redirect', 'path' => '/orders/4' }) + expect(row.new_values).to eq({ 'type' => 'File', 'name' => 'refunds.csv', 'mimeType' => 'text/csv' }) end - it 'stores no answer at all when the action raised' do - expect(record(failed: true).last.new_values).to eq({}) + it 'never keeps a file stream, a webhook body or headers, nor response headers' do + file = run(result: { type: 'File', name: 'r.csv', stream: 'id,amount' }).last + webhook = run(result: { type: 'Webhook', url: 'https://pay.test/refund', method: 'POST', + body: { 'secret' => 'x' }, headers: { 'Authorization' => 'Bearer t' }, + response_headers: { 'Set-Cookie' => 'session=x' } }).last + + expect(file.new_values).to eq({ 'type' => 'File', 'name' => 'r.csv' }) + expect(webhook.new_values).to eq({ 'type' => 'Webhook', 'url' => 'https://pay.test/refund', + 'method' => 'POST' }) end - end - it 'does nothing without a configured store' do - expect { record(capture: described_class.new(nil)) }.not_to raise_error - end + # A signed one-time token or a password in the userinfo would otherwise sit in the one table nobody + # deletes from. + it 'strips credentials and query tokens off a webhook url' do + row = run(result: { type: 'Webhook', url: 'https://user:pass@pay.test/refund?token=abc#frag' }).last + + expect(row.new_values['url']).to eq('https://pay.test/refund') + end - # The action already ran: reporting a failure for it because the audit database is down would be - # worse than losing the row. - it 'logs and swallows a store failure instead of breaking the action' do - logger = instance_spy(Services::LoggerService) - allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) - allow(store).to receive(:append).and_raise(StandardError, 'audit db is down') + it 'strips a query token off a redirect path' do + row = run(result: { type: 'Redirect', path: '/orders/4?signature=abc' }).last - expect { record }.not_to raise_error - expect(logger).to have_received(:log).with('Error', /audit db is down/) - end + expect(row.new_values['path']).to eq('/orders/4') + end - it 'falls back to a generated correlation key when the caller carries none' do - audit = record(caller: double('caller', id: 42, request_id: nil)).last + it 'still sanitises a url the parser refuses' do + row = run(result: { type: 'Webhook', url: 'https://user:pass@pay test/refund?token=abc' }).last - expect(audit.correlation_key).to match(/\A[0-9a-f-]{36}\z/) + expect(row.new_values['url']).to eq('https://pay test/refund') + end end end end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb index 841707bb1..7a173bdc4 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb @@ -5,16 +5,30 @@ module AuditTrail describe Capture do let(:column_schema) { ForestAdminDatasourceToolkit::Schema::ColumnSchema } + # Rows keyed by position, which stands in for the row id the store hands back. let(:store) do Class.new do - attr_reader :records + attr_reader :records, :discarded def initialize @records = [] + @discarded = [] end - def append(record) - @records << record + def append_all(rows) + first = @records.size + @records.concat(rows) + + (first...@records.size).to_a + end + + def confirm(id, attributes) + attributes.each { |key, value| @records[id][key] = value } + @records[id][:status] = ForestAdminAgent::AuditTrail::Recording::DONE + end + + def discard(ids) + @discarded.concat(ids) end end.new end @@ -28,8 +42,11 @@ def append(record) end let(:hooks) { {} } + let(:registrations) { [] } let(:collection) { double('collection') } - let(:caller_double) { double('caller', id: 42, request_id: 'req-xyz') } + let(:caller_double) do + double('caller', id: 42, first_name: 'Ada', last_name: 'L', email: 'ada@test', request_id: 'req-xyz') + end # What a hook context really hands out: the caller is already bound, so `list` takes # (filter, projection). A verifying double keeps that contract honest. @@ -37,7 +54,6 @@ def append(record) instance_double(ForestAdminDatasourceCustomizer::Context::RelaxedWrappers::RelaxedCollection) end - let(:registrations) { [] } let(:collection_customizer) do customizer = double('CollectionCustomizer', name: 'companies', collection: collection) allow(customizer).to receive(:add_hook) do |position, type, prepend: false, &block| @@ -57,151 +73,213 @@ def append(record) described_class.new.run(datasource_customizer, nil, store: store) end - # execute_after stops at the first exception, so being last would mean losing the record of a write - # that already happened whenever another customization raises in its own after hook. + def filter + ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: nil) + end + + def before_hook(type, patch: nil, data: nil) + hooks["Before_#{type}"].call( + double('before', caller: caller_double, filter: filter, collection: relaxed_collection, + patch: patch, data: data) + ) + end + + def after_hook(type, record: nil) + hooks["After_#{type}"].call( + double('after', caller: caller_double, filter: filter, collection: relaxed_collection, record: record) + ) + end + + # execute_after stops at the first exception, so being last would mean losing the record of a write that + # already happened whenever another customization raises in its own after hook. it 'registers its after hooks ahead of the ones already there, and its before hooks after them' do expect(registrations).to contain_exactly( - ['After', 'Create', true], ['Before', 'Update', false], ['After', 'Update', true], + ['Before', 'Create', false], ['After', 'Create', true], + ['Before', 'Update', false], ['After', 'Update', true], ['Before', 'Delete', false], ['After', 'Delete', true] ) end - it 'records a create with only the writable columns' do - record = { 'id' => 1, 'name' => 'Acme', 'address' => { 'city' => 'Paris' } } + describe 'creating a record' do + it 'records the attempt before the write, with no id yet' do + before_hook('Create', data: { 'name' => 'Acme' }) - hooks['After_Create'].call(double('ctx', caller: caller_double, record: record)) + row = store.records.last + expect(row.status).to eq(Recording::PENDING) + expect(row.operation).to eq('create') + expect(row.record_id).to be_nil + expect(row.new_values).to eq({ 'name' => 'Acme', 'address' => nil }) + end - audit = store.records.last - expect(audit.operation).to eq('create') - expect(audit.record_id).to eq('1') - expect(audit.user_id).to eq(42) - expect(audit.correlation_key).to eq('req-xyz') - expect(audit.previous_values).to eq({}) - expect(audit.new_values).to eq({ 'name' => 'Acme', 'address' => { 'city' => 'Paris' } }) - end + it 'confirms it with the id and the record that landed' do + before_hook('Create', data: { 'name' => 'Acme' }) + after_hook('Create', record: { 'id' => 1, 'name' => 'Acme', 'address' => { 'city' => 'Paris' } }) - def before_hook(type, patch: nil) - hooks["Before_#{type}"].call( - double('before', caller: caller_double, filter: Object.new, - collection: relaxed_collection, patch: patch) - ) - end + row = store.records.last + expect(row.status).to eq(Recording::DONE) + expect(row.record_id).to eq('1') + expect(row.new_values).to eq({ 'name' => 'Acme', 'address' => { 'city' => 'Paris' } }) + end + + it 'denormalises who acted' do + before_hook('Create', data: { 'name' => 'Acme' }) - def after_hook(type) - hooks["After_#{type}"].call(double('after', caller: caller_double, filter: Object.new)) + expect(store.records.last.user_email).to eq('ada@test') + expect(store.records.last.correlation_key).to eq('req-xyz') + end end - it "shares the caller's request id as the correlation key across records of one operation" do - allow(relaxed_collection).to receive(:list).and_return( - [{ 'id' => 1, 'name' => 'A' }, { 'id' => 2, 'name' => 'B' }] - ) + describe 'updating records' do + def update(before:, persisted:, patch:) + allow(relaxed_collection).to receive(:list).and_return(before, persisted) + before_hook('Update', patch: patch) + after_hook('Update') - before_hook('Update', patch: { 'name' => 'Z' }) - after_hook('Update') + store.records + end - expect(store.records.map(&:correlation_key)).to eq(%w[req-xyz req-xyz]) - end + it 'records one pending row per matched record before the write' do + allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'Acme' }, + { 'id' => 2, 'name' => 'Other' }]) + before_hook('Update', patch: { 'name' => 'Z' }) - it 'records an update with the minimal nested diff' do - before_record = { 'id' => 1, 'name' => 'Acme', 'address' => { 'city' => 'Paris', 'zip' => '1' } } - allow(relaxed_collection).to receive(:list).and_return([before_record]) + expect(store.records.map(&:record_id)).to eq(%w[1 2]) + expect(store.records.map(&:status).uniq).to eq([Recording::PENDING]) + end - before_hook('Update', patch: { 'address' => { 'city' => 'Lyon', 'zip' => '1' } }) - after_hook('Update') + # The diff is taken against the record as persisted, so normalisation and decorator side effects are + # what gets recorded — not merely what was asked for. + it 'diffs against the record as persisted, not the requested patch' do + rows = update(before: [{ 'id' => 1, 'name' => 'acme' }], + persisted: [{ 'id' => 1, 'name' => 'ACME NORMALISED' }], + patch: { 'name' => 'Acme' }) - audit = store.records.last - expect(audit.operation).to eq('update') - expect(audit.previous_values).to eq({ 'address' => { 'city' => 'Paris' } }) - expect(audit.new_values).to eq({ 'address' => { 'city' => 'Lyon' } }) - end + expect(rows.last.status).to eq(Recording::DONE) + expect(rows.last.previous_values).to eq({ 'name' => 'acme' }) + expect(rows.last.new_values).to eq({ 'name' => 'ACME NORMALISED' }) + end - it 'does not record an update when nothing writable changed' do - allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'Acme' }]) + # Otherwise the row would be filed under an id History never queries. + context 'when the primary key itself is writable' do + let(:fields) do + { + 'id' => column_schema.new(column_type: 'Number', is_primary_key: true), + 'name' => column_schema.new(column_type: 'String') + } + end - before_hook('Update', patch: { 'name' => 'Acme' }) - after_hook('Update') + it 'files the row under the id the record ended up with' do + rows = update(before: [{ 'id' => 1, 'name' => 'Acme' }], + persisted: [{ 'id' => 7, 'name' => 'Acme' }], + patch: { 'id' => 7 }) - expect(store.records).to be_empty - end + expect(rows.last.record_id).to eq('7') + expect(rows.last.new_values).to eq({ 'id' => 7 }) + end + end - # The before and after hooks are handed different filter objects (the after context always - # carries the caller's original), so pairing must not depend on that object. - it 'pairs the snapshot with its after hook without relying on the filter object' do - allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'Acme' }]) + it 'falls back to the patch when the record cannot be read back' do + rows = update(before: [{ 'id' => 1, 'name' => 'Acme' }], persisted: [], patch: { 'name' => 'Z' }) - before_hook('Update', patch: { 'name' => 'Effective' }) - after_hook('Update') + expect(rows.last.new_values).to eq({ 'name' => 'Z' }) + end + + # Nothing changed, so nothing is audited: the pending row goes rather than sitting there implying the + # write is unaccounted for. + it 'discards the pending row when the write changed nothing' do + update(before: [{ 'id' => 1, 'name' => 'Acme' }], + persisted: [{ 'id' => 1, 'name' => 'Acme' }], + patch: { 'name' => 'Acme' }) - expect(store.records.last.new_values).to eq({ 'name' => 'Effective' }) + expect(store.discarded).to eq([0]) + end end - it 'still audits the next write after one raised between its hooks' do - allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'Acme' }]) + describe 'deleting records' do + it 'records the rows before the write and settles them after' do + allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 7, 'name' => 'Gone', + 'address' => nil }]) - before_hook('Update', patch: { 'name' => 'Never written' }) - before_hook('Update', patch: { 'name' => 'Z' }) - after_hook('Update') + before_hook('Delete') + expect(store.records.last.status).to eq(Recording::PENDING) - expect(store.records.map { |record| record.new_values['name'] }).to eq(['Z']) + after_hook('Delete') + row = store.records.last + expect(row.status).to eq(Recording::DONE) + expect(row.operation).to eq('delete') + expect(row.previous_values).to eq({ 'name' => 'Gone', 'address' => nil }) + end end - it 'drops the oldest stranded snapshots instead of growing the thread-local without bound' do - allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'Acme' }]) + describe 'when the audit database is unreachable' do + before { allow(store).to receive(:append_all).and_raise(StandardError, 'audit db is down') } - (described_class::MAX_SNAPSHOTS + 4).times { before_hook('Update', patch: { 'name' => 'stranded' }) } - before_hook('Update', patch: { 'name' => 'Z' }) - after_hook('Update') + it 'lets the write through, logging the failure, when critical is off' do + logger = instance_spy(Services::LoggerService) + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) - expect(Thread.current[:forest_audit_trail_snapshots].size).to eq(described_class::MAX_SNAPSHOTS - 1) - expect(store.records.map { |record| record.new_values['name'] }).to eq(['Z']) + expect { before_hook('Create', data: { 'name' => 'Acme' }) }.not_to raise_error + expect(logger).to have_received(:log).with('Error', /audit db is down/) + end + + # Nothing has been written yet, so refusing costs nothing to repair. + it 'refuses the operation when critical is on' do + allow(ForestAdminAgent::AuditTrail).to receive(:critical?).and_return(true) + + expect { before_hook('Create', data: { 'name' => 'Acme' }) } + .to raise_error(StandardError, 'audit db is down') + end end - it 'records a delete with the previous values' do - allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 7, 'name' => 'Gone', 'address' => nil }]) + describe 'a selection wider than the cap' do + let(:cap) { ForestAdminAgent::AuditTrail::MAX_RECORDS_PER_OPERATION } - before_hook('Delete') - after_hook('Delete') + it 'audits up to the cap and says how many it left out' do + logger = instance_spy(Services::LoggerService) + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) + matched = (1..(cap + 1)).map { |id| { 'id' => id, 'name' => "n#{id}" } } + allow(relaxed_collection).to receive_messages(list: matched, aggregate: [{ 'value' => cap + 25 }]) - audit = store.records.last - expect(audit.operation).to eq('delete') - expect(audit.record_id).to eq('7') - expect(audit.previous_values).to eq({ 'name' => 'Gone', 'address' => nil }) - expect(audit.new_values).to eq({}) - end + before_hook('Delete') - # The write already happened when the after hook runs, so a broken audit database must not turn a - # successful change into a client-visible error. - it 'logs and swallows a store failure instead of failing the write' do - logger = instance_spy(Services::LoggerService) - allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) - allow(store).to receive(:append).and_raise(StandardError, 'audit db is down') + expect(store.records.size).to eq(cap) + expect(logger).to have_received(:log).with('Warn', /#{cap} records audited, 25 skipped/) + end - expect do - hooks['After_Create'].call(double('ctx', caller: caller_double, record: { 'id' => 1, 'name' => 'Acme' })) - end.not_to raise_error - expect(logger).to have_received(:log).with('Error', /audit db is down/) - end + it 'asks for one more than the cap, so it can tell it was truncated' do + allow(relaxed_collection).to receive(:list).and_return([]) - it 'does not block the write when the before-hook snapshot cannot be read' do - logger = instance_spy(Services::LoggerService) - allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) - allow(relaxed_collection).to receive(:list).and_raise(StandardError, 'read failed') + before_hook('Delete') - expect do - before_hook('Update', patch: { 'name' => 'Z' }) - after_hook('Update') - end.not_to raise_error - expect(store.records).to be_empty + expect(relaxed_collection).to have_received(:list) do |filter, _projection| + expect(filter.page.limit).to eq(cap + 1) + end + end end it 'masks redacted fields while still recording the change' do described_class.new.run(datasource_customizer, nil, store: store, redact: { 'companies' => ['name'] }) + before_hook('Create', data: { 'name' => 'Secret' }) + + expect(store.records.last.new_values['name']).to eq(Recording::REDACTED) + end - hooks['After_Create'].call( - double('ctx', caller: caller_double, record: { 'id' => 1, 'name' => 'Secret', 'address' => nil }) + # The after hook settles its own operation's rows, not whichever pending row happens to be around. + it 'pairs the after hook with the write that raised between the two, not the stranded one' do + allow(relaxed_collection).to receive(:list).and_return( + [{ 'id' => 1, 'name' => 'stranded' }], + [{ 'id' => 2, 'name' => 'before' }], + [{ 'id' => 2, 'name' => 'after' }] ) - expect(store.records.last.new_values['name']).to eq(described_class::REDACTED) + before_hook('Update', patch: { 'name' => 'never written' }) + before_hook('Update', patch: { 'name' => 'after' }) + after_hook('Update') + + settled = store.records.select { |row| row.status == Recording::DONE } + expect(settled.map(&:record_id)).to eq(['2']) + expect(settled.last.new_values).to eq({ 'name' => 'after' }) + expect(store.records.first.status).to eq(Recording::PENDING) end end end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb index 1632c9932..fe58c7e40 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb @@ -18,6 +18,8 @@ module Sql create_table: nil, add_index: nil, add_column: nil, + remove_index: nil, + change_column: nil, select_values: ['forest.audit_logs:001-create-audit-logs', 'forest.audit_logs:002-index-record-and-correlation'], quote_schema_name: '"forest"', diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb index ba31c2925..b4bcff448 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb @@ -8,13 +8,13 @@ module AuditTrail let(:store) { described_class.new(database: { adapter: 'sqlite3', database: db.path }) } after do - Sql::AuditConnectionBase.remove_connection + Sql::AuditConnectionBase.disconnect! db.close! end def record(over = {}) AuditRecord.new( - operation: 'update', collection: 'accounts', record_id: '1', + operation: 'update', collection: 'accounts', record_id: '1', status: Recording::DONE, previous_values: { 'status' => 'open' }, new_values: { 'status' => 'closed' }, timestamp: '2026-01-02T03:04:05.000Z', user_id: 42, correlation_key: 'req-1', **over ) @@ -32,7 +32,8 @@ def connection store.append(record) expect(store.send(:model).column_names.sort).to eq( - %w[collection correlation_key id new_values operation previous_values record_id timestamp user_id] + %w[action_name collection correlation_key id new_values operation previous_values record_id status + timestamp user_email user_first_name user_id user_last_name] ) end @@ -133,7 +134,9 @@ def connection names = connection.select_values('SELECT name FROM audit_migrations ORDER BY name') expect(names).to eq( - ['audit_logs:001-create-audit-logs', 'audit_logs:002-index-record-and-correlation'] + ['audit_logs:001-create-audit-logs', 'audit_logs:002-index-record-and-correlation', + 'audit_logs:003-add-user-identity', 'audit_logs:004-add-action-name', 'audit_logs:005-add-status', + 'audit_logs:006-record-id-nullable-text'] ) end @@ -219,6 +222,90 @@ def connection end end + describe 'the pending/confirm protocol' do + it 'inserts rows and hands their ids back, in order' do + ids = store.append_all([record(correlation_key: 'a'), record(correlation_key: 'b')]) + + expect(ids.size).to eq(2) + expect(store.list_by_record(collection: 'accounts', record_id: '1').map(&:id)).to eq(ids) + end + + it 'confirms a pending row into a done one, values included' do + id = store.append(record(status: Recording::PENDING, previous_values: {}, new_values: {})) + + store.confirm(id, record_id: '9', new_values: { 'status' => 'closed' }) + + audit = store.list_by_record(collection: 'accounts', record_id: '9').first + expect(audit.status).to eq(Recording::DONE) + expect(audit.new_values).to eq({ 'status' => 'closed' }) + end + + # A write that changed nothing leaves no trace, rather than a row implying it is unaccounted for. + it 'discards rows by id' do + ids = store.append_all([record, record]) + + store.discard(ids) + + expect(store.list_by_record(collection: 'accounts', record_id: '1')).to be_empty + end + + it 'accepts a pending create row with no record id yet' do + id = store.append(record(operation: 'create', record_id: nil, status: Recording::PENDING)) + + expect { store.confirm(id, record_id: '7') }.not_to raise_error + expect(store.list_by_record(collection: 'accounts', record_id: '7').first.operation).to eq('create') + end + + it 'stores a packed composite id far longer than a varchar would hold' do + long_id = (1..40).map { |part| "part-#{part}-#{"x" * 20}" }.join('|') + store.append(record(record_id: long_id)) + + expect(store.list_by_record(collection: 'accounts', record_id: long_id).size).to eq(1) + end + end + + describe '#authors_by_record' do + it 'lists the distinct authors of the matching entries, as they were when they acted' do + store.append(record(user_id: 7, user_first_name: 'Ada', user_last_name: 'L', user_email: 'ada@test')) + store.append(record(user_id: 7, user_first_name: 'Ada', user_last_name: 'L', user_email: 'ada@test')) + store.append(record(user_id: 9, user_first_name: 'Bob', user_last_name: 'K', user_email: 'bob@test')) + store.append(record(record_id: '2', user_id: 11, user_first_name: 'Other')) + + authors = store.authors_by_record(collection: 'accounts', record_id: '1') + + expect(authors).to contain_exactly( + { user_id: 7, user_first_name: 'Ada', user_last_name: 'L', user_email: 'ada@test' }, + { user_id: 9, user_first_name: 'Bob', user_last_name: 'K', user_email: 'bob@test' } + ) + end + + it 'stays inside the active filters but ignores paging' do + store.append(record(user_id: 7, timestamp: '2026-01-02T03:04:05.000Z')) + store.append(record(user_id: 9, timestamp: '2026-01-09T03:04:05.000Z')) + + authors = store.authors_by_record(collection: 'accounts', record_id: '1', + end_timestamp: '2026-01-03T00:00:00.000Z') + + expect(authors.map { |author| author[:user_id] }).to eq([7]) + end + + it 'leaves out entries with no author' do + store.append(record(user_id: nil)) + + expect(store.authors_by_record(collection: 'accounts', record_id: '1')).to be_empty + end + end + + # establish_connection is class-level: two stores on different databases would silently share one pool. + it 'refuses a second audit database rather than clobbering the first' do + store.append(record) + other = described_class.new(database: { adapter: 'sqlite3', database: ':memory:' }) + + expect { other.append(record) }.to raise_error( + ForestAdminDatasourceToolkit::Exceptions::ForestException, /One agent, one audit database/ + ) + end + it 'indexes record_id, correlation_key and user_id' do store.append(record) diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb index 620411e70..7d94078a9 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb @@ -21,29 +21,51 @@ module Action ) }, actions: { 'Refund' => double('action', scope: action_scope) } - } + }, + list: [{ 'id' => 4 }, { 'id' => 7 }] ) end - let(:context) { double('context', collection: collection) } + let(:context) { double('context', collection: collection, caller: build_caller) } + let(:filter) { ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: nil) } - def audited_ids(attributes) - route.send(:audited_record_ids, { params: { data: { attributes: attributes } } }, context) - end + describe 'the records it audits' do + def audited_ids + route.send(:audited_record_ids, context, filter) + end - it 'packs the ids of an explicit selection' do - expect(audited_ids({ ids: %w[4 7] })).to eq(%w[4 7]) - end + # The ids a client sends are a claim. In a compliance record, asserting an operator acted on a record + # their scope excludes is worse than a missing row, so the selection is read back through the caller's + # own filter. + it 'reads the targets back through the caller filter rather than trusting the request' do + expect(audited_ids).to eq(%w[4 7]) + expect(collection).to have_received(:list) do |_caller, listed, projection| + expect(listed.page.limit).to eq(ForestAdminAgent::AuditTrail::MAX_RECORDS_PER_OPERATION + 1) + expect(projection).to eq(['id']) + end + end + + it 'records a selection wider than the cap as attached to no record, and says so' do + logger = instance_spy(Services::LoggerService) + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) + cap = ForestAdminAgent::AuditTrail::MAX_RECORDS_PER_OPERATION + allow(collection).to receive(:list).and_return((1..(cap + 1)).map { |id| { 'id' => id } }) + + expect(audited_ids).to eq([]) + expect(logger).to have_received(:log).with('Warn', /records audited/) + end + + context 'with a global action' do + let(:action_scope) { ForestAdminDatasourceCustomizer::Decorators::Action::Types::ActionScope::GLOBAL } - # A select-all selection only carries the excluded ids: naming the targets would mean querying - # the whole selection, so the run is recorded unattached instead. - it 'returns no id for a select-all selection' do - expect(audited_ids({ ids: ['4'], all_records: true, all_records_ids_excluded: ['9'] })).to eq([]) + it 'names no record, and does not even query' do + expect(audited_ids).to eq([]) + expect(collection).not_to have_received(:list) + end + end end describe 'auditing the run' do - let(:store) { double('store', append: nil) } - let(:context) { double('context', collection: collection, caller: build_caller) } - let(:args) { { params: { data: { attributes: { ids: %w[4 7] } } } } } + let(:store) { double('store', append_all: [10, 11], confirm: nil) } before do allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) @@ -52,69 +74,78 @@ def audited_ids(attributes) def execute(result: { type: 'Success', message: 'done' }, &raising) allow(collection).to receive(:execute, &(raising || proc { result })) - route.send(:execute_and_audit, context, args, { 'amount' => 30, 'reason' => 'damaged' }, nil) + + route.send(:execute_and_audit, context, {}, { 'amount' => 30, 'reason' => 'damaged' }, filter) + end + + # Pending before the action, confirmed after: an action that takes the process down with it still + # leaves evidence that it started. + it 'records the run before it happens, one row per target' do + execute + + expect(store).to have_received(:append_all) do |rows| + expect(rows.map(&:record_id)).to eq(%w[4 7]) + expect(rows.map(&:status).uniq).to eq([ForestAdminAgent::AuditTrail::Recording::PENDING]) + expect(rows.first.action_name).to eq('Refund') + expect(rows.first.previous_values).to eq({ 'amount' => 30, 'reason' => '[redacted]' }) + end end - it 'records one row per selected record and returns the action result' do + it 'confirms both rows with what the action answered, and returns it' do expect(execute).to eq({ type: 'Success', message: 'done' }) - expect(store).to have_received(:append).twice - expect(store).to have_received(:append).with( - having_attributes(operation: 'action', collection: 'orders', record_id: '4', - previous_values: { 'amount' => 30, 'reason' => '[redacted]' }) + expect(store).to have_received(:confirm).with( + 10, hash_including(operation: 'action', new_values: { 'type' => 'Success', 'message' => 'done' }) ) + expect(store).to have_received(:confirm).with(11, any_args) end - it 'records the attempt and re-raises when the action fails' do + it 'confirms as failed and re-raises when the action raises' do expect { execute { raise StandardError, 'boom' } }.to raise_error(StandardError, 'boom') - expect(store).to have_received(:append) - .with(having_attributes(operation: 'action_failed', record_id: '4')).once + expect(store).to have_received(:confirm).with(10, hash_including(operation: 'action_failed')) end - it 'records nothing when no audit database is configured' do - allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache).and_return({}) - - expect(execute).to eq({ type: 'Success', message: 'done' }) - expect(store).not_to have_received(:append) - end - - # An action reporting failure through result_builder.error never raises, so only the result says so. - it 'records what the action answered' do - execute(result: { type: 'Success', message: 'done', response_headers: { 'X-Token' => 'x' } }) + it 'confirms as failed when the action answers with an Error result' do + execute(result: { type: 'Error', message: 'not allowed' }) - expect(store).to have_received(:append).with( - having_attributes(operation: 'action', new_values: { 'type' => 'Success', 'message' => 'done' }) - ).twice + expect(store).to have_received(:confirm).with(10, hash_including(operation: 'action_failed')) end - it 'records no answer when the action raised' do - expect { execute { raise StandardError, 'boom' } }.to raise_error(StandardError) + it 'does nothing at all without an audit database, not even reading the selection' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache).and_return({}) - expect(store).to have_received(:append).with(having_attributes(new_values: {})).twice + expect(execute).to eq({ type: 'Success', message: 'done' }) + expect(collection).not_to have_received(:list) + expect(store).not_to have_received(:append_all) end - it 'records an Error result as a failed run, and still returns it' do - result = execute(result: { type: 'Error', message: 'not allowed' }) + # The action has not run yet, so refusing costs nothing to repair — but only when asked to. + it 'refuses the action when the pending row cannot be written and critical is on' do + allow(ForestAdminAgent::AuditTrail).to receive(:critical?).and_return(true) + allow(store).to receive(:append_all).and_raise(StandardError, 'audit db is down') - expect(result).to eq({ type: 'Error', message: 'not allowed' }) - expect(store).to have_received(:append) - .with(having_attributes(operation: 'action_failed', record_id: '4')).once + expect { execute }.to raise_error(StandardError, 'audit db is down') + expect(collection).not_to have_received(:execute) end - it 'records any other result type as a run that went through' do - execute(result: { type: 'Webhook', url: 'https://example.test' }) + it 'runs the action anyway when the pending row cannot be written and critical is off' do + logger = instance_spy(Services::LoggerService) + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) + allow(store).to receive(:append_all).and_raise(StandardError, 'audit db is down') - expect(store).to have_received(:append) - .with(having_attributes(operation: 'action', record_id: '4')).once + expect(execute).to eq({ type: 'Success', message: 'done' }) + expect(logger).to have_received(:log).with('Error', /audit db is down/) end - end - context 'with a global action' do - let(:action_scope) { ForestAdminDatasourceCustomizer::Decorators::Action::Types::ActionScope::GLOBAL } + # Reading the selection happens inside the gate too, so a failure there cannot 500 an action. + it 'runs the action anyway when the selection cannot be read' do + allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return( + instance_spy(Services::LoggerService) + ) + allow(collection).to receive(:list).and_raise(StandardError, 'datasource down') - it 'returns no id, since a global action targets no record' do - expect(audited_ids({ ids: %w[4 7] })).to eq([]) + expect(execute).to eq({ type: 'Success', message: 'done' }) end end end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb index 2865a6e48..517857089 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -28,7 +28,8 @@ module Resources def route_with_store(records: []) allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) .and_return({ audit_trail: { store: store } }) - allow(store).to receive_messages(list_by_record: records, count_by_record: records.length) + allow(store).to receive_messages(list_by_record: records, count_by_record: records.length, + authors_by_record: []) route = described_class.new context = double('context', collection: collection, caller: build_caller, permissions: permissions) @@ -205,7 +206,7 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) expect(result[:content]).to eq( { data: [{ 'operation' => 'update', 'recordId' => '4', 'previousValues' => { 'first_name' => 'Jo' } }], - meta: { count: 1 } + meta: { count: 1, availableUsers: [] } } ) end @@ -245,6 +246,50 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) expect(result[:content][:data]).to eq([{ 'operation' => 'delete', 'recordId' => '4' }]) end + describe 'meta.availableUsers' do + let(:authors) do + [{ user_id: 12, user_first_name: 'Ada', user_last_name: 'L', user_email: 'ada@test' }] + end + + # The distinct authors of what the filters match, whatever page was asked for, in the shape the + # filter dropdown wants. + it 'lists the authors of the matching entries on the first fetch' do + route = route_with_store + allow(store).to receive(:authors_by_record).and_return(authors) + + result = route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) + + expect(result[:content][:meta][:availableUsers]).to eq( + [{ id: 12, firstName: 'Ada', lastName: 'L', email: 'ada@test' }] + ) + expect(store).to have_received(:authors_by_record).with(collection: 'projects', record_id: '4') + end + + # The front keeps the list it saw, so later pages leave it out. + it 'leaves it out past the first page, and does not even ask for it' do + route = route_with_store + allow(store).to receive(:authors_by_record).and_return(authors) + + result = route.handle_request( + { headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'page' => { 'number' => '2' } } } + ) + + expect(result[:content][:meta]).to eq({ count: 0 }) + expect(store).not_to have_received(:authors_by_record) + end + + it 'answers the active filters, not the whole history' do + route = route_with_store + allow(store).to receive(:authors_by_record).and_return(authors) + + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'userIds' => '12' } }) + + expect(store).to have_received(:authors_by_record).with(hash_including(user_ids: [12])) + end + end + it 'defaults to newest-first and switches to oldest-first on sort=timestamp' do route = route_with_store route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4', 'sort' => 'timestamp' } }) diff --git a/packages/forest_admin_agent/spec/spec_helper.rb b/packages/forest_admin_agent/spec/spec_helper.rb index 686c44ed5..91a64768f 100644 --- a/packages/forest_admin_agent/spec/spec_helper.rb +++ b/packages/forest_admin_agent/spec/spec_helper.rb @@ -39,6 +39,10 @@ config.include ForestAdminTestToolkit::Factory::Column config.before do + # The audit connection is class-level, so it is handed back between examples: a store pointed at another + # database would otherwise be refused. + ForestAdminAgent::AuditTrail::Sql::AuditConnectionBase.disconnect! + cache = FileCache.new('app', 'tmp/cache/forest_admin') cache.clear From 4ab1ea774d2fc66e8e45927c0aacb10533d129f6 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Mon, 17 Aug 2026 17:34:43 +0200 Subject: [PATCH 26/35] refactor(audit_trail): one migration, tracked beside the table it builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing has shipped, so the six migrations collapse into the single create they would have produced: every column present from the start, `record_id` text and nullable, and the record_id index carrying MySQL's length prefix since the column is text from the outset. The tracker moves from a schema-wide `audit_migrations` to `audit_logs_migration`, named after the table it describes. One tracker per audited table is what the per-table key prefix was working around, so that goes too and migration names are plain again. `status` loses its default with it: the default only existed to backfill rows written before the column, and every write sets it — a row arriving without one is a bug worth a NOT NULL violation rather than a row quietly claiming to be done. Co-Authored-By: Claude Opus 5 (1M context) --- packages/forest_admin_agent/AUDIT_TRAIL.md | 12 ++-- .../audit_trail/sql/migrations.rb | 71 +++++-------------- .../audit_trail/sql/migrator.rb | 19 ++--- .../audit_trail/sql/migrator_spec.rb | 5 +- .../audit_trail/store_spec.rb | 19 ++--- 5 files changed, 47 insertions(+), 79 deletions(-) diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index 3a514255d..aff02c1bc 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -275,7 +275,11 @@ hook. ## Schema migrations & concurrency -The table is created/evolved through an ordered, append-only migration list tracked in a dedicated -`forest.audit_migrations` table. On Postgres the migrations run inside a transaction-scoped advisory -lock so several agents booting at once apply them one after another; the schema is created (and -committed, idempotently) first since the lock can't cover a not-yet-existing schema. +The table is created and evolved through an ordered, append-only migration list, tracked in a companion table +named after the audited one — `forest.audit_logs_migration` beside `forest.audit_logs`. One tracker per audited +table, so two stores configured with different `table_name`s each keep their own schema history rather than +reading the other's as done. + +On Postgres the migrations run inside a transaction-scoped advisory lock, so several agents booting at once +apply them one after another; the schema is created (and committed, idempotently) first, since the lock cannot +cover a schema that does not exist yet. diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb index 1a09a7c14..bea0adad0 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb @@ -1,10 +1,10 @@ module ForestAdminAgent module AuditTrail module Sql - # The audit table's schema, as an ordered, append-only list. Never edit an entry that has shipped: - # add another one, since a database out there has already recorded the earlier ones as applied. Every - # statement is written to tolerate being replayed (`if_not_exists`), because a store that predates the - # per-table tracking key replays them once on upgrade. + # The audit table's schema, as an ordered, append-only list. Nothing has shipped yet, so this is still a + # single entry; once a release is out, never edit one — add another, since a database out there has + # already recorded the earlier ones as applied. Every statement tolerates being replayed + # (`if_not_exists`), which is what makes a lost race harmless. module Migrations ALL = [ { @@ -15,61 +15,28 @@ module Migrations t.datetime :timestamp, null: false t.string :operation, null: false t.string :collection, null: false - t.string :record_id, null: false + # Nullable, because a create's pending row has no id yet; text, because a packed composite id + # outgrows a varchar. + t.text :record_id + # No default on purpose: every write sets it, so a row arriving without one is a bug worth + # hearing about rather than a row that quietly claims to be done. + t.string :status, null: false t.integer :user_id + # Denormalised from the caller at write time: who acted then, not whoever holds that id today. + t.text :user_first_name + t.text :user_last_name + t.text :user_email + # Smart-action rows only. + t.text :action_name t.string :correlation_key t.json :previous_values t.json :new_values end - end - }, - { - name: '002-index-record-and-correlation', - up: lambda do |connection, table| - base = table.split('.').last - connection.add_index(table, :record_id, name: "#{base}_record_id", if_not_exists: true) - connection.add_index(table, :correlation_key, name: "#{base}_correlation_key", if_not_exists: true) - connection.add_index(table, :user_id, name: "#{base}_user_id", if_not_exists: true) - end - }, - { - # Denormalised from the caller at write time: who acted then, not whoever holds that id today. - name: '003-add-user-identity', - up: lambda do |connection, table| - connection.add_column(table, :user_first_name, :text, if_not_exists: true) - connection.add_column(table, :user_last_name, :text, if_not_exists: true) - connection.add_column(table, :user_email, :text, if_not_exists: true) - end - }, - { - name: '004-add-action-name', - up: lambda do |connection, table| - connection.add_column(table, :action_name, :text, if_not_exists: true) - end - }, - { - # The default is what backfills the rows written before the pending/confirm protocol existed. - name: '005-add-status', - up: lambda do |connection, table| - connection.add_column(table, :status, :string, null: false, default: 'done', if_not_exists: true) - end - }, - { - # A create's pending row has no id yet, and a packed composite id outgrows a VARCHAR(255). - name: '006-record-id-nullable-text', - up: lambda do |connection, table| - base = table.split('.').last - mysql = connection.adapter_name.downcase.match?(/mysql|maria/) - # MySQL refuses to widen an indexed column to TEXT, so the index goes first. SQLite rebuilds - # the whole table to change a column, dropping every index 002 created along with it — which is - # why all three are (re)created below rather than just the one dropped here. - connection.remove_index(table, name: "#{base}_record_id", if_exists: true) - connection.change_column(table, :record_id, :text, null: true) - - # ... and MySQL cannot index unbounded TEXT at all: that one needs a length prefix. + base = table.split('.').last + # MySQL cannot index unbounded TEXT, so that one index needs a length prefix. record_id_index = { name: "#{base}_record_id", if_not_exists: true } - record_id_index[:length] = 255 if mysql + record_id_index[:length] = 255 if connection.adapter_name.downcase.match?(/mysql|maria/) connection.add_index(table, :record_id, **record_id_index) connection.add_index(table, :correlation_key, name: "#{base}_correlation_key", if_not_exists: true) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb index c0963ee4c..d396ad432 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrator.rb @@ -1,15 +1,16 @@ module ForestAdminAgent module AuditTrail module Sql - # Applies {Migrations::ALL} to the audit table, tracking what has run in a dedicated - # `audit_migrations` table (namespaced in the `forest` schema on Postgres). + # Applies {Migrations::ALL} to the audit table, tracking what has run in a companion table named after + # it — `audit_logs_migration` beside `audit_logs` (both namespaced in the `forest` schema on Postgres). + # One tracker per audited table, so two stores configured with different `table_name`s each get their + # own schema history instead of reading each other's as done. # # On Postgres the migrations run inside a transaction-scoped advisory lock, so several agent # instances booting at once apply them one after another instead of racing on the same DDL. The # schema is created (and committed) first, made idempotent (CREATE SCHEMA IF NOT EXISTS + # tolerating a concurrent create), because the lock cannot cover a not-yet-existing schema. class Migrator - MIGRATIONS_TABLE = 'audit_migrations'.freeze # Arbitrary but stable key pair identifying the audit-trail migration critical section. ADVISORY_LOCK = [0x464f, 0x5254].freeze # "FO", "RT" # duplicate_schema, and the unique violation on pg_namespace the same race can raise instead. @@ -85,18 +86,12 @@ def apply_pending table = qualified(@table_name) Migrations::ALL.each do |migration| - # The tracking table is shared by every audit table of the database/schema, so the target - # table belongs in the key: a store configured with another `table_name` must still get its - # own table created instead of reading someone else's migration as done. Rows written by - # earlier versions (bare migration name) simply replay, which the `if_not_exists` DDL above - # makes a no-op. - key = "#{table}:#{migration[:name]}" - next if done.include?(key) + next if done.include?(migration[:name]) migration[:up].call(@connection, table) @connection.execute( "INSERT INTO #{@connection.quote_table_name(migrations_table)} (name) " \ - "VALUES (#{@connection.quote(key)})" + "VALUES (#{@connection.quote(migration[:name])})" ) end end @@ -114,7 +109,7 @@ def ensure_migrations_table end def migrations_table - qualified(MIGRATIONS_TABLE) + qualified("#{@table_name}_migration") end end end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb index fe58c7e40..76e82c31a 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb @@ -20,10 +20,9 @@ module Sql add_column: nil, remove_index: nil, change_column: nil, - select_values: ['forest.audit_logs:001-create-audit-logs', - 'forest.audit_logs:002-index-record-and-correlation'], + select_values: ['001-create-audit-logs'], quote_schema_name: '"forest"', - quote_table_name: '"forest.audit_migrations"', + quote_table_name: '"forest.audit_logs_migration"', quote: "'x'" ) end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb index b4bcff448..c28fb675e 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb @@ -128,25 +128,28 @@ def connection expect(store.list_by_correlations(collection: 'accounts', record_id: '1', correlation_keys: [])).to eq([]) end - it 'tracks applied migrations and is idempotent across stores' do + it 'tracks applied migrations beside the table they built, and is idempotent across stores' do store.append(record) described_class.new(database: { adapter: 'sqlite3', database: db.path }).append(record) - names = connection.select_values('SELECT name FROM audit_migrations ORDER BY name') - expect(names).to eq( - ['audit_logs:001-create-audit-logs', 'audit_logs:002-index-record-and-correlation', - 'audit_logs:003-add-user-identity', 'audit_logs:004-add-action-name', 'audit_logs:005-add-status', - 'audit_logs:006-record-id-nullable-text'] - ) + names = connection.select_values('SELECT name FROM audit_logs_migration ORDER BY name') + expect(names).to eq(['001-create-audit-logs']) end - it 'migrates a second table in the same database instead of reading the first one as done' do + # One tracker per audited table, so a second store does not read the first one's history as its own. + it 'migrates a second table in the same database, tracking it separately' do store.append(record) other = described_class.new(database: { adapter: 'sqlite3', database: db.path }, table_name: 'other_logs') expect { other.append(record) }.not_to raise_error expect(other.list_by_record(collection: 'accounts', record_id: '1').size).to eq(1) expect(connection.indexes('other_logs').map(&:name)).to include('other_logs_record_id') + expect(connection.select_values('SELECT name FROM other_logs_migration')).to eq(['001-create-audit-logs']) + end + + # Every write sets it, so a row without one is a bug rather than a row quietly claiming to be done. + it 'refuses a row with no status' do + expect { store.append(record(status: nil)) }.to raise_error(ActiveRecord::NotNullViolation) end it 'binds each store to its own model class instead of mutating a shared one' do From 66bea98631744aa93f959ba0a3b137077d657ce1 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Tue, 18 Aug 2026 09:20:54 +0200 Subject: [PATCH 27/35] fix(audit_trail): keep the critical invariant, and stop inventing what is unknown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four review findings, all of them about claiming more than is known. - Under `critical: true`, a selection wider than the cap was audited in part while the write touched every match — the one thing that mode exists to prevent. It is refused now, before the write. A snapshot that cannot be read at all goes through the same gate, for the same reason: knowing what an operation is about to touch is part of being able to record it. - A record the agent cannot read back after an update left its row confirmed (or discarded) from the patch, claiming values that may never have been written. It stays pending, which is what that state means. - Reads no longer let a pending row act as fact: the state reconstruction skips them, since undoing a change that may never have happened would invent a state the record was never in. The history keeps them — they are evidence of an attempt, and `status` says so. - A caller with no request id gets no correlation key rather than a fabricated one. A generated uuid grouped the row into a request of its own, indistinguishable from a genuine single-row request. Also documents that a non-Rails host has to mount CorrelationIdMiddleware itself, since without its reset a pooled thread hands its previous key to the next request. Co-Authored-By: Claude Opus 5 (1M context) --- packages/forest_admin_agent/AUDIT_TRAIL.md | 27 +++++++++----- .../forest_admin_agent/audit_trail/capture.rb | 8 +++-- .../audit_trail/recording.rb | 6 ++-- .../audit_trail/snapshots.rb | 23 +++++++++--- .../forest_admin_agent/audit_trail/store.rb | 6 +++- .../audit_trail/action_capture_spec.rb | 1 + .../audit_trail/capture_spec.rb | 36 +++++++++++++++++-- .../audit_trail/store_spec.rb | 20 +++++++++++ 8 files changed, 108 insertions(+), 19 deletions(-) diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index aff02c1bc..3d0615c06 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -74,10 +74,18 @@ means the write may or may not have landed: that residue is evidence, and it is the pending insert stays best-effort in both modes, because by then the write has happened and raising would report a failure for an operation that succeeded. -Two consequences worth knowing. A write that turns out to change nothing has its pending row **discarded** -rather than confirmed, so no-op updates leave no trace. And one operation audits at most 500 records: a -wider selection is truncated, with `N records audited, M skipped` logged at `Warn` — an incomplete audit -somebody knows about beats an OOM, and beats silence. +Consequences worth knowing: + +- A write that turns out to change nothing has its pending row **discarded** rather than confirmed, so no-op + updates leave no trace. +- A record the agent cannot read back after the write keeps its row **pending**. Confirming from the patch + would claim values that may never have been written. +- One operation audits at most 500 records. Under `critical: false` a wider selection is truncated, with + `N records audited, M skipped` logged at `Warn`. Under `critical: true` it is **refused**: auditing a subset + while the write touches every match is precisely the invariant that mode exists for. +- Pending rows stay visible in the history — they are evidence of an attempt, and `status` says so — but the + state reconstruction ignores them, since undoing a change that may never have happened would invent a state + the record was never in. ## Routes @@ -245,7 +253,7 @@ one table nobody deletes from. | `user_id` | the Forest user who made the change | | `user_first_name`, `user_last_name`, `user_email` | denormalised from the caller at write time: who acted then, not whoever holds that id today | | `action_name` | smart-action rows only | -| `correlation_key` | per-request id; groups every change made within one request | +| `correlation_key` | per-request id; groups every change made within one request. Empty for a write outside any request — inventing one would make the row look like a single-row request of its own | | `previous_values` | values before the change (JSON) | | `new_values` | values after the change (JSON) | @@ -253,9 +261,12 @@ one table nobody deletes from. arrays of hashes are diffed structurally, so a single sub-field change records just that leaf. Only writable columns are audited — read-only, computed and DB-managed fields are never written by Forest. -The `correlation_key` is the agent's per-request id (`caller.request_id`), generated by the agent and -echoed back to the client in the `X-Forest-Correlation-Id` response header — so every change made in -one request shares a key, and the caller can tie it to its own activity log. +The `correlation_key` is the agent's per-request id (`caller.request_id`), generated by the agent and echoed +back to the client in the `X-Forest-Correlation-Id` response header — so every change made in one request +shares a key, and the caller can tie it to its own activity log. + +Outside Rails, mount `ForestAdminAgent::Http::CorrelationIdMiddleware` yourself: it resets the id at the start +of each request, and without it a pooled thread would hand its previous request's key to the next one. The capture layer registers its **after** hooks ahead of any other customization's (`prepend: true`, since `execute_after` stops at the first exception, and by then the write has already happened) and its diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb index 1833f3d9b..83583d865 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb @@ -122,8 +122,12 @@ def confirm_updates(context, pending, target) empty = [] pending[:records].each_with_index do |record, index| - expected = record.merge(pending[:patch]) - after = persisted[record_id(expected, pks)] || expected + # No row read back: the write may or may not have landed, and inventing after-values from the patch + # would confirm — or worse, discard — a row for something that may never have happened. Left pending, + # which is exactly what that state means. + after = persisted[record_id(record.merge(pending[:patch]), pks)] + next if after.nil? + delta = Diff.changed_values(record, after, target[:columns]) if delta[:new_values].empty? diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb index 614263c0a..39092d28e 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/recording.rb @@ -1,4 +1,3 @@ -require 'securerandom' require 'time' module ForestAdminAgent @@ -24,8 +23,11 @@ def identity_of(caller) # Same id for every change made within one request — set on the caller by the agent (see # CallerParser), mirroring the Node agent's caller.requestId. + # + # nil when the caller carries none, which is what a write outside any request looks like. Inventing one + # would group the row into a request of its own, indistinguishable from a genuine single-row request. def correlation_key_for(caller) - (caller.respond_to?(:request_id) && caller.request_id) || SecureRandom.uuid + caller.respond_to?(:request_id) ? caller.request_id : nil end def redact(values, redacted_fields) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb index 68c93950f..fcd671c73 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb @@ -31,21 +31,36 @@ def pop # was already overwritten. # # An empty list on failure rather than no snapshot at all: the after hook pops unconditionally, so - # skipping the push would pair it with an unrelated entry. + # skipping the push would pair it with an unrelated entry. Reading it goes through the gate, not + # `audit_safely`: knowing what an operation is about to touch is part of being able to record it, so + # under `critical: true` a snapshot that cannot be read refuses the operation. def take(context, projection) cap = AuditTrail::MAX_RECORDS_PER_OPERATION page = ForestAdminDatasourceToolkit::Components::Query::Page.new(offset: 0, limit: cap + 1) - records = audit_safely { context.collection.list(context.filter.override(page: page), projection) } || [] + records = AuditTrail.gate { context.collection.list(context.filter.override(page: page), projection) } || [] return records if records.size <= cap + refuse_or_truncate(context, records, cap) + end + + private + + # Truncating means the operation writes more records than it audits. Tolerable when the audit trail is + # advisory; under `critical: true` it breaks the one invariant the mode exists for, so the operation is + # refused instead — before the write, so there is nothing to repair. + def refuse_or_truncate(context, records, cap) + if AuditTrail.critical? + raise ForestAdminDatasourceToolkit::Exceptions::ForestException, + 'The audit trail is configured as critical and cannot record an operation touching more than ' \ + "#{cap} records at once. Narrow the selection." + end + kept = records.first(cap) AuditTrail.log_truncation(kept.size, audit_safely { count_matching(context) }) kept end - private - def count_matching(context) aggregation = ForestAdminDatasourceToolkit::Components::Query::Aggregation.new(operation: 'Count') diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb index 7e1c07382..38e4f3d72 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb @@ -91,8 +91,12 @@ def authors_by_record(collection:, record_id:, user_ids: nil, start_timestamp: n # Entries recorded strictly after `timestamp`, newest first: what a state reconstruction has to undo. # Strictly after, so an entry stamped exactly at the requested instant counts as part of that state # instead of being reverted out of it. + # Confirmed rows only: a pending one records an attempt whose outcome is unknown, and undoing a change + # that may never have happened would invent a state the record was never in. The history reads keep + # pending rows — they are evidence, and `status` tells the reader what they are — but a reconstruction + # cannot act on them. def list_since(collection:, record_id:, timestamp:) - model.where(collection: collection, record_id: record_id) + model.where(collection: collection, record_id: record_id, status: Recording::DONE) .where('timestamp > ?', as_time(timestamp)) .order(timestamp: :desc, id: :desc) .map { |row| from_row(row) } diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb index ad74c8612..54e1ab821 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb @@ -70,6 +70,7 @@ def run(capture: described_class.new(store), result: { type: 'Success', message: capture.pending(**invocation(caller: double('caller'))) expect(store.records.last.user_email).to be_nil + expect(store.records.last.correlation_key).to be_nil end it 'writes one row per targeted record, sharing timestamp and correlation key' do diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb index 7a173bdc4..8d816214a 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb @@ -127,6 +127,16 @@ def after_hook(type, record: nil) expect(store.records.last.user_email).to eq('ada@test') expect(store.records.last.correlation_key).to eq('req-xyz') end + + # A write outside any request belongs to no request: inventing a key would make the row look like a + # single-row request of its own. + it 'leaves the correlation key empty when the caller carries none' do + allow(caller_double).to receive(:request_id).and_return(nil) + + before_hook('Create', data: { 'name' => 'Acme' }) + + expect(store.records.last.correlation_key).to be_nil + end end describe 'updating records' do @@ -178,10 +188,13 @@ def update(before:, persisted:, patch:) end end - it 'falls back to the patch when the record cannot be read back' do + # Confirming from the patch would claim values that may never have been written; discarding would erase + # the evidence that something was attempted. + it 'leaves the row pending when the record cannot be read back' do rows = update(before: [{ 'id' => 1, 'name' => 'Acme' }], persisted: [], patch: { 'name' => 'Z' }) - expect(rows.last.new_values).to eq({ 'name' => 'Z' }) + expect(rows.last.status).to eq(Recording::PENDING) + expect(store.discarded).to be_empty end # Nothing changed, so nothing is audited: the pending row goes rather than sitting there implying the @@ -214,6 +227,14 @@ def update(before:, persisted:, patch:) describe 'when the audit database is unreachable' do before { allow(store).to receive(:append_all).and_raise(StandardError, 'audit db is down') } + # Knowing what an operation is about to touch is part of being able to record it. + it 'refuses the operation when the snapshot cannot even be read and critical is on' do + allow(ForestAdminAgent::AuditTrail).to receive(:critical?).and_return(true) + allow(relaxed_collection).to receive(:list).and_raise(StandardError, 'datasource down') + + expect { before_hook('Delete') }.to raise_error(StandardError, 'datasource down') + end + it 'lets the write through, logging the failure, when critical is off' do logger = instance_spy(Services::LoggerService) allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) @@ -246,6 +267,17 @@ def update(before:, persisted:, patch:) expect(logger).to have_received(:log).with('Warn', /#{cap} records audited, 25 skipped/) end + # Auditing 500 of them while the write touches every match breaks the one invariant critical exists + # for, so the operation is refused instead — before the write, with nothing to repair. + it 'refuses the operation when critical is on' do + allow(ForestAdminAgent::AuditTrail).to receive(:critical?).and_return(true) + allow(relaxed_collection).to receive(:list).and_return((1..(cap + 1)).map { |id| { 'id' => id } }) + + expect { before_hook('Delete') }.to raise_error( + ForestAdminDatasourceToolkit::Exceptions::ForestException, /cannot record an operation touching more/ + ) + end + it 'asks for one more than the cap, so it can tell it was truncated' do allow(relaxed_collection).to receive(:list).and_return([]) diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb index c28fb675e..4345d494a 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb @@ -203,6 +203,26 @@ def connection end describe '#list_since' do + # A pending row records an attempt whose outcome is unknown; undoing it would invent a state the + # record was never in. + it 'leaves a pending row out, so a reconstruction cannot undo a change that may never have happened' do + store.append(record(timestamp: '2026-01-02T03:04:07.000Z', status: Recording::PENDING, + correlation_key: 'attempt')) + store.append(record(timestamp: '2026-01-02T03:04:07.000Z', correlation_key: 'confirmed')) + + history = store.list_since(collection: 'accounts', record_id: '1', + timestamp: '2026-01-02T03:04:06.000Z') + + expect(history.map(&:correlation_key)).to eq(['confirmed']) + end + + # They are evidence, and `status` tells the reader what they are. + it 'keeps them in the history, where the payload says what they are' do + store.append(record(status: Recording::PENDING)) + + expect(store.list_by_record(collection: 'accounts', record_id: '1').map(&:status)).to eq(['pending']) + end + it 'returns entries strictly newer than the instant, newest first' do store.append(record(timestamp: '2026-01-02T03:04:05.000Z', correlation_key: 'older')) store.append(record(timestamp: '2026-01-02T03:04:06.000Z', correlation_key: 'at')) From f97981f9368cc58054b790dcd2d6c038f5b3c449 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Tue, 18 Aug 2026 11:40:44 +0200 Subject: [PATCH 28/35] test(capabilities): cover canUseAuditTrail, and read it from one place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flag was in the working tree without the two specs it changes. Its lookup now goes through `AuditTrail.store`, the same one the record-history route mounts itself on, so the capability cannot drift from what the routes actually serve — and the specs pin it both ways, since the front gates its History tab on it. Co-Authored-By: Claude Opus 5 (1M context) --- .../routes/capabilities/collections.rb | 12 +++++++++++- .../routes/capabilities/collections_spec.rb | 13 +++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/capabilities/collections.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/capabilities/collections.rb index b25d63ff3..869d9cd3f 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/capabilities/collections.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/capabilities/collections.rb @@ -67,12 +67,22 @@ def handle_request(args = {}) canUseProjectionOnGetOne: true, canUseProjectionViaHeader: true, canUseProjectionViaHeaderOnList: true, - canUseMultipleFieldsProjectionOnRelation: true + canUseMultipleFieldsProjectionOnRelation: true, + canUseAuditTrail: audit_trail_enabled? } }, status: 200 } end + + private + + # True only where the store the record-history route reads from exists — the same lookup that route + # mounts itself on, so the capability cannot drift from what the routes actually serve. The front gates + # its History tab on this. + def audit_trail_enabled? + !::ForestAdminAgent::AuditTrail.store.nil? + end end end end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/capabilities/collections_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/capabilities/collections_spec.rb index a265e153c..76e08369d 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/capabilities/collections_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/capabilities/collections_spec.rb @@ -82,13 +82,21 @@ module Capabilities ) end + it 'announces the audit trail once a store is configured' do + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) + .and_return({ audit_trail: { store: Object.new } }) + + expect(result[:content][:agentCapabilities][:canUseAuditTrail]).to be true + end + it 'returns agentCapabilities' do expect(result[:content][:agentCapabilities]).to eq( { canUseProjectionOnGetOne: true, canUseProjectionViaHeader: true, canUseProjectionViaHeaderOnList: true, - canUseMultipleFieldsProjectionOnRelation: true + canUseMultipleFieldsProjectionOnRelation: true, + canUseAuditTrail: false } ) end @@ -140,7 +148,8 @@ module Capabilities canUseProjectionOnGetOne: true, canUseProjectionViaHeader: true, canUseProjectionViaHeaderOnList: true, - canUseMultipleFieldsProjectionOnRelation: true + canUseMultipleFieldsProjectionOnRelation: true, + canUseAuditTrail: false } ) end From 4c5cef4729df24d1c133325eb98cf2ec06441bc5 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Tue, 18 Aug 2026 11:40:44 +0200 Subject: [PATCH 29/35] feat(audit_trail): search the history by free text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The activity logs can only match a term against their own description, so nothing finds the edit that set a city to Lyon. The agent is the only side holding the recorded values, so the record-history route takes an optional `search`. Matched case-insensitively, as a substring, against the action name, the actor's name and email, and the keys and values of both value objects at any depth — the JSON document as text, so one condition reaches every leaf and composes with pagination and the count instead of filtering in memory. Not matched against operation, correlation_key, record_id, collection, status or timestamp: machine identifiers nobody searches for, whose hits read as noise. Two things it has to get right. A masked field never matches — the `[redacted]` mask is stripped before comparison, and the value it hid was never recorded, so searching either finds nothing: a search must not confirm what the trail refused to keep. And LIKE wildcards in the term are escaped with `!` rather than a backslash, which MySQL also treats as an escape inside string literals. The per-adapter text cast lives in Sql::TextSearch beside the field filter, with its SQL pinned for Postgres and MySQL since the suite only runs SQLite. Co-Authored-By: Claude Opus 5 (1M context) --- packages/forest_admin_agent/AUDIT_TRAIL.md | 13 +++ .../audit_trail/sql/text_search.rb | 62 ++++++++++++ .../forest_admin_agent/audit_trail/store.rb | 22 +++-- .../routes/resources/audit_trail.rb | 8 ++ .../audit_trail/sql/text_search_spec.rb | 82 ++++++++++++++++ .../audit_trail/store_spec.rb | 94 +++++++++++++++++++ .../routes/resources/audit_trail_spec.rb | 30 ++++++ 7 files changed, 303 insertions(+), 8 deletions(-) create mode 100644 packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/text_search.rb create mode 100644 packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/text_search_spec.rb diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index 3d0615c06..4710c0a11 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -113,6 +113,19 @@ or removed still reads as they were when they acted. Optional filters (all combi | `startDate` | `YYYY-MM-DD` or datetime (incl.) | keep entries from this lower bound onward | | `endDate` | `YYYY-MM-DD` or datetime (incl.) | keep entries up to this upper bound | | `fields` | comma-separated field names | keep only entries whose diff touched one of them | +| `search` | free text, trimmed | keep only entries the term matches | + +`search` is matched case-insensitively, as a substring, against the action's name, the actor's first name, +last name and email, and the **keys and values of both value objects at any depth** — searching `Lyon` finds +`{"address": {"city": "Lyon"}}`, which is what only the agent can answer, since it is the only side holding +the recorded values. It is matched in SQL, not in memory, so it composes with pagination and `meta.count` +like every other filter. + +It deliberately does **not** match `operation`, `correlationKey`, `recordId`, `collection`, `status` or +`timestamp`: machine identifiers nobody searches for, whose matches read as noise. + +A field masked by `redact` never matches — neither by its `[redacted]` mask nor by the value it hid, which +was never recorded. A search must not confirm a value the trail refused to keep. `fields` matches whole keys, never paths, so a name holding a dot (`address.city`) is quoted before it reaches SQL. Both sides of the diff are searched, since a field the change added exists in `newValues` diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/text_search.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/text_search.rb new file mode 100644 index 000000000..bc0509789 --- /dev/null +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/text_search.rb @@ -0,0 +1,62 @@ +module ForestAdminAgent + module AuditTrail + module Sql + # SQL keeping only the audit entries a free-text term matches, case-insensitively and as a substring: + # the action's name, who acted, and the keys and values recorded on both sides of the change — at any + # depth, since only the changed leaves of a JSON column are stored and the term has to reach them. + # + # Deliberately not searched: operation, correlation_key, record_id, collection, status and timestamp. + # Machine identifiers nobody searches for, and matching them turns one term into a pile of confusing + # hits. + # + # The values are matched against the JSON document as text, which is what lets one condition reach any + # depth and compose with pagination and the count. It cannot use an index, which is affordable here + # because a history query is already narrowed to one record. + class TextSearch + TEXT_COLUMNS = %w[action_name user_first_name user_last_name user_email].freeze + JSON_COLUMNS = %w[previous_values new_values].freeze + # `!` rather than a backslash: MySQL treats a backslash as an escape inside string literals too, so + # `ESCAPE '\'` needs doubling there and nowhere else. + ESCAPE = '!'.freeze + # A masked value is stored as this. It is removed before matching, so a search for "redacted" cannot + # hit it — and since the real value was never recorded, searching that finds nothing either. A search + # must never confirm a value the trail refused to keep. + REDACTED = Recording::REDACTED + + def initialize(connection) + @connection = connection + end + + def condition(term) + pattern = @connection.quote("%#{escape(term.to_s.downcase)}%") + columns = TEXT_COLUMNS.map { |column| column } + JSON_COLUMNS.map { |column| searchable_json(column) } + + columns.map { |expression| "LOWER(#{expression}) LIKE #{pattern} ESCAPE '#{ESCAPE}'" }.join(' OR ') + end + + private + + def searchable_json(column) + "REPLACE(#{as_text(column)}, #{@connection.quote(REDACTED)}, '')" + end + + def as_text(column) + adapter = @connection.adapter_name.downcase + + case adapter + when /postgres/ then "#{column}::text" + when /sqlite/ then column + when /mysql|maria/ then "CAST(#{column} AS CHAR)" + else + raise ForestAdminDatasourceToolkit::Exceptions::ForestException, + "Searching the audit trail is not supported on #{adapter}" + end + end + + def escape(term) + term.gsub(/[!%_]/) { |char| "#{ESCAPE}#{char}" } + end + end + end + end +end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb index 38e4f3d72..9902c7ab1 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb @@ -58,11 +58,12 @@ def discard(ids) model.where(id: ids).delete_all unless ids.empty? end - def list_by_record(collection:, record_id:, skip: 0, limit: nil, user_ids: nil, - start_timestamp: nil, end_timestamp: nil, fields: nil, order: 'asc') + def list_by_record(collection:, record_id:, skip: 0, limit: nil, user_ids: nil, start_timestamp: nil, + end_timestamp: nil, fields: nil, search: nil, order: 'asc') # `id` (insertion order) breaks ties on equal timestamps in both directions, keeping pages # deterministic and stable. - relation = scope(collection, record_id, user_ids, start_timestamp, end_timestamp, fields) + relation = scope(collection, record_id, user_ids: user_ids, start_timestamp: start_timestamp, + end_timestamp: end_timestamp, fields: fields, search: search) .order(timestamp: order.to_s == 'desc' ? :desc : :asc, id: :asc) .offset(skip || 0) relation = relation.limit(limit) unless limit.nil? @@ -71,16 +72,18 @@ def list_by_record(collection:, record_id:, skip: 0, limit: nil, user_ids: nil, end def count_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil, - end_timestamp: nil, fields: nil) - scope(collection, record_id, user_ids, start_timestamp, end_timestamp, fields).count + end_timestamp: nil, fields: nil, search: nil) + scope(collection, record_id, user_ids: user_ids, start_timestamp: start_timestamp, + end_timestamp: end_timestamp, fields: fields, search: search).count end # The distinct authors of the entries the current filters match, whatever page is being asked for. The # identity comes from the rows themselves, so a user who has since been renamed or removed still reads # as they were when they acted. def authors_by_record(collection:, record_id:, user_ids: nil, start_timestamp: nil, - end_timestamp: nil, fields: nil) - scope(collection, record_id, user_ids, start_timestamp, end_timestamp, fields) + end_timestamp: nil, fields: nil, search: nil) + scope(collection, record_id, user_ids: user_ids, start_timestamp: start_timestamp, + end_timestamp: end_timestamp, fields: fields, search: search) .where.not(user_id: nil) .distinct .pluck(*AUTHOR_COLUMNS) @@ -116,10 +119,13 @@ def list_by_correlations(collection:, record_id:, correlation_keys:) private - def scope(collection, record_id, user_ids, start_timestamp, end_timestamp, fields = nil) + # Every filter is an AND, so the count matches exactly what a page of this history holds. + def scope(collection, record_id, user_ids: nil, start_timestamp: nil, end_timestamp: nil, + fields: nil, search: nil) relation = model.where(collection: collection, record_id: record_id) relation = relation.where(user_id: user_ids) if user_ids relation = relation.where(Sql::FieldFilter.new(model.connection).condition(fields)) if fields&.any? + relation = relation.where(Sql::TextSearch.new(model.connection).condition(search)) if search # Compare as Time so ActiveRecord casts the bound to the datetime column's storage format # (raw ISO strings with a `Z` would compare lexically against the cast rows and never match). relation = relation.where('timestamp >= ?', as_time(start_timestamp)) if start_timestamp diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb index dafa4d137..f783d1516 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb @@ -158,11 +158,19 @@ def parse_filters(args) { user_ids: parse_user_ids(args.dig(:params, 'userIds')), fields: parse_fields(args.dig(:params, 'fields')), + search: parse_search(args.dig(:params, 'search')), start_timestamp: parse_date_boundary(args.dig(:params, 'startDate'), timezone, :start), end_timestamp: parse_date_boundary(args.dig(:params, 'endDate'), timezone, :end) }.compact end + # Free text, trimmed; blank means no filter rather than a term that matches everything. + def parse_search(raw) + term = raw.to_s.strip + + term.empty? ? nil : term + end + # Comma-separated field names, kept verbatim (a name may hold a dot). Empty after parsing → no filter. def parse_fields(raw) return nil if raw.nil? diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/text_search_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/text_search_spec.rb new file mode 100644 index 000000000..9eb01564d --- /dev/null +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/text_search_spec.rb @@ -0,0 +1,82 @@ +require 'spec_helper' + +module ForestAdminAgent + module AuditTrail + module Sql + # The store specs exercise this against SQLite; these pin the SQL for the adapters no local database + # covers, and the two properties that matter whatever the dialect. + describe TextSearch do + let(:adapters) { %w[PostgreSQL SQLite Mysql2] } + let(:searched_columns) { %w[action_name user_first_name user_last_name user_email] } + + def filter_for(adapter) + # rubocop:disable RSpec/VerifiedDoubles + connection = double(adapter.to_s, adapter_name: adapter) + # rubocop:enable RSpec/VerifiedDoubles + allow(connection).to receive(:quote) { |value| "'#{value}'" } + + described_class.new(connection) + end + + it 'casts the JSON columns to text on Postgres' do + condition = filter_for('PostgreSQL').condition('lyon') + + expect(condition).to include("LOWER(REPLACE(previous_values::text, '[redacted]', '')) LIKE '%lyon%'") + expect(condition).to include("LOWER(REPLACE(new_values::text, '[redacted]', '')) LIKE '%lyon%'") + end + + it 'reads the JSON columns as they are stored on SQLite' do + expect(filter_for('SQLite').condition('lyon')).to include( + "LOWER(REPLACE(previous_values, '[redacted]', '')) LIKE '%lyon%'" + ) + end + + it 'casts the JSON columns to CHAR on MySQL' do + expect(filter_for('Mysql2').condition('lyon')).to include( + "LOWER(REPLACE(CAST(previous_values AS CHAR), '[redacted]', '')) LIKE '%lyon%'" + ) + end + + it 'searches the action name and the actor, on every adapter' do + adapters.each do |adapter| + condition = filter_for(adapter).condition('ada') + + searched_columns.each do |column| + expect(condition).to include("LOWER(#{column}) LIKE '%ada%'") + end + end + end + + # Machine identifiers nobody searches for. + it 'never searches an identifier column' do + condition = filter_for('PostgreSQL').condition('x') + + %w[operation correlation_key record_id collection status timestamp].each do |column| + expect(condition).not_to include(column) + end + end + + # `!` and not a backslash: MySQL treats a backslash as an escape inside string literals too. + it 'escapes LIKE wildcards in the term rather than letting them match anything' do + condition = filter_for('SQLite').condition('50%_off') + + expect(condition).to include("LIKE '%50!%!_off%' ESCAPE '!'") + end + + it 'escapes the escape character itself' do + expect(filter_for('SQLite').condition('a!b')).to include("LIKE '%a!!b%'") + end + + it 'lowercases the term, since the comparison lowercases the column' do + expect(filter_for('SQLite').condition('LyOn')).to include("LIKE '%lyon%'") + end + + it 'refuses an adapter it has no text cast for' do + expect { filter_for('Informix').condition('x') }.to raise_error( + ForestAdminDatasourceToolkit::Exceptions::ForestException, /not supported on informix/ + ) + end + end + end + end +end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb index 4345d494a..2f3236bc4 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb @@ -287,6 +287,100 @@ def connection end end + describe 'searching free text' do + def matching_search(term) + store.list_by_record(collection: 'accounts', record_id: '1', search: term).map(&:correlation_key) + end + + it 'matches a value, case-insensitively and as a substring' do + store.append(record(correlation_key: 'lyon', new_values: { 'city' => 'Lyon' })) + store.append(record(correlation_key: 'paris', new_values: { 'city' => 'Paris' })) + + expect(matching_search('lyo')).to eq(['lyon']) + expect(matching_search('LYON')).to eq(['lyon']) + end + + # Only the changed leaves of a JSON column are stored, so the match has to reach them. + it 'reaches a value nested at any depth' do + store.append(record(correlation_key: 'nested', + new_values: { 'address' => { 'city' => 'Lyon', 'zip' => '69001' } })) + + expect(matching_search('Lyon')).to eq(['nested']) + expect(matching_search('69001')).to eq(['nested']) + end + + it 'matches a key as well as a value' do + store.append(record(correlation_key: 'keyed', new_values: { 'first_name' => 'Jo' })) + + expect(matching_search('first_name')).to eq(['keyed']) + end + + it 'matches the previous side too, not only the new one' do + store.append(record(correlation_key: 'was', previous_values: { 'city' => 'Lyon' }, new_values: {})) + + expect(matching_search('Lyon')).to eq(['was']) + end + + it 'matches the action name and who acted' do + store.append(record(correlation_key: 'refund', operation: 'action', action_name: 'Refund order', + previous_values: {}, new_values: {})) + store.append(record(correlation_key: 'ada', user_first_name: 'Ada', user_last_name: 'Lovelace', + user_email: 'ada@test', previous_values: {}, new_values: {})) + + expect(matching_search('refund ord')).to eq(['refund']) + expect(matching_search('lovelace')).to eq(['ada']) + expect(matching_search('ada@')).to eq(['ada']) + end + + # Machine identifiers nobody searches for: matching them turns one term into confusing hits. + it 'ignores the operation, the correlation key, the record id and the status' do + store.append(record(correlation_key: 'searchable-key', operation: 'update', record_id: '1', + previous_values: {}, new_values: {})) + + expect(matching_search('searchable-key')).to be_empty + expect(matching_search('update')).to be_empty + expect(matching_search('done')).to be_empty + end + + # A search must never confirm a value the trail refused to record. + it 'never matches a redacted field, by its mask or by the value it hid' do + store.append(record(correlation_key: 'masked', previous_values: { 'email' => '[redacted]' }, + new_values: { 'email' => '[redacted]' })) + + expect(matching_search('redacted')).to be_empty + expect(matching_search('[redacted]')).to be_empty + expect(matching_search('secret@test')).to be_empty + end + + it 'treats LIKE wildcards in the term as ordinary characters' do + store.append(record(correlation_key: 'literal', new_values: { 'code' => '50%_off' })) + store.append(record(correlation_key: 'other', new_values: { 'code' => 'anything' })) + + expect(matching_search('50%_')).to eq(['literal']) + expect(matching_search('%')).to eq(['literal']) + end + + it 'composes with the other filters, and the count agrees with it' do + store.append(record(correlation_key: 'keep', user_id: 7, new_values: { 'city' => 'Lyon' })) + store.append(record(correlation_key: 'wrong-user', user_id: 9, new_values: { 'city' => 'Lyon' })) + store.append(record(correlation_key: 'wrong-term', user_id: 7, new_values: { 'city' => 'Paris' })) + + expect(store.list_by_record(collection: 'accounts', record_id: '1', search: 'lyon', + user_ids: [7]).map(&:correlation_key)).to eq(['keep']) + expect(store.count_by_record(collection: 'accounts', record_id: '1', search: 'lyon', + user_ids: [7])).to eq(1) + end + + it 'narrows the authors it offers to those the term matches' do + store.append(record(user_id: 7, new_values: { 'city' => 'Lyon' })) + store.append(record(user_id: 9, new_values: { 'city' => 'Paris' })) + + authors = store.authors_by_record(collection: 'accounts', record_id: '1', search: 'lyon') + + expect(authors.map { |author| author[:user_id] }).to eq([7]) + end + end + describe '#authors_by_record' do it 'lists the distinct authors of the matching entries, as they were when they acted' do store.append(record(user_id: 7, user_first_name: 'Ada', user_last_name: 'L', user_email: 'ada@test')) diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb index 517857089..75321a3f7 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -313,6 +313,36 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) expect(store).to have_received(:list_by_record).with(hash_including(skip: 0, limit: 20)) end + it 'passes a search term through, trimmed' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', + 'search' => ' Lyon ' } }) + + expect(store).to have_received(:list_by_record).with(hash_including(search: 'Lyon')) + # The count has to agree with the filter, like every other one. + expect(store).to have_received(:count_by_record).with(hash_including(search: 'Lyon')) + end + + it 'sends no search when the term is blank' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'search' => ' ' } }) + + expect(store).to have_received(:list_by_record).with(hash_excluding(:search)) + end + + it 'combines a search with the other filters as one AND' do + route = route_with_store + route.handle_request({ headers: {}, + params: { 'collection_name' => 'projects', 'id' => '4', 'search' => 'Lyon', + 'userIds' => '12', 'fields' => 'address.city' } }) + + expect(store).to have_received(:list_by_record).with( + hash_including(search: 'Lyon', user_ids: [12], fields: ['address.city']) + ) + end + it 'passes a fields filter through, keeping names that hold a dot' do route = route_with_store route.handle_request({ headers: {}, From 55ef0ebc68e8a71154644f59c1deba70e033e085 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Tue, 18 Aug 2026 14:28:24 +0200 Subject: [PATCH 30/35] fix(audit_trail): follow a record's history across a primary-key change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Filing an update under the record's new id — which later lookups use — left every row written before the rename under the old one, unreachable. So the history endpoint started the story at the rename, and /state reconstructed from half a timeline and called it the state. An update that moves a writable primary key now records the id it left, in `previous_record_id`, and both routes walk that chain back before querying: the current id plus every id the record has been filed under. Capped at ten hops, and a chain that comes back on itself stops rather than looping. The column stays out of the payload — it is how the agent follows a record, not part of the contract the front reads. Co-Authored-By: Claude Opus 5 (1M context) --- packages/forest_admin_agent/AUDIT_TRAIL.md | 7 +++ .../audit_trail/audit_record.rb | 2 +- .../forest_admin_agent/audit_trail/capture.rb | 8 +++- .../audit_trail/sql/migrations.rb | 3 ++ .../forest_admin_agent/audit_trail/store.rb | 14 +++++- .../routes/resources/audit_trail.rb | 9 ++-- .../routes/resources/audit_trail_route.rb | 25 ++++++++++- .../audit_trail/capture_spec.rb | 6 ++- .../audit_trail/store_spec.rb | 20 ++++++++- .../routes/capabilities/collections_spec.rb | 7 ++- .../routes/resources/audit_trail_spec.rb | 45 ++++++++++++++++--- 11 files changed, 127 insertions(+), 19 deletions(-) diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index 4710c0a11..54d943cba 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -158,6 +158,12 @@ the merge tiebreaker. Inside the value objects: a record's column names pass through untouched, while an action answer's keys are Forest's own and so are camelCase (`mimeType`, not `mime_type`) — the agent transforms them on write. +**A record that was renamed keeps one timeline.** An update that moves a writable primary key files its row +under the record's new id — the id later lookups use — and remembers the one it left. Both the history and the +state routes walk that chain back, so asking for the current id returns everything the record has ever been +filed under, rather than starting the story at the rename. The chain is walked at most ten hops, and a cycle +stops it. + A record that no longer exists keeps its history: only a record that still exists *outside* the caller's permission scope is refused (404). Inspecting what was deleted is much of the point of an audit trail, and the delete event itself is the last thing recorded. @@ -263,6 +269,7 @@ one table nobody deletes from. | `operation` | `create` / `update` / `delete` | | `collection` | audited collection name | | `record_id` | packed record id (primary keys joined by `\|`), TEXT and nullable — a create's pending row has none yet, and a composite id outgrows a varchar | +| `previous_record_id` | set only on an update that moved a writable primary key: the id the row was filed under before | | `user_id` | the Forest user who made the change | | `user_first_name`, `user_last_name`, `user_email` | denormalised from the caller at write time: who acted then, not whoever holds that id today | | `action_name` | smart-action rows only | diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb index 63750812b..1fa3e8d7f 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb @@ -7,7 +7,7 @@ module AuditTrail # follows the write protocol — inserted as {Recording::PENDING} before the write and confirmed # {Recording::DONE} after, so a row left pending means the write may or may not have landed. AuditRecord = Struct.new( - :id, :timestamp, :operation, :collection, :record_id, :status, + :id, :timestamp, :operation, :collection, :record_id, :previous_record_id, :status, :user_id, :user_first_name, :user_last_name, :user_email, :action_name, :correlation_key, :previous_values, :new_values, keyword_init: true diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb index 83583d865..f90063dcf 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb @@ -133,8 +133,14 @@ def confirm_updates(context, pending, target) if delta[:new_values].empty? empty << pending[:ids][index] else + before_id = record_id(record, pks) + after_id = record_id(after, pks) + confirm(pending[:ids][index], - record_id: record_id(after, pks), + record_id: after_id, + # Only when the key actually moved, so a history query can walk back to the rows filed + # under the id this record used to have. + previous_record_id: after_id == before_id ? nil : before_id, previous_values: redacted(target[:name], delta[:previous_values]), new_values: redacted(target[:name], delta[:new_values])) end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb index bea0adad0..9cada5f0d 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/migrations.rb @@ -26,6 +26,9 @@ module Migrations t.text :user_first_name t.text :user_last_name t.text :user_email + # Set only on an update that moved a writable primary key: the id the row was filed under + # before. What lets a history query follow a record across a rename. + t.text :previous_record_id # Smart-action rows only. t.text :action_name t.string :correlation_key diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb index 9902c7ab1..d4a2b5dbd 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb @@ -11,8 +11,8 @@ module AuditTrail class Store DEFAULT_SCHEMA = 'forest'.freeze DEFAULT_TABLE = 'audit_logs'.freeze - COLUMNS = %i[timestamp operation collection record_id status user_id user_first_name user_last_name - user_email action_name correlation_key previous_values new_values].freeze + COLUMNS = %i[timestamp operation collection record_id previous_record_id status user_id user_first_name + user_last_name user_email action_name correlation_key previous_values new_values].freeze AUTHOR_COLUMNS = %i[user_id user_first_name user_last_name user_email].freeze def initialize(database:, schema: DEFAULT_SCHEMA, table_name: DEFAULT_TABLE) @@ -91,6 +91,16 @@ def authors_by_record(collection:, record_id:, user_ids: nil, start_timestamp: n .uniq { |author| author[:user_id] } end + # The id this record was filed under before an update moved its primary key, if one did. Walking that + # back is what lets a history query reach rows written before a rename — they stay under the id they were + # written with, since that is the id they were true of. + def previous_record_ids(collection:, record_id:) + model.where(collection: collection, record_id: record_id) + .where.not(previous_record_id: nil) + .distinct + .pluck(:previous_record_id) + end + # Entries recorded strictly after `timestamp`, newest first: what a state reconstruction has to undo. # Strictly after, so an entry stamped exactly at the requested instant counts as part of that state # instead of being reverted out of it. diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb index f783d1516..37a4df6ea 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb @@ -44,8 +44,9 @@ def handle_request(args = {}) skip, limit = parse_pagination(args) filters = { collection: context.collection.name, - # args[:params]['id'] is already Forest's packed id, the form the audit store keys on. - record_id: args[:params]['id'], + # args[:params]['id'] is already Forest's packed id, the form the audit store keys on — plus any id + # this record was filed under before a rename. + record_id: record_ids_history(context.collection, args[:params]['id']), **parse_filters(args) } @@ -72,7 +73,9 @@ def handle_state(args = {}) timestamp = parse_state_timestamp(args) entries = store.list_since( - collection: context.collection.name, record_id: args[:params]['id'], timestamp: timestamp + collection: context.collection.name, + record_id: record_ids_history(context.collection, args[:params]['id']), + timestamp: timestamp ) # Fully qualified: inside this class, `AuditTrail` is the route itself. state = ::ForestAdminAgent::AuditTrail::RecordState.at(current, entries) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb index 6e09af5e9..9859c65ea 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb @@ -41,13 +41,36 @@ def scoped_record(context, collection, packed_id, projection = nil) # merging pages ordered by (timestamp, id). Value hashes keep the keys they were stored with: a record's # own column names, or an action answer's camelCase Forest names. def serialize_record(record) - record.to_h.transform_keys { |key| key.to_s.camelize(:lower) } + # `previous_record_id` stays out: it is how the agent follows a record across a rename, not something + # the payload contract carries. + record.to_h.except(:previous_record_id).transform_keys { |key| key.to_s.camelize(:lower) } end def store ::ForestAdminAgent::AuditTrail.store end + # Every id this record has been filed under, newest first. Rows written before an update moved a + # writable primary key stay under the id they were true of, so a history query that asked for the + # current id alone would start at the rename and call that the whole story. + # + # ponytail: 10 hops is far more renaming than any record sees; the guard is against a cycle, not depth. + MAX_RENAMES = 10 + + def record_ids_history(collection, packed_id) + ids = [packed_id] + + MAX_RENAMES.times do + previous = store.previous_record_ids(collection: collection.name, record_id: ids.last) + previous -= ids + break if previous.empty? + + ids.concat(previous) + end + + ids + end + # What the audit trail actually records: primary keys, so a state can be identified, plus the writable # columns. Reading read-only ones would hand them back at their present value inside an answer that # claims to describe a past instant. diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb index 8d816214a..c59c84d24 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb @@ -164,6 +164,8 @@ def update(before:, persisted:, patch:) persisted: [{ 'id' => 1, 'name' => 'ACME NORMALISED' }], patch: { 'name' => 'Acme' }) + # No rename, so nothing to remember. + expect(rows.last.previous_record_id).to be_nil expect(rows.last.status).to eq(Recording::DONE) expect(rows.last.previous_values).to eq({ 'name' => 'acme' }) expect(rows.last.new_values).to eq({ 'name' => 'ACME NORMALISED' }) @@ -178,13 +180,15 @@ def update(before:, persisted:, patch:) } end - it 'files the row under the id the record ended up with' do + it 'files the row under the id the record ended up with, remembering the one it left' do rows = update(before: [{ 'id' => 1, 'name' => 'Acme' }], persisted: [{ 'id' => 7, 'name' => 'Acme' }], patch: { 'id' => 7 }) expect(rows.last.record_id).to eq('7') expect(rows.last.new_values).to eq({ 'id' => 7 }) + # How a history query reaches the rows written while it was still 1. + expect(rows.last.previous_record_id).to eq('1') end end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb index 2f3236bc4..fa9ba38c0 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb @@ -32,8 +32,8 @@ def connection store.append(record) expect(store.send(:model).column_names.sort).to eq( - %w[action_name collection correlation_key id new_values operation previous_values record_id status - timestamp user_email user_first_name user_id user_last_name] + %w[action_name collection correlation_key id new_values operation previous_record_id previous_values + record_id status timestamp user_email user_first_name user_id user_last_name] ) end @@ -287,6 +287,22 @@ def connection end end + describe '#previous_record_ids' do + it 'returns the ids this record was filed under before a rename, once each' do + store.append(record(record_id: '7', previous_record_id: '1')) + store.append(record(record_id: '7', previous_record_id: '1')) + store.append(record(record_id: '7')) + + expect(store.previous_record_ids(collection: 'accounts', record_id: '7')).to eq(['1']) + end + + it 'returns nothing for a record that was never renamed' do + store.append(record) + + expect(store.previous_record_ids(collection: 'accounts', record_id: '1')).to be_empty + end + end + describe 'searching free text' do def matching_search(term) store.list_by_record(collection: 'accounts', record_id: '1', search: term).map(&:correlation_key) diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/capabilities/collections_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/capabilities/collections_spec.rb index 76e08369d..d3a5482d1 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/capabilities/collections_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/capabilities/collections_spec.rb @@ -83,8 +83,11 @@ module Capabilities end it 'announces the audit trail once a store is configured' do - allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) - .and_return({ audit_trail: { store: Object.new } }) + # Merged rather than replaced: the rest of the request reads this config too. + configured = ForestAdminAgent::Facades::Container.config_from_cache.merge( + audit_trail: { store: Object.new } + ) + allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache).and_return(configured) expect(result[:content][:agentCapabilities][:canUseAuditTrail]).to be true end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb index 75321a3f7..06e054df5 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -29,7 +29,7 @@ def route_with_store(records: []) allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) .and_return({ audit_trail: { store: store } }) allow(store).to receive_messages(list_by_record: records, count_by_record: records.length, - authors_by_record: []) + authors_by_record: [], previous_record_ids: []) route = described_class.new context = double('context', collection: collection, caller: build_caller, permissions: permissions) @@ -41,7 +41,7 @@ def route_with_store(records: []) def state_route(entries: [], record: { 'id' => 4, 'status' => 'shipped' }) allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) .and_return({ audit_trail: { store: store } }) - allow(store).to receive(:list_since).and_return(entries) + allow(store).to receive_messages(list_since: entries, previous_record_ids: []) allow(collection).to receive(:list).and_return([record].compact) route = described_class.new @@ -116,13 +116,23 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) expect(get_state(route)[:content][:data]).to eq({ 'status' => 'shipped' }) end + # Rows written before an update moved a writable primary key stay under the id they were true of. + it 'reconstructs state from every id the record has been filed under' do + route = state_route + allow(store).to receive(:previous_record_ids).and_return(['1'], [], []) + + get_state(route) + + expect(store).to have_received(:list_since).with(hash_including(record_id: %w[4 1])) + end + # Strictly after the requested instant: an entry stamped exactly at it belongs to that state. it 'asks the store for entries strictly newer than the instant' do route = state_route get_state(route) expect(store).to have_received(:list_since).with( - collection: 'projects', record_id: '4', timestamp: '2026-01-02T10:00:00.000Z' + collection: 'projects', record_id: ['4'], timestamp: '2026-01-02T10:00:00.000Z' ) end @@ -199,9 +209,9 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) ) expect(store).to have_received(:list_by_record).with( - collection: 'projects', record_id: '4', skip: 0, limit: 20, order: 'desc' + collection: 'projects', record_id: ['4'], skip: 0, limit: 20, order: 'desc' ) - expect(store).to have_received(:count_by_record).with(collection: 'projects', record_id: '4') + expect(store).to have_received(:count_by_record).with(collection: 'projects', record_id: ['4']) # Top-level keys are camelCased for the frontend; nested value hashes keep the column names. expect(result[:content]).to eq( { @@ -246,6 +256,29 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) expect(result[:content][:data]).to eq([{ 'operation' => 'delete', 'recordId' => '4' }]) end + # Rows written before an update moved a writable primary key stay under the id they were true of, so + # asking for the current id alone would start the story at the rename. + describe 'a record that was renamed' do + it 'reads the history of every id it has been filed under' do + route = route_with_store + allow(store).to receive(:previous_record_ids).and_return(['1'], [], []) + + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) + + expect(store).to have_received(:list_by_record).with(hash_including(record_id: %w[4 1])) + expect(store).to have_received(:count_by_record).with(hash_including(record_id: %w[4 1])) + end + + it 'stops walking rather than looping on a chain that comes back to itself' do + route = route_with_store + allow(store).to receive(:previous_record_ids).and_return(['1'], ['4'], ['1']) + + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) + + expect(store).to have_received(:list_by_record).with(hash_including(record_id: %w[4 1])) + end + end + describe 'meta.availableUsers' do let(:authors) do [{ user_id: 12, user_first_name: 'Ada', user_last_name: 'L', user_email: 'ada@test' }] @@ -262,7 +295,7 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) expect(result[:content][:meta][:availableUsers]).to eq( [{ id: 12, firstName: 'Ada', lastName: 'L', email: 'ada@test' }] ) - expect(store).to have_received(:authors_by_record).with(collection: 'projects', record_id: '4') + expect(store).to have_received(:authors_by_record).with(collection: 'projects', record_id: ['4']) end # The front keeps the list it saw, so later pages leave it out. From 22cdb7b865458548f9e755a6e068fd2e1dba1fe7 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Tue, 18 Aug 2026 14:40:46 +0200 Subject: [PATCH 31/35] fix(audit_trail): eight review findings on the search and rename work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - a value holding a quote, a backslash or a newline was unfindable: the value objects are matched as serialized JSON, where those are escaped, so the term is escaped the same way. It also stops a bare quote matching the document's own structure, and so every row. - `insert_all ... RETURNING` order is not promised by Postgres, and the ids were paired with their rows positionally — one pending row could be confirmed with another record's diff. Matched by `record_id` now, which is unique within a batch, falling back to one insert per row when it is not. - an earlier id in a rename chain is now bounded by the moment it was left: a primary key a record abandons can be taken by another record, whose rows are none of this record's business. - the rename walk has no depth cap: subtracting the ids already held is both the cycle guard and the terminator, where a cap only truncated real chains. - `connect_to` puts the check, the connect and the assignment under one class-level mutex, so two stores racing cannot both pass the check with the loser writing to the winner's database. - `sanitize_url` strips credentials off a scheme-relative `//user:pass@host` too, which parses fine and kept them. - the action docs claimed recording is always best-effort, which stopped being true with `critical`. - documents that a row attached to no record — a global or over-cap action run — is recorded but not readable through any route today, and what the rename bound still cannot separate. Co-Authored-By: Claude Opus 5 (1M context) --- packages/forest_admin_agent/AUDIT_TRAIL.md | 26 ++++++-- .../audit_trail/action_capture.rb | 4 +- .../audit_trail/sql/audit_connection_base.rb | 30 ++++++--- .../audit_trail/sql/text_search.rb | 22 ++++++- .../forest_admin_agent/audit_trail/store.rb | 63 +++++++++++++++---- .../routes/resources/audit_trail.rb | 6 +- .../routes/resources/audit_trail_route.rb | 40 +++++++----- .../audit_trail/store_spec.rb | 59 +++++++++++++++-- .../routes/resources/audit_trail_spec.rb | 33 ++++++---- 9 files changed, 216 insertions(+), 67 deletions(-) diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index 54d943cba..545a83786 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -160,9 +160,16 @@ Forest's own and so are camelCase (`mimeType`, not `mime_type`) — the agent tr **A record that was renamed keeps one timeline.** An update that moves a writable primary key files its row under the record's new id — the id later lookups use — and remembers the one it left. Both the history and the -state routes walk that chain back, so asking for the current id returns everything the record has ever been -filed under, rather than starting the story at the rename. The chain is walked at most ten hops, and a cycle -stops it. +state routes walk that back, so asking for the current id returns everything the record has ever been filed +under, rather than starting the story at the rename. + +Each earlier id counts only **up to the moment it was left**, because a primary key a record abandons can be +taken by another record afterwards, and those rows are none of this record's business. What the trail cannot +separate is the opposite case: rows written under an id *before* the record that holds it now arrived — a +reused key, or a delete followed by a recreate — since the packed id is the only identity the trail has. That +is deliberate for delete/recreate (the state reconstruction walks into an earlier life on purpose) and the +same limitation for a reused key. Telling those apart needs a lineage of its own on every row, which is a +bigger change than this one. A record that no longer exists keeps its history: only a record that still exists *outside* the caller's permission scope is refused (404). Inspecting what was deleted is much of the point of an @@ -241,8 +248,17 @@ Because these are the same columns the field filter searches, filtering by a fie A **global** action targets no record, and a **select-all** selection only tells the agent which ids were *excluded*, so naming the targets would mean querying the whole selection: those runs are recorded once, -attached to no record. Recording is best-effort — a failing audit database logs an error rather than -breaking the action. +attached to no record. + +Recording follows the same policy as a write: the row goes in before the action runs, so under +`critical: false` a failing audit database logs an error and the action goes ahead, while under +`critical: true` it refuses the run — nothing has happened yet, so there is nothing to repair. Everything +after that point, the answer included, is best-effort either way. + +**A row attached to no record is not readable through any route today.** Every history route is scoped to a +record id, and the correlation routes reject an empty one, so global and over-cap action runs are recorded but +cannot be fetched. They are evidence in the table rather than something the UI can show — reaching them needs +a collection-level endpoint that does not exist yet. The targeted records are read back through the caller's own filter rather than taken from the request: the ids a client sends are a claim, and in a compliance record asserting that an operator acted on a record their diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb index 412d77b23..0adaef2cd 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb @@ -82,7 +82,9 @@ def summarize(result) # `userinfo = nil` is a no-op on URI, so the credentials come off textually; the parser then takes care # of the query and fragment. def sanitize_url(value) - bare = value.to_s.sub(%r{\A([a-z][a-z0-9+.-]*://)[^/@]*@}i, '\1') + # Both `https://user:pass@host` and the scheme-relative `//user:pass@host`, which parses fine and + # would otherwise keep its credentials. + bare = value.to_s.sub(%r{\A([a-z][a-z0-9+.-]*:)?//[^/@]*@}i, '\1//') uri = URI.parse(bare) uri.query = nil uri.fragment = nil diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb index 061ea9047..446a4d47d 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/audit_connection_base.rb @@ -19,20 +19,32 @@ class << self # clobber each other's pool and both end up writing to whichever connected last. One audit database # per agent is the supported shape; a second, different one is a configuration mistake worth hearing # about at boot. + # + # Under one mutex for all stores, not one each: the check, the connect and the assignment have to be + # one step, or two stores connecting at once both pass the check and the loser writes to the winner's + # database. def connect_to(database) - if @database && @database != database - raise ForestAdminDatasourceToolkit::Exceptions::ForestException, - 'The audit trail is already connected to another database. One agent, one audit database.' - end - return if @database + connection_mutex.synchronize do + if @database && @database != database + raise ForestAdminDatasourceToolkit::Exceptions::ForestException, + 'The audit trail is already connected to another database. One agent, one audit database.' + end + next if @database - establish_connection(database) - @database = database + establish_connection(database) + @database = database + end end def disconnect! - @database = nil - remove_connection + connection_mutex.synchronize do + @database = nil + remove_connection + end + end + + def connection_mutex + @connection_mutex ||= Mutex.new end end end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/text_search.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/text_search.rb index bc0509789..d8404e9d1 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/text_search.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/sql/text_search.rb @@ -1,3 +1,5 @@ +require 'json' + module ForestAdminAgent module AuditTrail module Sql @@ -28,14 +30,28 @@ def initialize(connection) end def condition(term) - pattern = @connection.quote("%#{escape(term.to_s.downcase)}%") - columns = TEXT_COLUMNS.map { |column| column } + JSON_COLUMNS.map { |column| searchable_json(column) } + text = term.to_s.downcase + # The value objects are matched as serialized JSON, where a quote, a backslash or a newline is + # escaped — so `15" monitor` sits in the document as `15\" monitor` and the raw term would never + # find it. Escaping the term the same way makes it match, and stops a bare quote from matching the + # document's own structure. + clauses = TEXT_COLUMNS.map { |column| like(column, text) } + clauses += JSON_COLUMNS.map { |column| like(searchable_json(column), json_escaped(text)) } - columns.map { |expression| "LOWER(#{expression}) LIKE #{pattern} ESCAPE '#{ESCAPE}'" }.join(' OR ') + clauses.join(' OR ') end private + def like(expression, text) + "LOWER(#{expression}) LIKE #{@connection.quote("%#{escape(text)}%")} ESCAPE '#{ESCAPE}'" + end + + # What JSON generation would have done to the term: `to_json` on the string, minus its own quotes. + def json_escaped(text) + text.to_json[1..-2] + end + def searchable_json(column) "REPLACE(#{as_text(column)}, #{@connection.quote(REDACTED)}, '')" end diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb index d4a2b5dbd..104b3e888 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb @@ -35,15 +35,30 @@ def append(record) # Inserts rows and returns their ids, in the order given. Batched, because a "delete all" snapshot can # be thousands of records and the pending/confirm protocol writes each of them twice. + # + # The ids are matched to their rows by `record_id` rather than by the order RETURNING happens to come + # back in, which Postgres does not promise: pairing them positionally would confirm each pending row with + # another record's diff. One row per record per operation, so that key is unique within a batch — bar a + # pending create, which has no id yet and is always a batch of one. def append_all(records) return [] if records.empty? rows = records.map { |record| to_row(record) } + return rows.map { |row| model.create!(row).id } unless batch_returning?(rows) - return model.insert_all(rows, returning: %w[id]).rows.flatten if model.connection.supports_insert_returning? + returned = model.insert_all(rows, returning: %i[id record_id]).rows.to_h { |id, key| [key, id] } - # MySQL has no RETURNING: fall back to one insert per row, which does hand the id back. - rows.map { |row| model.create!(row).id } + rows.map { |row| returned[row[:record_id]] } + end + + # One insert per row when the ids cannot be matched back: no RETURNING on this adapter (MySQL), or a + # batch whose record ids are not distinct enough to pair on. + def batch_returning?(rows) + return false unless model.connection.supports_insert_returning? + + keys = rows.map { |row| row[:record_id] } + + keys.none?(&:nil?) && keys.uniq.size == keys.size end def confirm(id, attributes) @@ -91,14 +106,16 @@ def authors_by_record(collection:, record_id:, user_ids: nil, start_timestamp: n .uniq { |author| author[:user_id] } end - # The id this record was filed under before an update moved its primary key, if one did. Walking that - # back is what lets a history query reach rows written before a rename — they stay under the id they were - # written with, since that is the id they were true of. - def previous_record_ids(collection:, record_id:) + # The ids this record was renamed from, each with the moment it stopped being that id. Walking those back + # is what lets a history query reach rows written before a rename — they stay under the id they were + # written with, since that is the id they were true of — and the moment bounds how far: the id it left may + # have been taken by another record afterwards, whose rows are none of this record's business. + def renamed_from(collection:, record_id:) model.where(collection: collection, record_id: record_id) .where.not(previous_record_id: nil) - .distinct - .pluck(:previous_record_id) + .pluck(:previous_record_id, :timestamp) + .group_by(&:first) + .map { |id, rows| { id: id, until: as_iso(rows.map(&:last).max) } } end # Entries recorded strictly after `timestamp`, newest first: what a state reconstruction has to undo. @@ -109,7 +126,8 @@ def previous_record_ids(collection:, record_id:) # pending rows — they are evidence, and `status` tells the reader what they are — but a reconstruction # cannot act on them. def list_since(collection:, record_id:, timestamp:) - model.where(collection: collection, record_id: record_id, status: Recording::DONE) + model.where(collection: collection, status: Recording::DONE) + .where(*segments_condition(record_id)) .where('timestamp > ?', as_time(timestamp)) .order(timestamp: :desc, id: :desc) .map { |row| from_row(row) } @@ -132,7 +150,7 @@ def list_by_correlations(collection:, record_id:, correlation_keys:) # Every filter is an AND, so the count matches exactly what a page of this history holds. def scope(collection, record_id, user_ids: nil, start_timestamp: nil, end_timestamp: nil, fields: nil, search: nil) - relation = model.where(collection: collection, record_id: record_id) + relation = model.where(collection: collection).where(*segments_condition(record_id)) relation = relation.where(user_id: user_ids) if user_ids relation = relation.where(Sql::FieldFilter.new(model.connection).condition(fields)) if fields&.any? relation = relation.where(Sql::TextSearch.new(model.connection).condition(search)) if search @@ -147,6 +165,29 @@ def as_time(value) value.is_a?(::Time) ? value : ::Time.iso8601(value.to_s) end + def as_iso(value) + value.respond_to?(:iso8601) ? value.iso8601(3) : value.to_s + end + + # One record's history is its current id plus every id it was renamed from, each earlier one only up to + # the rename: `record_id = '7' OR (record_id = '1' AND timestamp <= …)`. A plain `IN` would hand over the + # rows of whichever record holds that id now. + # + # Takes an id, several, or segments — `{ id:, until: }` — so a caller that has no rename to care about + # simply passes the id. + def segments_condition(record_id) + binds = [] + sql = Array(record_id).map { |value| value.is_a?(Hash) ? value : { id: value, until: nil } }.map do |segment| + binds << segment[:id] + next 'record_id = ?' unless segment[:until] + + binds << as_time(segment[:until]) + '(record_id = ? AND timestamp <= ?)' + end + + [sql.join(' OR '), *binds] + end + def model ensure_ready @model diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb index 37a4df6ea..b34109550 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb @@ -45,8 +45,8 @@ def handle_request(args = {}) filters = { collection: context.collection.name, # args[:params]['id'] is already Forest's packed id, the form the audit store keys on — plus any id - # this record was filed under before a rename. - record_id: record_ids_history(context.collection, args[:params]['id']), + # this record was filed under before a rename, each bounded by when it stopped being that id. + record_id: record_segments(context.collection, args[:params]['id']), **parse_filters(args) } @@ -74,7 +74,7 @@ def handle_state(args = {}) timestamp = parse_state_timestamp(args) entries = store.list_since( collection: context.collection.name, - record_id: record_ids_history(context.collection, args[:params]['id']), + record_id: record_segments(context.collection, args[:params]['id']), timestamp: timestamp ) # Fully qualified: inside this class, `AuditTrail` is the route itself. diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb index 9859c65ea..10f40f303 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb @@ -50,25 +50,33 @@ def store ::ForestAdminAgent::AuditTrail.store end - # Every id this record has been filed under, newest first. Rows written before an update moved a - # writable primary key stay under the id they were true of, so a history query that asked for the - # current id alone would start at the rename and call that the whole story. + # Every id this record has been filed under, each with the moment it stopped being that id. Rows + # written before an update moved a writable primary key stay under the id they were true of, so a + # history query that asked for the current id alone would start at the rename and call that the whole + # story — and one that asked for the bare ids would sweep up whatever record holds an abandoned id now. # - # ponytail: 10 hops is far more renaming than any record sees; the guard is against a cycle, not depth. - MAX_RENAMES = 10 - - def record_ids_history(collection, packed_id) - ids = [packed_id] - - MAX_RENAMES.times do - previous = store.previous_record_ids(collection: collection.name, record_id: ids.last) - previous -= ids - break if previous.empty? - - ids.concat(previous) + # Breadth-first, and no depth limit: every hop adds an id not already seen and there are finitely many + # of those, so skipping what we hold is both the cycle guard and the terminator. A cap would have + # truncated a record renamed often enough, which reads exactly like missing history. + def record_segments(collection, packed_id) + segments = [{ id: packed_id, until: nil }] + queue = segments.dup + + until queue.empty? + segment = queue.shift + + store.renamed_from(collection: collection.name, record_id: segment[:id]).each do |previous| + next if segments.any? { |seen| seen[:id] == previous[:id] } + + # Bounded by its own rename and by everything walked through to reach it: an id abandoned twice + # only belongs to this record up to the earlier of them. + found = { id: previous[:id], until: [previous[:until], segment[:until]].compact.min } + segments << found + queue << found + end end - ids + segments end # What the audit trail actually records: primary keys, so a state can be identified, plus the writable diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb index fa9ba38c0..62193d8f5 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb @@ -287,19 +287,48 @@ def connection end end - describe '#previous_record_ids' do - it 'returns the ids this record was filed under before a rename, once each' do - store.append(record(record_id: '7', previous_record_id: '1')) - store.append(record(record_id: '7', previous_record_id: '1')) + describe '#renamed_from' do + it 'returns each id it was renamed from, with the moment it stopped being that id' do + store.append(record(record_id: '7', previous_record_id: '1', timestamp: '2026-01-02T00:00:01.000Z')) + store.append(record(record_id: '7', previous_record_id: '1', timestamp: '2026-01-02T00:00:09.000Z')) store.append(record(record_id: '7')) - expect(store.previous_record_ids(collection: 'accounts', record_id: '7')).to eq(['1']) + expect(store.renamed_from(collection: 'accounts', record_id: '7')).to eq( + [{ id: '1', until: '2026-01-02T00:00:09.000Z' }] + ) end it 'returns nothing for a record that was never renamed' do store.append(record) - expect(store.previous_record_ids(collection: 'accounts', record_id: '1')).to be_empty + expect(store.renamed_from(collection: 'accounts', record_id: '1')).to be_empty + end + end + + # An id a record left may since have been taken by another record: its rows are none of this one's + # business, so an earlier segment only reaches up to the rename. + describe 'reading a renamed record' do + it 'takes the earlier id only up to the moment it was left' do + store.append(record(record_id: '1', timestamp: '2026-01-02T00:00:01.000Z', correlation_key: 'mine')) + store.append(record(record_id: '7', timestamp: '2026-01-02T00:00:05.000Z', correlation_key: 'renamed', + previous_record_id: '1')) + store.append(record(record_id: '1', timestamp: '2026-01-02T00:00:09.000Z', correlation_key: 'someone-else')) + + history = store.list_by_record( + collection: 'accounts', + record_id: [{ id: '7', until: nil }, { id: '1', until: '2026-01-02T00:00:05.000Z' }] + ) + + expect(history.map(&:correlation_key)).to eq(%w[mine renamed]) + end + + it 'counts the same rows it lists' do + store.append(record(record_id: '1', timestamp: '2026-01-02T00:00:01.000Z')) + store.append(record(record_id: '1', timestamp: '2026-01-02T00:00:09.000Z')) + + segments = [{ id: '1', until: '2026-01-02T00:00:05.000Z' }] + + expect(store.count_by_record(collection: 'accounts', record_id: segments)).to eq(1) end end @@ -376,6 +405,24 @@ def matching_search(term) expect(matching_search('%')).to eq(['literal']) end + # The values sit in a serialized document, where a quote is escaped: the raw term would never find them. + it 'finds a value holding a quote, a backslash or a newline' do + store.append(record(correlation_key: 'quote', new_values: { 'label' => '15" monitor' })) + store.append(record(correlation_key: 'path', new_values: { 'path' => 'C:\\Users\\ada' })) + store.append(record(correlation_key: 'multiline', new_values: { 'note' => "first\nsecond" })) + + expect(matching_search('15" monitor')).to eq(['quote']) + expect(matching_search('C:\\Users')).to eq(['path']) + expect(matching_search("first\nsecond")).to eq(['multiline']) + end + + # A bare quote would otherwise match the document's own structure, so every row. + it 'does not let a bare quote match every row' do + store.append(record(new_values: { 'city' => 'Lyon' })) + + expect(matching_search('"')).to be_empty + end + it 'composes with the other filters, and the count agrees with it' do store.append(record(correlation_key: 'keep', user_id: 7, new_values: { 'city' => 'Lyon' })) store.append(record(correlation_key: 'wrong-user', user_id: 9, new_values: { 'city' => 'Lyon' })) diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb index 06e054df5..cb2ee896d 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -29,7 +29,7 @@ def route_with_store(records: []) allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) .and_return({ audit_trail: { store: store } }) allow(store).to receive_messages(list_by_record: records, count_by_record: records.length, - authors_by_record: [], previous_record_ids: []) + authors_by_record: [], renamed_from: []) route = described_class.new context = double('context', collection: collection, caller: build_caller, permissions: permissions) @@ -41,7 +41,7 @@ def route_with_store(records: []) def state_route(entries: [], record: { 'id' => 4, 'status' => 'shipped' }) allow(ForestAdminAgent::Facades::Container).to receive(:config_from_cache) .and_return({ audit_trail: { store: store } }) - allow(store).to receive_messages(list_since: entries, previous_record_ids: []) + allow(store).to receive_messages(list_since: entries, renamed_from: []) allow(collection).to receive(:list).and_return([record].compact) route = described_class.new @@ -119,11 +119,12 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) # Rows written before an update moved a writable primary key stay under the id they were true of. it 'reconstructs state from every id the record has been filed under' do route = state_route - allow(store).to receive(:previous_record_ids).and_return(['1'], [], []) + allow(store).to receive(:renamed_from).and_return([{ id: '1', until: nil }], []) get_state(route) - expect(store).to have_received(:list_since).with(hash_including(record_id: %w[4 1])) + expect(store).to have_received(:list_since) + .with(hash_including(record_id: [{ id: '4', until: nil }, { id: '1', until: nil }])) end # Strictly after the requested instant: an entry stamped exactly at it belongs to that state. @@ -132,7 +133,8 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) get_state(route) expect(store).to have_received(:list_since).with( - collection: 'projects', record_id: ['4'], timestamp: '2026-01-02T10:00:00.000Z' + collection: 'projects', record_id: [{ id: '4', until: nil }], + timestamp: '2026-01-02T10:00:00.000Z' ) end @@ -209,9 +211,10 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) ) expect(store).to have_received(:list_by_record).with( - collection: 'projects', record_id: ['4'], skip: 0, limit: 20, order: 'desc' + collection: 'projects', record_id: [{ id: '4', until: nil }], skip: 0, limit: 20, order: 'desc' ) - expect(store).to have_received(:count_by_record).with(collection: 'projects', record_id: ['4']) + expect(store).to have_received(:count_by_record) + .with(collection: 'projects', record_id: [{ id: '4', until: nil }]) # Top-level keys are camelCased for the frontend; nested value hashes keep the column names. expect(result[:content]).to eq( { @@ -261,21 +264,24 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) describe 'a record that was renamed' do it 'reads the history of every id it has been filed under' do route = route_with_store - allow(store).to receive(:previous_record_ids).and_return(['1'], [], []) + allow(store).to receive(:renamed_from) + .and_return([{ id: '1', until: '2026-01-02T00:00:05.000Z' }], []) route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) - expect(store).to have_received(:list_by_record).with(hash_including(record_id: %w[4 1])) - expect(store).to have_received(:count_by_record).with(hash_including(record_id: %w[4 1])) + expected = [{ id: '4', until: nil }, { id: '1', until: '2026-01-02T00:00:05.000Z' }] + expect(store).to have_received(:list_by_record).with(hash_including(record_id: expected)) + expect(store).to have_received(:count_by_record).with(hash_including(record_id: expected)) end it 'stops walking rather than looping on a chain that comes back to itself' do route = route_with_store - allow(store).to receive(:previous_record_ids).and_return(['1'], ['4'], ['1']) + allow(store).to receive(:renamed_from).and_return([{ id: '1', until: nil }], [{ id: '4', until: nil }]) route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) - expect(store).to have_received(:list_by_record).with(hash_including(record_id: %w[4 1])) + expect(store).to have_received(:list_by_record) + .with(hash_including(record_id: [{ id: '4', until: nil }, { id: '1', until: nil }])) end end @@ -295,7 +301,8 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) expect(result[:content][:meta][:availableUsers]).to eq( [{ id: 12, firstName: 'Ada', lastName: 'L', email: 'ada@test' }] ) - expect(store).to have_received(:authors_by_record).with(collection: 'projects', record_id: ['4']) + expect(store).to have_received(:authors_by_record) + .with(collection: 'projects', record_id: [{ id: '4', until: nil }]) end # The front keeps the list it saw, so later pages leave it out. From 9bb4c5ce68fd2ab486eb652edb8037f5bfb294aa Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Tue, 18 Aug 2026 14:53:16 +0200 Subject: [PATCH 32/35] fix(audit_trail): bound a rename segment by (timestamp, row id), as the trail orders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bound knew only the timestamp, so a record taking an abandoned key inside the same millisecond as the rename landed in the previous occupant's history. The trail orders itself by (timestamp, id) — a bound in timestamps alone was a slightly different notion of "before" than the one it sorts by. `renamed_from` carries the rename row's id along, and a bounded segment reads `timestamp < t OR (timestamp = t AND id <= row)`. A caller that supplies only a timestamp still gets the inclusive form, so the simple case stays simple. Co-Authored-By: Claude Opus 5 (1M context) --- .../forest_admin_agent/audit_trail/store.rb | 17 ++++++++-- .../routes/resources/audit_trail_route.rb | 15 +++++++-- .../audit_trail/store_spec.rb | 22 ++++++++++++- .../routes/resources/audit_trail_spec.rb | 31 ++++++++++++------- 4 files changed, 67 insertions(+), 18 deletions(-) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb index 104b3e888..dd5d1272b 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb @@ -113,9 +113,15 @@ def authors_by_record(collection:, record_id:, user_ids: nil, start_timestamp: n def renamed_from(collection:, record_id:) model.where(collection: collection, record_id: record_id) .where.not(previous_record_id: nil) - .pluck(:previous_record_id, :timestamp) + .pluck(:previous_record_id, :timestamp, :id) .group_by(&:first) - .map { |id, rows| { id: id, until: as_iso(rows.map(&:last).max) } } + .map do |id, rows| + # The row id comes along as the tie-breaker: the trail orders itself by (timestamp, id), so a + # bound that knew only the timestamp would mean something slightly different from "before". + _, at, row = rows.max_by { |(_, timestamp, row_id)| [timestamp, row_id] } + + { id: id, until: as_iso(at), until_row: row } + end end # Entries recorded strictly after `timestamp`, newest first: what a state reconstruction has to undo. @@ -182,7 +188,12 @@ def segments_condition(record_id) next 'record_id = ?' unless segment[:until] binds << as_time(segment[:until]) - '(record_id = ? AND timestamp <= ?)' + # Same millisecond as the rename, and the id says which side of it a row falls on: another record + # taking the abandoned key that fast would otherwise land in this record's history. + next '(record_id = ? AND timestamp <= ?)' unless segment[:until_row] + + binds << as_time(segment[:until]) << segment[:until_row] + '(record_id = ? AND (timestamp < ? OR (timestamp = ? AND id <= ?)))' end [sql.join(' OR '), *binds] diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb index 10f40f303..06557568e 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb @@ -59,7 +59,7 @@ def store # of those, so skipping what we hold is both the cycle guard and the terminator. A cap would have # truncated a record renamed often enough, which reads exactly like missing history. def record_segments(collection, packed_id) - segments = [{ id: packed_id, until: nil }] + segments = [{ id: packed_id, until: nil, until_row: nil }] queue = segments.dup until queue.empty? @@ -70,7 +70,7 @@ def record_segments(collection, packed_id) # Bounded by its own rename and by everything walked through to reach it: an id abandoned twice # only belongs to this record up to the earlier of them. - found = { id: previous[:id], until: [previous[:until], segment[:until]].compact.min } + found = earlier_bound(previous, segment).merge(id: previous[:id]) segments << found queue << found end @@ -79,6 +79,17 @@ def record_segments(collection, packed_id) segments end + # Bounds compare as the trail orders, (timestamp, row id); nil is "no bound yet", which is later than + # any of them. + def earlier_bound(one, other) + return other.slice(:until, :until_row) if one[:until].nil? + return one.slice(:until, :until_row) if other[:until].nil? + + pair = ->(bound) { [bound[:until], bound[:until_row].to_i] } + + (pair[one] <= pair[other] ? one : other).slice(:until, :until_row) + end + # What the audit trail actually records: primary keys, so a state can be identified, plus the writable # columns. Reading read-only ones would hand them back at their present value inside an answer that # claims to describe a past instant. diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb index 62193d8f5..7f9002a6d 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/store_spec.rb @@ -293,9 +293,13 @@ def connection store.append(record(record_id: '7', previous_record_id: '1', timestamp: '2026-01-02T00:00:09.000Z')) store.append(record(record_id: '7')) - expect(store.renamed_from(collection: 'accounts', record_id: '7')).to eq( + renamed = store.renamed_from(collection: 'accounts', record_id: '7') + + expect(renamed.map { |segment| segment.slice(:id, :until) }).to eq( [{ id: '1', until: '2026-01-02T00:00:09.000Z' }] ) + # The row id of that rename, so an equal-timestamp row can be placed either side of it. + expect(renamed.first[:until_row]).to be_a(Integer) end it 'returns nothing for a record that was never renamed' do @@ -322,6 +326,22 @@ def connection expect(history.map(&:correlation_key)).to eq(%w[mine renamed]) end + # Same millisecond as the rename: the trail orders by (timestamp, id), so the bound does too. + it 'places a row written in the very millisecond of the rename by insertion order' do + mine = store.append(record(record_id: '1', timestamp: '2026-01-02T00:00:05.000Z', correlation_key: 'mine')) + rename = store.append(record(record_id: '7', timestamp: '2026-01-02T00:00:05.000Z', + previous_record_id: '1', correlation_key: 'renamed')) + store.append(record(record_id: '1', timestamp: '2026-01-02T00:00:05.000Z', correlation_key: 'theirs')) + + history = store.list_by_record( + collection: 'accounts', + record_id: [{ id: '7', until: nil }, { id: '1', until: '2026-01-02T00:00:05.000Z', until_row: rename }] + ) + + expect(history.map(&:correlation_key)).to eq(%w[mine renamed]) + expect(mine).to be < rename + end + it 'counts the same rows it lists' do store.append(record(record_id: '1', timestamp: '2026-01-02T00:00:01.000Z')) store.append(record(record_id: '1', timestamp: '2026-01-02T00:00:09.000Z')) diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb index cb2ee896d..ce0f02988 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -119,12 +119,14 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) # Rows written before an update moved a writable primary key stay under the id they were true of. it 'reconstructs state from every id the record has been filed under' do route = state_route - allow(store).to receive(:renamed_from).and_return([{ id: '1', until: nil }], []) + allow(store).to receive(:renamed_from).and_return([{ id: '1', until: nil, until_row: nil }], []) get_state(route) - expect(store).to have_received(:list_since) - .with(hash_including(record_id: [{ id: '4', until: nil }, { id: '1', until: nil }])) + expect(store).to have_received(:list_since).with( + hash_including(record_id: [{ id: '4', until: nil, until_row: nil }, + { id: '1', until: nil, until_row: nil }]) + ) end # Strictly after the requested instant: an entry stamped exactly at it belongs to that state. @@ -133,7 +135,7 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) get_state(route) expect(store).to have_received(:list_since).with( - collection: 'projects', record_id: [{ id: '4', until: nil }], + collection: 'projects', record_id: [{ id: '4', until: nil, until_row: nil }], timestamp: '2026-01-02T10:00:00.000Z' ) end @@ -211,10 +213,10 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) ) expect(store).to have_received(:list_by_record).with( - collection: 'projects', record_id: [{ id: '4', until: nil }], skip: 0, limit: 20, order: 'desc' + collection: 'projects', record_id: [{ id: '4', until: nil, until_row: nil }], skip: 0, limit: 20, order: 'desc' ) expect(store).to have_received(:count_by_record) - .with(collection: 'projects', record_id: [{ id: '4', until: nil }]) + .with(collection: 'projects', record_id: [{ id: '4', until: nil, until_row: nil }]) # Top-level keys are camelCased for the frontend; nested value hashes keep the column names. expect(result[:content]).to eq( { @@ -265,23 +267,28 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) it 'reads the history of every id it has been filed under' do route = route_with_store allow(store).to receive(:renamed_from) - .and_return([{ id: '1', until: '2026-01-02T00:00:05.000Z' }], []) + .and_return([{ id: '1', until: '2026-01-02T00:00:05.000Z', until_row: 12 }], []) route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) - expected = [{ id: '4', until: nil }, { id: '1', until: '2026-01-02T00:00:05.000Z' }] + expected = [{ id: '4', until: nil, until_row: nil }, + { id: '1', until: '2026-01-02T00:00:05.000Z', until_row: 12 }] expect(store).to have_received(:list_by_record).with(hash_including(record_id: expected)) expect(store).to have_received(:count_by_record).with(hash_including(record_id: expected)) end it 'stops walking rather than looping on a chain that comes back to itself' do route = route_with_store - allow(store).to receive(:renamed_from).and_return([{ id: '1', until: nil }], [{ id: '4', until: nil }]) + allow(store).to receive(:renamed_from).and_return( + [{ id: '1', until: nil, until_row: nil }], [{ id: '4', until: nil, until_row: nil }] + ) route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '4' } }) - expect(store).to have_received(:list_by_record) - .with(hash_including(record_id: [{ id: '4', until: nil }, { id: '1', until: nil }])) + expect(store).to have_received(:list_by_record).with( + hash_including(record_id: [{ id: '4', until: nil, until_row: nil }, + { id: '1', until: nil, until_row: nil }]) + ) end end @@ -302,7 +309,7 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) [{ id: 12, firstName: 'Ada', lastName: 'L', email: 'ada@test' }] ) expect(store).to have_received(:authors_by_record) - .with(collection: 'projects', record_id: [{ id: '4', until: nil }]) + .with(collection: 'projects', record_id: [{ id: '4', until: nil, until_row: nil }]) end # The front keeps the list it saw, so later pages leave it out. From bf862d89364cde247ee4b1ebec076dcc4c3a6345 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Tue, 18 Aug 2026 15:01:22 +0200 Subject: [PATCH 33/35] fix(audit_trail): compare rename bounds through <=>, since Array is not Comparable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pair[one] <= pair[other]` raised NoMethodError, so a record renamed twice took down the history and state routes rather than merely mis-bounding them. Array has `<=>` but none of the operators Comparable would have added. The specs never reached it: every case had at most one bounded segment, the other being the root's nil bound, which short-circuits before the comparison. Two now walk a chain of two renames — one where the older segment keeps its own bound, one where it inherits the earlier of the two — and they fail with the original NoMethodError against the previous code. Co-Authored-By: Claude Opus 5 (1M context) --- .../routes/resources/audit_trail_route.rb | 5 ++- .../routes/resources/audit_trail_spec.rb | 41 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb index 06557568e..550e23fee 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail_route.rb @@ -80,14 +80,15 @@ def record_segments(collection, packed_id) end # Bounds compare as the trail orders, (timestamp, row id); nil is "no bound yet", which is later than - # any of them. + # any of them. Through `<=>`, since Array is not Comparable and `<=` on one raises. def earlier_bound(one, other) return other.slice(:until, :until_row) if one[:until].nil? return one.slice(:until, :until_row) if other[:until].nil? pair = ->(bound) { [bound[:until], bound[:until_row].to_i] } + earlier = (pair[one] <=> pair[other]) <= 0 ? one : other - (pair[one] <= pair[other] ? one : other).slice(:until, :until_row) + earlier.slice(:until, :until_row) end # What the audit trail actually records: primary keys, so a state can be identified, plus the writable diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb index ce0f02988..f45216649 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb @@ -277,6 +277,47 @@ def get_state(route, timestamp: '2026-01-02T10:00:00.000Z', extra: {}) expect(store).to have_received(:count_by_record).with(hash_including(record_id: expected)) end + # Two hops means two real bounds to compare, which is the path a single rename never reaches. + it 'carries the earlier bound down a chain of two renames' do + route = route_with_store + allow(store).to receive(:renamed_from).and_return( + [{ id: '7', until: '2026-01-02T00:00:09.000Z', until_row: 20 }], + [{ id: '1', until: '2026-01-02T00:00:05.000Z', until_row: 10 }], + [] + ) + + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '9' } }) + + expect(store).to have_received(:list_by_record).with( + hash_including(record_id: [ + { id: '9', until: nil, until_row: nil }, + { id: '7', until: '2026-01-02T00:00:09.000Z', until_row: 20 }, + { id: '1', until: '2026-01-02T00:00:05.000Z', until_row: 10 } + ]) + ) + end + + # The middle id was left later than the one before it, so the older segment keeps its own, earlier + # bound rather than inheriting the looser one. + it 'keeps the earlier of the two bounds when the chain reports a later one' do + route = route_with_store + allow(store).to receive(:renamed_from).and_return( + [{ id: '7', until: '2026-01-02T00:00:05.000Z', until_row: 10 }], + [{ id: '1', until: '2026-01-02T00:00:09.000Z', until_row: 20 }], + [] + ) + + route.handle_request({ headers: {}, params: { 'collection_name' => 'projects', 'id' => '9' } }) + + expect(store).to have_received(:list_by_record).with( + hash_including(record_id: [ + { id: '9', until: nil, until_row: nil }, + { id: '7', until: '2026-01-02T00:00:05.000Z', until_row: 10 }, + { id: '1', until: '2026-01-02T00:00:05.000Z', until_row: 10 } + ]) + ) + end + it 'stops walking rather than looping on a chain that comes back to itself' do route = route_with_store allow(store).to receive(:renamed_from).and_return( From 87b05e9cf4f1bc591e803be5fd43e4115ce5d9a9 Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Tue, 18 Aug 2026 16:31:24 +0200 Subject: [PATCH 34/35] fix(audit_trail): one cap for both agents, and refuse an over-cap action too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cap was 500 here and 1000 in the Node agent, so a bulk operation on 700 records was fully audited on one and truncated on the other, behind the same config key. Now 1000, with a comment saying it moves in both or neither. Past the cap the two paths inside this agent also disagreed: a bulk write refused the operation under `critical: true`, while a smart action recorded one row attached to no record and carried on. A partial audit is exactly what that mode exists to refuse, and the gate sits ahead of `execute`, so refusing costs nothing to repair. The message now lives in one place, since two paths raise it. AUDIT_TRAIL.md says plainly what it only implied: `critical` defaults to false, so on a default configuration a write can succeed with no audit row — configuring the database buys a best-effort trail, `critical: true` buys the invariant. It also records that rename-chaining is Ruby-only until `previous_record_id` is ported, rather than leaving it to read as an accident. Co-Authored-By: Claude Opus 5 (1M context) --- packages/forest_admin_agent/AUDIT_TRAIL.md | 18 ++++++++++++++---- .../lib/forest_admin_agent/audit_trail.rb | 15 +++++++++++++-- .../audit_trail/snapshots.rb | 6 +----- .../routes/action/actions.rb | 5 +++++ .../routes/action/actions_spec.rb | 11 +++++++++++ 5 files changed, 44 insertions(+), 11 deletions(-) diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index 545a83786..db307a21a 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -74,15 +74,22 @@ means the write may or may not have landed: that residue is evidence, and it is the pending insert stays best-effort in both modes, because by then the write has happened and raising would report a failure for an operation that succeeded. +> **The guarantee is opt-in.** `critical` defaults to `false`, so on a default configuration a write can +> succeed with no audit row at all — an unreachable audit database costs rows, not writes. Configuring the +> database gets you a best-effort trail; `critical: true` is what makes "no unaudited write" true. The Node +> agent defaults the same way, so the two agree. + Consequences worth knowing: - A write that turns out to change nothing has its pending row **discarded** rather than confirmed, so no-op updates leave no trace. - A record the agent cannot read back after the write keeps its row **pending**. Confirming from the patch would claim values that may never have been written. -- One operation audits at most 500 records. Under `critical: false` a wider selection is truncated, with - `N records audited, M skipped` logged at `Warn`. Under `critical: true` it is **refused**: auditing a subset - while the write touches every match is precisely the invariant that mode exists for. +- One operation audits at most **1000** records — the same number as the Node agent's `MAX_SNAPSHOT_RECORDS`, + so a bulk operation is not audited on one agent and truncated on the other. Under `critical: false` a wider + selection is truncated, with `N records audited, M skipped` logged at `Warn`; a smart action over the cap is + recorded as one row attached to no record. Under `critical: true` both are **refused**: auditing a subset + while the operation touches every match is precisely the invariant that mode exists for. - Pending rows stay visible in the history — they are evidence of an attempt, and `status` says so — but the state reconstruction ignores them, since undoing a change that may never have happened would invent a state the record was never in. @@ -158,7 +165,10 @@ the merge tiebreaker. Inside the value objects: a record's column names pass through untouched, while an action answer's keys are Forest's own and so are camelCase (`mimeType`, not `mime_type`) — the agent transforms them on write. -**A record that was renamed keeps one timeline.** An update that moves a writable primary key files its row +**A record that was renamed keeps one timeline** — on this agent. The Node agent files an update under the +record's new id too, but has no `previous_record_id` and so cannot walk back past the rename: the same record +shows a complete chain here and a history beginning at the rename there, until that column is ported. + An update that moves a writable primary key files its row under the record's new id — the id later lookups use — and remembers the one it left. Both the history and the state routes walk that back, so asking for the current id returns everything the record has ever been filed under, rather than starting the story at the rename. diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb index 6ef79960f..eebf51aec 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb @@ -6,8 +6,19 @@ module AuditTrail # One operation must not materialise an unbounded number of records: a "delete all" would otherwise read # every matched row and, with the pending/confirm protocol, write each of them twice. Truncation is logged, # never silent. - # ponytail: 500 covers any hand-made bulk edit; raise it if a real workflow needs more. - MAX_RECORDS_PER_OPERATION = 500 + # + # Matches the Node agent's `MAX_SNAPSHOT_RECORDS`: the same feature behind the same config key, so a bulk + # operation must not be audited on one agent and truncated on the other. Change it in both or neither. + MAX_RECORDS_PER_OPERATION = 1000 + + # Auditing a subset while the write touches every match is the one thing `critical` exists to prevent, so + # over the cap the operation is refused instead — before anything is written, in both the write and the + # action path, which is why the message lives here rather than in either of them. + def self.refuse_over_cap! + raise ForestAdminDatasourceToolkit::Exceptions::ForestException, + 'The audit trail is configured as critical and cannot record an operation touching more than ' \ + "#{MAX_RECORDS_PER_OPERATION} records at once. Narrow the selection." + end def self.log_truncation(kept, total) skipped = total ? total - kept : 'further' diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb index fcd671c73..b6e02e910 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb @@ -49,11 +49,7 @@ def take(context, projection) # advisory; under `critical: true` it breaks the one invariant the mode exists for, so the operation is # refused instead — before the write, so there is nothing to repair. def refuse_or_truncate(context, records, cap) - if AuditTrail.critical? - raise ForestAdminDatasourceToolkit::Exceptions::ForestException, - 'The audit trail is configured as critical and cannot record an operation touching more than ' \ - "#{cap} records at once. Narrow the selection." - end + AuditTrail.refuse_over_cap! if AuditTrail.critical? kept = records.first(cap) AuditTrail.log_truncation(kept.size, audit_safely { count_matching(context) }) diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb index 0942c34d6..e10e71605 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/routes/action/actions.rb @@ -183,6 +183,11 @@ def audited_record_ids(context, filter) return records.map { |record| Utils::Id.pack_id(context.collection, record) } if records.size <= cap + # Same rule as a bulk write: recording one unattached row for a run that touched more records than + # the cap is a partial audit, which `critical` exists to refuse. Nothing has run yet — the gate is + # ahead of `execute` — so refusing costs nothing to repair. + ForestAdminAgent::AuditTrail.refuse_over_cap! if ForestAdminAgent::AuditTrail.critical? + ForestAdminAgent::AuditTrail.log_truncation(0, nil) [] end diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb index 7d94078a9..2a42a043c 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb @@ -44,6 +44,17 @@ def audited_ids end end + # Same rule as a bulk write: one unattached row for a run that touched more is a partial audit. + it 'refuses a selection wider than the cap when critical is on' do + allow(ForestAdminAgent::AuditTrail).to receive(:critical?).and_return(true) + cap = ForestAdminAgent::AuditTrail::MAX_RECORDS_PER_OPERATION + allow(collection).to receive(:list).and_return((1..(cap + 1)).map { |id| { 'id' => id } }) + + expect { audited_ids }.to raise_error( + ForestAdminDatasourceToolkit::Exceptions::ForestException, /cannot record an operation touching more/ + ) + end + it 'records a selection wider than the cap as attached to no record, and says so' do logger = instance_spy(Services::LoggerService) allow(ForestAdminAgent::Facades::Container).to receive(:logger).and_return(logger) From fcaa198ca972e3aeb96f93ac7574b6b612661b1f Mon Sep 17 00:00:00 2001 From: Olivier Bex-Chauvet Date: Tue, 18 Aug 2026 17:07:03 +0200 Subject: [PATCH 35/35] fix(audit_trail): pair a snapshot with its own operation, not the newest one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Taking the newest entry off the stack is wrong as soon as writes nest: an inner write that fails and is rescued skips its after hook and stays there, so the outer hook confirmed the failed operation's rows as done and left its own stranded — lies in both directions. Entries are keyed by the object the hook decorator hands to both contexts, the filter or the data on a create, which is the identity the two hooks genuinely share. Where a customization replaced it there is nothing to match on: with one operation in flight that is unambiguous and still pairs; with several it does not guess, and the rows stay pending, which is what pending means. The specs built a fresh filter per hook call, so they modelled something production never does — that is why the case was reachable. They share one object per operation now, and three examples cover the nested failure, the replaced filter, and the ambiguous pair. Co-Authored-By: Claude Opus 5 (1M context) --- packages/forest_admin_agent/AUDIT_TRAIL.md | 5 ++ .../forest_admin_agent/audit_trail/capture.rb | 15 ++-- .../audit_trail/snapshots.rb | 26 +++++-- .../audit_trail/capture_spec.rb | 73 ++++++++++++++----- 4 files changed, 89 insertions(+), 30 deletions(-) diff --git a/packages/forest_admin_agent/AUDIT_TRAIL.md b/packages/forest_admin_agent/AUDIT_TRAIL.md index db307a21a..ce3799507 100644 --- a/packages/forest_admin_agent/AUDIT_TRAIL.md +++ b/packages/forest_admin_agent/AUDIT_TRAIL.md @@ -85,6 +85,11 @@ Consequences worth knowing: updates leave no trace. - A record the agent cannot read back after the write keeps its row **pending**. Confirming from the patch would claim values that may never have been written. +- A write nested inside another that fails and is rescued keeps its row **pending** too. Each snapshot is + matched to its own operation by the object the hook decorator hands to both of its hooks, so the outer write + settles its own rows rather than the failed inner one's. Where a customization replaced that object there is + nothing to match on: with one operation in flight it still pairs, with several the rows stay pending rather + than the wrong ones being marked done. - One operation audits at most **1000** records — the same number as the Node agent's `MAX_SNAPSHOT_RECORDS`, so a bulk operation is not audited on one agent and truncated on the other. Under `critical: false` a wider selection is truncated, with `N records audited, M skipped` logged at `Warn`; a smart action over the cap is diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb index f90063dcf..b8ce59496 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb @@ -50,12 +50,13 @@ def instrument(collection_customizer) def add_create_hooks(collection_customizer, target) collection_customizer.add_hook('Before', 'Create') do |context| # No record id yet — that is what the column being nullable is for. - @snapshots.push(ids: pending_rows(context.caller, 'create', target[:name], - [{ record_id: nil, new_values: pick(context.data, target[:columns]) }])) + rows = [{ record_id: nil, new_values: pick(context.data, target[:columns]) }] + + @snapshots.push(context.data, ids: pending_rows(context.caller, 'create', target[:name], rows)) end collection_customizer.add_hook('After', 'Create', prepend: true) do |context| - pending = @snapshots.pop + pending = @snapshots.pop_for(context.data) next unless pending confirm(pending[:ids].first, @@ -68,6 +69,7 @@ def add_update_hooks(collection_customizer, target) collection_customizer.add_hook('Before', 'Update') do |context| records = @snapshots.take(context, target[:projection]) @snapshots.push( + context.filter, records: records, patch: context.patch, ids: pending_rows( @@ -82,7 +84,7 @@ def add_update_hooks(collection_customizer, target) end collection_customizer.add_hook('After', 'Update', prepend: true) do |context| - pending = @snapshots.pop + pending = @snapshots.pop_for(context.filter) next unless pending confirm_updates(context, pending, target) @@ -93,6 +95,7 @@ def add_delete_hooks(collection_customizer, target) collection_customizer.add_hook('Before', 'Delete') do |context| records = @snapshots.take(context, target[:projection]) @snapshots.push( + context.filter, records: records, ids: pending_rows( context.caller, 'delete', target[:name], @@ -104,8 +107,8 @@ def add_delete_hooks(collection_customizer, target) ) end - collection_customizer.add_hook('After', 'Delete', prepend: true) do |_context| - pending = @snapshots.pop + collection_customizer.add_hook('After', 'Delete', prepend: true) do |context| + pending = @snapshots.pop_for(context.filter) next unless pending pending[:ids].each { |id| confirm(id) } diff --git a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb index b6e02e910..c72903432 100644 --- a/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb +++ b/packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/snapshots.rb @@ -3,22 +3,36 @@ module AuditTrail # What a "before" hook leaves for the matching "after" hook: the records as they stood, the patch, and the # ids of the pending rows to confirm. # - # Both hooks bracket one operation on one thread, so a LIFO stack pairs them without relying on the filter - # object reaching both unchanged. An operation raising in between strands its entry, hence the cap. + # Entries are keyed by the object the hook decorator hands to both contexts — the filter, or the data on a + # create — because taking the newest entry is wrong as soon as writes nest: an inner write that fails skips + # its after hook and stays on the stack, and the outer hook would then confirm the failed operation's rows + # as done and leave its own stranded. Both directions of that are lies. + # + # An operation raising between the two hooks strands its entry, hence the cap; its rows stay `pending` in + # the table, which is the truthful state for a write that may not have landed. class Snapshots include Recording # ponytail: 16 deep is far past any legitimate nesting; raise it if one ever gets that far. MAX_PENDING = 16 - def push(snapshot) + def push(key, snapshot) stack = pending stack.shift while stack.size >= MAX_PENDING - stack.push(snapshot) + stack.push(snapshot.merge(key: key)) end - def pop - pending.pop + # The entry this operation left, or nothing rather than someone else's. + # + # A customization that replaced the filter or the data leaves no identity to match on: our before hook saw + # the replacement and the after context carries the original. With a single operation in flight that is + # unambiguous, so it still pairs; with several it does not guess, and the rows stay pending. + def pop_for(key) + stack = pending + index = stack.rindex { |entry| entry[:key].equal?(key) } + index = 0 if index.nil? && stack.size == 1 + + index.nil? ? nil : stack.delete_at(index) end # The records an operation is about to touch, capped. Reading "delete all" unbounded would materialise diff --git a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb index c59c84d24..4210bf112 100644 --- a/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb +++ b/packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb @@ -4,6 +4,9 @@ module ForestAdminAgent module AuditTrail describe Capture do let(:column_schema) { ForestAdminDatasourceToolkit::Schema::ColumnSchema } + # The hook decorator hands the same filter (or data) object to both contexts, which is what pairs a + # snapshot with its own after hook — so the helpers share one per operation, as production does. + let(:filter) { new_filter } # Rows keyed by position, which stands in for the row id the store hands back. let(:store) do @@ -73,20 +76,21 @@ def discard(ids) described_class.new.run(datasource_customizer, nil, store: store) end - def filter + def new_filter ForestAdminDatasourceToolkit::Components::Query::Filter.new(condition_tree: nil) end - def before_hook(type, patch: nil, data: nil) + def before_hook(type, patch: nil, data: nil, on: filter) hooks["Before_#{type}"].call( - double('before', caller: caller_double, filter: filter, collection: relaxed_collection, + double('before', caller: caller_double, filter: on, collection: relaxed_collection, patch: patch, data: data) ) end - def after_hook(type, record: nil) + def after_hook(type, record: nil, data: nil, on: filter) hooks["After_#{type}"].call( - double('after', caller: caller_double, filter: filter, collection: relaxed_collection, record: record) + double('after', caller: caller_double, filter: on, collection: relaxed_collection, + record: record, data: data) ) end @@ -112,8 +116,10 @@ def after_hook(type, record: nil) end it 'confirms it with the id and the record that landed' do - before_hook('Create', data: { 'name' => 'Acme' }) - after_hook('Create', record: { 'id' => 1, 'name' => 'Acme', 'address' => { 'city' => 'Paris' } }) + data = { 'name' => 'Acme' } + before_hook('Create', data: data) + after_hook('Create', record: { 'id' => 1, 'name' => 'Acme', 'address' => { 'city' => 'Paris' } }, + data: data) row = store.records.last expect(row.status).to eq(Recording::DONE) @@ -300,22 +306,53 @@ def update(before:, persisted:, patch:) expect(store.records.last.new_values['name']).to eq(Recording::REDACTED) end - # The after hook settles its own operation's rows, not whichever pending row happens to be around. - it 'pairs the after hook with the write that raised between the two, not the stranded one' do + # A write nested inside another, rescued after it failed, leaves its snapshot behind. Taking the newest + # entry would confirm that failed operation's rows as done and strand the outer operation's own — both + # of them lies. + it 'settles its own operation, not a failed inner one left on the stack' do + outer = new_filter + inner = new_filter allow(relaxed_collection).to receive(:list).and_return( - [{ 'id' => 1, 'name' => 'stranded' }], - [{ 'id' => 2, 'name' => 'before' }], - [{ 'id' => 2, 'name' => 'after' }] + [{ 'id' => 1, 'name' => 'outer' }], + [{ 'id' => 2, 'name' => 'inner' }], + [{ 'id' => 1, 'name' => 'outer written' }] ) - before_hook('Update', patch: { 'name' => 'never written' }) - before_hook('Update', patch: { 'name' => 'after' }) - after_hook('Update') + before_hook('Update', patch: { 'name' => 'outer written' }, on: outer) + before_hook('Update', patch: { 'name' => 'never written' }, on: inner) + after_hook('Update', on: outer) settled = store.records.select { |row| row.status == Recording::DONE } - expect(settled.map(&:record_id)).to eq(['2']) - expect(settled.last.new_values).to eq({ 'name' => 'after' }) - expect(store.records.first.status).to eq(Recording::PENDING) + expect(settled.map(&:record_id)).to eq(['1']) + expect(settled.last.new_values).to eq({ 'name' => 'outer written' }) + # The inner write may or may not have landed, which is what pending says. + expect(store.records.map(&:record_id).zip(store.records.map(&:status))).to include(['2', Recording::PENDING]) + end + + # A customization replacing the filter leaves nothing to match on: our before hook saw the replacement, + # the after context carries the original. One operation in flight is unambiguous, so it still pairs. + it 'still settles a single operation whose filter was replaced' do + allow(relaxed_collection).to receive(:list).and_return( + [{ 'id' => 1, 'name' => 'Acme' }], [{ 'id' => 1, 'name' => 'Z' }] + ) + + before_hook('Update', patch: { 'name' => 'Z' }, on: new_filter) + after_hook('Update', on: new_filter) + + expect(store.records.last.status).to eq(Recording::DONE) + end + + # Replaced *and* nested: nothing identifies which entry is ours, so neither is confirmed rather than the + # wrong one being marked done. + it 'leaves both pending when it cannot tell which operation is which' do + allow(relaxed_collection).to receive(:list).and_return([{ 'id' => 1, 'name' => 'a' }], + [{ 'id' => 2, 'name' => 'b' }]) + + before_hook('Update', patch: { 'name' => 'x' }, on: new_filter) + before_hook('Update', patch: { 'name' => 'y' }, on: new_filter) + after_hook('Update', on: new_filter) + + expect(store.records.map(&:status).uniq).to eq([Recording::PENDING]) end end end