feat(audit_trail): record and expose record history in the agent, gated on an audit database

.rubocop.yml

@@ -257,6 +257,9 @@ Naming/PredicatePrefix:
 
 Metrics/ParameterLists:
   Exclude:
+    - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb'
+    - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb'
+    - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb'
     - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/configuration.rb'
     - 'packages/forest_admin_datasource_zendesk/lib/forest_admin_datasource_zendesk/collections/base_collection.rb'
     - 'packages/forest_admin_datasource_snowflake/lib/forest_admin_datasource_snowflake/datasource.rb'
@@ -357,6 +360,9 @@ Metrics/BlockLength:
 
 Metrics/ClassLength:
   Exclude:
+    - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/capture.rb'
+    - 'packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/store.rb'
+    - 'packages/forest_admin_agent/lib/forest_admin_agent/routes/resources/audit_trail.rb'
     - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/collection.rb'
     - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/introspection/introspector.rb'
     - 'packages/forest_admin_datasource_graphql_hasura/lib/forest_admin_datasource_graphql_hasura/introspection/schema_converter.rb'
@@ -454,6 +460,12 @@ Layout/LineLength:
 
 RSpec/VerifiedDoubles:
   Exclude:
+    - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/action_capture_spec.rb'
+    - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/capture_spec.rb'
+    - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/audit_trail/sql/migrator_spec.rb'
+    - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/action/actions_spec.rb'
+    - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_spec.rb'
+    - 'packages/forest_admin_agent/spec/lib/forest_admin_agent/routes/resources/audit_trail_correlation_spec.rb'
     - 'packages/forest_admin_datasource_customizer/spec/lib/forest_admin_datasource_customizer/composite_datasource_spec.rb'
 
 RSpec/VerifiedDoubleReference:

packages/forest_admin_agent/Gemfile

@@ -3,6 +3,7 @@ source "https://rubygems.org"
 gemspec
 
 group :development, :test do
+  gem 'activerecord', '>= 6.1'
   gem 'forest_admin_datasource_customizer', path: '../forest_admin_datasource_customizer'
   gem 'forest_admin_datasource_toolkit', path: '../forest_admin_datasource_toolkit'
   gem 'forest_admin_test_toolkit', path: '../forest_admin_test_toolkit'
@@ -13,4 +14,5 @@ group :development, :test do
   gem 'simplecov', '~> 0.22', require: false
   gem 'simplecov-html', '~> 0.12.3'
   gem 'simplecov_json_formatter', '~> 0.1.4'
+  gem 'sqlite3', '>= 2.1'
 end

packages/forest_admin_agent/Gemfile-test

@@ -3,6 +3,7 @@ source "https://rubygems.org"
 gemspec
 
 group :development, :test do
+  gem 'activerecord', '>= 6.1'
   gem 'forest_admin_datasource_customizer', path: '../forest_admin_datasource_customizer'
   gem 'forest_admin_datasource_toolkit', path: '../forest_admin_datasource_toolkit'
   gem 'forest_admin_test_toolkit', path: '../forest_admin_test_toolkit'
@@ -13,4 +14,5 @@ group :development, :test do
   gem 'simplecov', '~> 0.22', require: false
   gem 'simplecov-html', '~> 0.12.3'
   gem 'simplecov_json_formatter', '~> 0.1.4'
+  gem 'sqlite3', '>= 2.1'
 end

packages/forest_admin_agent/lib/forest_admin_agent.rb

@@ -4,7 +4,11 @@
 
 loader = Zeitwerk::Loader.for_gem
 loader.inflector.inflect('oauth2' => 'OAuth2')
+loader.inflector.inflect('sql' => 'Sql')
 loader.inflector.inflect('sse_cache_invalidation' => 'SSECacheInvalidation')
+# ActiveRecord is only needed by agents configuring an audit-trail database, and Rails eager loads
+# every gem loader (Zeitwerk::Loader.eager_load_all), so these files must stay strictly autoloaded.
+loader.do_not_eager_load("#{__dir__}/forest_admin_agent/audit_trail/sql")
 loader.setup
 
 module ForestAdminAgent

packages/forest_admin_agent/lib/forest_admin_agent/audit_trail.rb

@@ -0,0 +1,68 @@
+module ForestAdminAgent
+  # The audit trail is inert unless `config.audit_trail[:database]` was set: the agent factory builds the
+  # store during setup — connecting and migrating there rather than on first write — and everything (capture
+  # layers and routes) resolves it from here.
+  module AuditTrail
+    # One operation must not materialise an unbounded number of records: a "delete all" would otherwise read
+    # every matched row and, with the pending/confirm protocol, write each of them twice. Truncation is logged,
+    # never silent.
+    #
+    # Matches the Node agent's `MAX_SNAPSHOT_RECORDS`: the same feature behind the same config key, so a bulk
+    # operation must not be audited on one agent and truncated on the other. Change it in both or neither.
+    MAX_RECORDS_PER_OPERATION = 1000
+
+    # Auditing a subset while the write touches every match is the one thing `critical` exists to prevent, so
+    # over the cap the operation is refused instead — before anything is written, in both the write and the
+    # action path, which is why the message lives here rather than in either of them.
+    def self.refuse_over_cap!
+      raise ForestAdminDatasourceToolkit::Exceptions::ForestException,
+            'The audit trail is configured as critical and cannot record an operation touching more than ' \
+            "#{MAX_RECORDS_PER_OPERATION} records at once. Narrow the selection."
+    end
+
+    def self.log_truncation(kept, total)
+      skipped = total ? total - kept : 'further'
+
+      Facades::Container.logger.log(
+        'Warn',
+        "[ForestAdmin] Audit trail: #{kept} records audited, #{skipped} skipped " \
+        "(cap #{MAX_RECORDS_PER_OPERATION} per operation)"
+      )
+    end
+
+    def self.options
+      config = Facades::Container.config_from_cache
+
+      (config && config[:audit_trail]) || {}
+    end
+
+    def self.store
+      options[:store]
+    end
+
+    # `critical: true` makes the pending insert a precondition of the write: if the audit trail cannot record
+    # that an operation is about to happen, the operation is refused. Nothing was written, so there is nothing
+    # to repair and no compensating write ever happens. Default false keeps today's behaviour, where a broken
+    # audit database costs rows rather than writes.
+    def self.critical?
+      options[:critical] == true
+    end
+
+    def self.log_failure(error)
+      Facades::Container.logger.log('Error', "[ForestAdmin] Audit trail unavailable, skipping: #{error.message}")
+    end
+
+    # Runs the pending insert under the configured policy: refusing the operation when critical, logging and
+    # carrying on otherwise.
+    def self.gate
+      return yield if critical?
+
+      begin
+        yield
+      rescue StandardError => e
+        log_failure(e)
+        nil
+      end
+    end
+  end
+end

packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/action_capture.rb

@@ -0,0 +1,99 @@
+require 'uri'
+
+module ForestAdminAgent
+  module AuditTrail
+    # Records smart-action runs into the same table as the field-level history: the submitted form on the
+    # `previous_values` side, what the action answered on the `new_values` side.
+    #
+    # {Capture} cannot see them — the customizer has no `Execute` hook — and an action's writes are only
+    # audited when they go through the Forest data layer, so a direct ORM write stays invisible. What lands
+    # here is the run itself: who ran which action, on which records, with which form, and how it ended.
+    #
+    # Same protocol as a write: {#pending} before the action, {#confirm} after. The route owns the gate, so a
+    # pending insert that fails refuses the run under `critical: true`.
+    class ActionCapture
+      include Recording
+
+      EXECUTED = 'action'.freeze
+      FAILED = 'action_failed'.freeze
+      # A global action, and a bulk run over a selection wider than the cap, name no single target: they get
+      # one row attached to no record rather than none at all.
+      NO_RECORD = ''.freeze
+      # What of the action's answer is worth keeping — an allowlist, not a denylist: a result also carries the
+      # file's contents, a webhook's body and headers and arbitrary response headers. File bytes have no
+      # business in an audit table and the other two routinely hold credentials, and an allowlist means a field
+      # added to a result later is not stored until someone decides it should be. `html` is left out too:
+      # operator-facing markup, sometimes large, and the message already says what happened.
+      RESULT_FIELDS = %i[type message name mime_type method url path].freeze
+      # Either can carry userinfo credentials or a signed one-time token, which would then sit permanently in
+      # the one table nobody deletes from.
+      URL_FIELDS = %i[url path].freeze
+
+      def initialize(store, redact = {})
+        @store = store
+        @redact = redact || {}
+      end
+
+      # One row per targeted record, provisionally an `action` — {#confirm} settles which it really was. Returns
+      # the row ids to confirm.
+      def pending(caller:, collection:, action_name:, form_values:, record_ids:)
+        return [] unless @store
+
+        timestamp = now
+        correlation_key = correlation_key_for(caller)
+        identity = identity_of(caller)
+        submitted = redact(form_values || {}, @redact[collection] || [])
+        ids = record_ids.empty? ? [NO_RECORD] : record_ids
+
+        @store.append_all(
+          ids.map do |record_id|
+            AuditRecord.new(
+              timestamp: timestamp, operation: EXECUTED, collection: collection, record_id: record_id,
+              status: PENDING, action_name: action_name, correlation_key: correlation_key,
+              previous_values: submitted, new_values: {}, **identity
+            )
+          end
+        )
+      end
+
+      # Best-effort: the action has already run, so a failure here loses the answer, never the run.
+      def confirm(ids, result: nil, failed: false)
+        return if ids.nil? || ids.empty?
+
+        audit_safely do
+          answer = summarize(result)
+
+          ids.each { |id| @store.confirm(id, operation: failed ? FAILED : EXECUTED, new_values: answer) }
+        end
+      end
+
+      private
+
+      # Keys of an action's answer are Forest's own, so they are camelCase on the wire — unlike a record's
+      # column names, which pass through untouched.
+      def summarize(result)
+        return {} unless result.is_a?(Hash)
+
+        result.slice(*RESULT_FIELDS).compact.to_h do |field, value|
+          [field.to_s.camelize(:lower), URL_FIELDS.include?(field) ? sanitize_url(value) : value]
+        end
+      end
+
+      # `userinfo = nil` is a no-op on URI, so the credentials come off textually; the parser then takes care
+      # of the query and fragment.
+      def sanitize_url(value)
+        # Both `https://user:pass@host` and the scheme-relative `//user:pass@host`, which parses fine and
+        # would otherwise keep its credentials.
+        bare = value.to_s.sub(%r{\A([a-z][a-z0-9+.-]*:)?//[^/@]*@}i, '\1//')
+        uri = URI.parse(bare)
+        uri.query = nil
+        uri.fragment = nil
+
+        uri.to_s
+      rescue StandardError
+        # Not something the parser accepts: keep the shape, drop everything that can carry a secret.
+        bare.to_s.split(/[?#]/).first.to_s
+      end
+    end
+  end
+end

packages/forest_admin_agent/lib/forest_admin_agent/audit_trail/audit_record.rb

@@ -0,0 +1,16 @@
+module ForestAdminAgent
+  module AuditTrail
+    # One audited change. Mirrors the columns of `forest.audit_logs`.
+    #
+    # The actor's name and email are denormalised from the caller at write time: the row says who acted then,
+    # not whoever holds that user id today. `action_name` is set on smart-action rows only, and `status`
+    # follows the write protocol — inserted as {Recording::PENDING} before the write and confirmed
+    # {Recording::DONE} after, so a row left pending means the write may or may not have landed.
+    AuditRecord = Struct.new(
+      :id, :timestamp, :operation, :collection, :record_id, :previous_record_id, :status,
+      :user_id, :user_first_name, :user_last_name, :user_email, :action_name,
+      :correlation_key, :previous_values, :new_values,
+      keyword_init: true
+    )
+  end
+end