adapter: Add query policy admission controls - #38762
Conversation
dbaf512 to
995edc1
Compare
Executed SQL demoThe demo ran against Both peek paths produced warnings with correct result rows in The internal Prometheus counters were absent before the demo and reported 16 rejected queries and 2 warned queries afterward, matching the transcript: System configuration and system/quickstart grants use the The transcript contains the actual SQL, result rows, notices, and errors. The script asserts expected rows, warning delivery, and SQLSTATEs. SQL and actual outputRunnable Python script (requires psycopg)import os
import psycopg
notices = []
def notice(diag):
notices.append(diag.message_primary)
print(f"NOTICE [{diag.sqlstate}]: {diag.message_primary}")
if diag.message_detail:
print(f"DETAIL: {diag.message_detail}")
if diag.message_hint:
print(f"HINT: {diag.message_hint}")
def run(conn, sql, *, error=None, rows=None, warning=None):
print(f"\n> {sql}", flush=True)
before_notices = len(notices)
try:
cur = conn.execute(sql)
actual = cur.fetchall() if cur.description else None
if actual is not None:
for row in actual:
print(" | ".join(str(v) for v in row))
else:
print(cur.statusmessage)
assert error is None, f"expected SQLSTATE {error}"
if rows is not None:
assert actual == rows, (actual, rows)
if warning is not None:
assert any(warning in text for text in notices[before_notices:]), notices[before_notices:]
except psycopg.Error as exc:
print(f"ERROR [{exc.sqlstate}]: {exc.diag.message_primary}")
if exc.diag.message_detail:
print(f"DETAIL: {exc.diag.message_detail}")
if exc.diag.message_hint:
print(f"HINT: {exc.diag.message_hint}")
assert error is not None and exc.sqlstate == error, str(exc)
admin = psycopg.connect(
os.getenv("QP_SYSTEM_URL", "postgresql://mz_system@127.0.0.1:6877/materialize"),
autocommit=True,
)
client = psycopg.connect(
os.getenv("QP_SQL_URL", "postgresql://materialize@127.0.0.1:6875/materialize"),
autocommit=True,
)
client.add_notice_handler(notice)
for setting in ["enable_query_policies", "enable_query_policy_enforcement"]:
run(admin, f"ALTER SYSTEM SET {setting} = true")
run(admin, "ALTER SYSTEM SET persist_fast_path_limit = 1000")
run(client, "SET auto_route_catalog_queries = false")
print("\n## Delegated policy creation", flush=True)
run(client, "CREATE ROLE qp_creator")
creator = psycopg.connect(client.info.dsn, user="qp_creator", autocommit=True)
run(creator, "CREATE QUERY POLICY qp_delegated (RULES (r(action='reject', metric='query_plan_includes', value='persist_read')))", error="42501")
run(admin, "GRANT CREATEQUERYPOLICY ON SYSTEM TO qp_creator")
run(creator, "CREATE QUERY POLICY qp_delegated (RULES (r(action='reject', metric='query_plan_includes', value='persist_read')))")
run(creator, "DROP QUERY POLICY qp_delegated")
creator.close()
run(admin, "REVOKE CREATEQUERYPOLICY ON SYSTEM FROM qp_creator")
run(client, "DROP ROLE qp_creator")
run(client, "CREATE TABLE qp_indexed (a int, b int)")
run(client, "INSERT INTO qp_indexed VALUES (1, 10), (1, 30), (2, 20)")
run(client, "CREATE DEFAULT INDEX ON qp_indexed")
run(client, "CREATE MATERIALIZED VIEW qp_raw AS SELECT * FROM qp_indexed")
run(client, "CREATE TABLE qp_target (a int, b int)")
run(client, "CREATE DEFAULT INDEX ON qp_target")
run(client, "CREATE QUERY POLICY qp_fast (RULES (no_dataflows (action='reject', metric='query_plan_includes', value='slow_path_query')))")
run(client, "CREATE QUERY POLICY qp_memory (MODE='enforce', RULES (no_storage (action='reject', metric='query_plan_includes', value='persist_read')))")
for frontend in [False, True]:
print(f"\n## {'Frontend' if frontend else 'Backend'} peek path", flush=True)
run(admin, f"ALTER SYSTEM SET enable_frontend_peek_sequencing = {str(frontend).lower()}")
dsn = client.info.dsn
client.close()
client = psycopg.connect(dsn, autocommit=True)
client.add_notice_handler(notice)
run(client, "SET auto_route_catalog_queries = false")
run(client, "ALTER QUERY POLICY qp_fast SET (MODE='warn')")
run(client, "ALTER CLUSTER quickstart SET (QUERY POLICY = qp_fast)")
run(client, "SELECT a, sum(b) FROM qp_indexed GROUP BY a ORDER BY a", rows=[(1, 40), (2, 20)], warning="qp_fast")
run(client, "ALTER QUERY POLICY qp_fast SET (MODE='enforce')")
run(client, "SELECT 7", rows=[(7,)])
run(client, "SELECT * FROM qp_indexed ORDER BY a, b", rows=[(1, 10), (1, 30), (2, 20)])
run(client, "SELECT a, sum(b) FROM qp_indexed GROUP BY a", error="53000")
run(client, "EXPLAIN SELECT a, sum(b) FROM qp_indexed GROUP BY a")
run(client, "INSERT INTO qp_target SELECT a, sum(b)::int FROM qp_indexed GROUP BY a", error="53000")
run(client, "SELECT * FROM qp_target", rows=[])
run(client, "UPDATE qp_indexed SET b = b + 1 WHERE b < (SELECT avg(b) FROM qp_indexed)", error="53000")
run(client, "DELETE FROM qp_indexed WHERE b < (SELECT avg(b) FROM qp_indexed)", error="53000")
run(client, "SELECT * FROM qp_indexed ORDER BY a, b", rows=[(1, 10), (1, 30), (2, 20)])
run(client, "SUBSCRIBE qp_indexed", error="53000")
run(client, "ALTER CLUSTER quickstart SET (QUERY POLICY = qp_memory)")
run(client, "SELECT a, sum(b) FROM qp_indexed GROUP BY a ORDER BY a", rows=[(1, 40), (2, 20)])
run(client, "SELECT * FROM qp_raw LIMIT 10", error="53000")
run(client, "SELECT a, sum(b) FROM qp_raw GROUP BY a", error="53000")
print("\n## Cluster and role restrictions compose", flush=True)
run(client, "CREATE ROLE qp_agent")
run(admin, "GRANT USAGE ON CLUSTER quickstart TO qp_agent")
run(client, "GRANT SELECT ON qp_indexed, qp_raw TO qp_agent")
run(client, "GRANT USAGE ON QUERY POLICY qp_memory TO qp_agent")
run(client, "ALTER ROLE qp_agent SET query_policy = qp_memory")
run(client, "ALTER CLUSTER quickstart SET (QUERY POLICY = qp_fast)")
agent = psycopg.connect(client.info.dsn, user="qp_agent", autocommit=True)
run(agent, "SELECT a, sum(b) FROM qp_indexed GROUP BY a", error="53000")
run(agent, "SELECT * FROM qp_raw LIMIT 10", error="53000")
agent.close()
print("\n## DDL is exempt", flush=True)
run(client, "CREATE MATERIALIZED VIEW qp_totals AS SELECT a, sum(b) FROM qp_indexed GROUP BY a")
run(client, "CREATE DEFAULT INDEX ON qp_totals")
run(client, "SELECT * FROM qp_totals ORDER BY a", rows=[(1, 40), (2, 20)])
print("\n## Enforcement rollback switch", flush=True)
run(admin, "ALTER SYSTEM SET enable_query_policy_enforcement = false")
run(client, "SELECT a, sum(b) FROM qp_indexed GROUP BY a ORDER BY a", rows=[(1, 40), (2, 20)])
run(client, "SELECT name, mode FROM mz_internal.mz_query_policies ORDER BY name")
run(client, "SELECT name, action, metric, value FROM mz_internal.mz_query_policy_rules ORDER BY name")
run(client, "ALTER ROLE qp_agent RESET query_policy")
run(client, "ALTER CLUSTER quickstart RESET (QUERY POLICY)")
run(client, "DROP QUERY POLICY qp_fast, qp_memory")
print("\nDemo assertions passed.", flush=True) |
Add durable cluster and role query policies with warn/enforce modes and independent management and enforcement rollout flags. Check optimized plans before frontend and backend query execution, including read-then-write statements, and expose lightweight metrics and catalog introspection. Add parser and SQL regression coverage and rollout documentation. Release note: None (experimental, disabled by default). Co-authored-by: Amp <amp@ampcode.com> Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed
Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Use the available upstream registry for Kubernetes fixtures and historical documentation manifests. Docker Hub no longer serves the referenced MinIO image. Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Gate curated metric sink checks on their introduction in v26.41, preserving user-sink coverage during rollback to v26.40. Match Iceberg connection output to the SQL formatter. Replace the container-owned lock file before writing the malformed version fixture. Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
94f94e1 to
b4689b1
Compare
Remove unused compute and storage-operator dependencies. Gate EXCLUDE CONSTRAINTS coverage on v26.42, the first release line carrying the option. Isolate replica introspection compaction from curated metric sink read holds, alongside the existing subscribe exclusion, without changing the compaction assertions. Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Include CREATEQUERYPOLICY in system privilege expectations, including GRANT ALL. Require v26.42 for curated metric sinks: they are absent from the released v26.41.0 catalog despite the development version at introduction. Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Exclude curated sink arrangements from the dedicated arrangement-count test. Select a replica when reading the default cluster logging baseline. Compare compressed Kafka sink output as a multiset, retaining all expected records. Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
The released v26.41 binary still defaults the feature flag off. Preserve the override until v26.42, where the flag is removed, so mixed-version checks can exercise WAIT UNTIL READY. Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-01a08f2d-9030-70e9-b346-5d29d078e2ed Co-authored-by: Amp <amp@ampcode.com>
Motivation
Ad hoc queries can create temporary dataflows or read object storage on a serving cluster. Operators need an enforceable admission policy, with an observe-only rollout mode, without restricting the DDL that maintains that cluster.
Description
Implements milestone 1 of query policies:
QUERY POLICYobjects with ownership, USAGE grants, CREATE/ALTER/DROP/SHOW, and cluster and effective-role attachments.rejectrules forquery_plan_includes = slow_path_query | persist_read. Policies default towarn;enforcerejects with SQLSTATE53000, the matching policy/rule/cluster, and an EXPLAIN/index hint.enable_query_policiesfor management andenable_query_policy_enforcementfor warnings/rejections.mz_query_policy_queries_total{outcome="warned"|"rejected"}counts each affected query once. Introspection exposes policy definitions and rules.Duration/heap limits, cancellation, active-statement introspection, and violation logging remain out of scope.
Attachments and authorization
Each cluster and role can have at most one policy attached, and each policy can contain multiple rules. A query can therefore be subject to two distinct policies.
This adds an optional
QueryPolicyIdto durable cluster configuration and role configuration. It does not add a general attachment framework. SQL resolves the policy name when attaching it; subsequent queries resolve that stable ID to the policy definition, so editing a policy affects all its attachments without copying its rules into clusters or roles.Admission combines the executing cluster's policy and the current effective role's policy. Both constrain the query, with a shared policy evaluated only once. Role attachments are separate from session defaults:
SET,RESET, andRESET ALLcannot change them. Catalog transactions reject dangling attachments and reject dropping an attached policy, including with CASCADE. Existing clusters and roles have no attachment after upgrade.Creation requires the grantable
CREATEQUERYPOLICYsystem privilege, followingCREATENETWORKPOLICY. Owners can alter/drop policies. Cluster attachment requires CREATE on the cluster and USAGE on the policy; role attachment requires CREATEROLE and policy USAGE. This combined cluster/role attachment model is new; the underlying catalog and privilege machinery is reused.Review guide
The diff is large because a dedicated catalog object requires substantial catalog boilerplate: durable storage and serialization, a versioned catalog snapshot and migration, object IDs and exhaustive matches, ownership/privilege handling, and SQL syntax/planning. Most of that follows the existing flat-object patterns.
Start with the behavior:
src/adapter/src/query_policy.rs: classification of the optimized plan and combined cluster/role admission decisions.src/adapter/src/frontend_peek.rsandsrc/adapter/src/coord/sequencer/inner/{peek,subscribe}.rs: enforcement before execution in both paths, including synthetic read-then-write peeks.src/adapter/src/frontend_read_then_write.rs: admission before frontend OCC writes start their internal subscription. Background maintenance is exempt.src/adapter/src/catalog/state.rs: lookup of durable cluster and effective-role attachments.src/sql/src/plan/statement/ddl.rsandsrc/sql/src/rbac.rs: rule validation and attachment authorization.Then review the catalog lifecycle separately from admission behavior.
test/sqllogictest/query_policy.sltdescribes the main user-visible contracts.Verification
Adds parser regression coverage and
test/sqllogictest/query_policy.sltfor admission distinctions, both execution paths, read-then-write rejection, rollout, role/cluster composition, and lifecycle. Reference documentation and a serving-cluster rollout guide are included.An executed SQL demo with actual results and a runnable script demonstrates both execution paths, warnings/rejections, unchanged rows after rejected writes, role/cluster composition, and the rollout switch. Its Prometheus counter increases match the 16 rejected and 2 warned queries.
Release note
None. Query policies are experimental and disabled by default.