From 88cd8135063cc34973dbbd74ccabf5f04e747363 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 6 Oct 2026 12:03:02 +0200 Subject: [PATCH 1/2] feat: Add affinity to OPA Pods --- CHANGELOG.md | 1 + .../usage-guide/operations/pod-placement.adoc | 23 ++++- .../src/controller/validate.rs | 11 +++ rust/operator-binary/src/crd/affinity.rs | 97 +++++++++++++++---- rust/operator-binary/src/crd/mod.rs | 7 +- 5 files changed, 118 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6e6441d..be28a040 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Support floating tags for product images via the new `spec.image.stackableVersionPolicy` field ([#809]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#812]). +- Masters and region servers now have a default affinity to the OPA Pods when OPA authorization is configured ([#XXX]). ### Changed diff --git a/docs/modules/hbase/pages/usage-guide/operations/pod-placement.adoc b/docs/modules/hbase/pages/usage-guide/operations/pod-placement.adoc index 3401afa1..1ac105b2 100644 --- a/docs/modules/hbase/pages/usage-guide/operations/pod-placement.adoc +++ b/docs/modules/hbase/pages/usage-guide/operations/pod-placement.adoc @@ -9,7 +9,8 @@ The default affinities created by the operator are: 1. Co-locate all the HBase Pods (weight 20) 2. Co-locate HBase regionservers with the underlying HDFS datanodes (weight 50) -3. Distribute all Pods within the same role across nodes so multiple instances don't end up on the same Kubernetes node (masters, regionservers, rest servers) (weight 70) +3. If OPA authorization is configured: co-locate HBase masters and regionservers with the OPA Pods (weight 50) +4. Distribute all Pods within the same role across nodes so multiple instances don't end up on the same Kubernetes node (masters, regionservers, rest servers) (weight 70) NOTE: All default affinities are only preferred and not enforced, as we can not expect all setups to have multiple Kubernetes nodes. If you want to have them enforced, you need to specify you own `requiredDuringSchedulingIgnoredDuringExecution` affinities. @@ -108,6 +109,26 @@ The `hdfs-cluster-name` is the name of the HDFS cluster that was configured in t NOTE: It is important that the `hdfsConfigMapName` property contains the name the HDFS cluster. You could instead configure ConfigMaps of specific name or data roles, but for the purpose of Pod placement, this leads to faulty behavior. +If OPA authorization is configured, masters and region servers additionally get the following affinity. +Rest servers do not get it, as they do not load the OPA access controller coprocessor. + +[source,yaml] +---- +affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: server + app.kubernetes.io/instance: opa-cluster-name + app.kubernetes.io/name: opa + topologyKey: kubernetes.io/hostname + weight: 50 +---- + +`opa-cluster-name` is the `configMapName` from `spec.clusterConfig.authorization.opa`, which by convention is the name of the OpaCluster. + == Use custom pod placement For general configuration of Pod placement, see the xref:concepts:operations/pod_placement.adoc[Pod placement concepts] page. One example use-case for HBase would be to *require* the HBase masters to run on different Kubernetes nodes as follows: diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 73cf44a3..63923183 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -139,6 +139,13 @@ pub fn validate_cluster( .vector_aggregator_config_map_name .clone(); + let opa_config = hbase + .spec + .cluster_config + .authorization + .as_ref() + .and_then(|authorization| authorization.opa.as_ref()); + for hbase_role in HbaseRole::iter() { let group_configs = match hbase_role { HbaseRole::Master => validate_role_group_configs( @@ -147,6 +154,7 @@ pub fn validate_cluster( &hbase_role, &cluster_name, hdfs_discovery_cm_name, + opa_config, ), AnyServiceConfig::Master, &vector_aggregator_config_map_name, @@ -157,6 +165,7 @@ pub fn validate_cluster( &hbase_role, &cluster_name, hdfs_discovery_cm_name, + opa_config, ), AnyServiceConfig::RegionServer, &vector_aggregator_config_map_name, @@ -167,6 +176,7 @@ pub fn validate_cluster( &hbase_role, &cluster_name, hdfs_discovery_cm_name, + opa_config, ), AnyServiceConfig::RestServer, &vector_aggregator_config_map_name, @@ -357,6 +367,7 @@ spec: &HbaseRole::Master, &hbase.name_any(), hbase.spec.cluster_config.hdfs_config_map_name.as_ref(), + None, ); let validated = with_validated_config::< diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 540bee68..6a5485e4 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -1,6 +1,9 @@ use stackable_operator::{ - commons::affinity::{ - StackableAffinityFragment, affinity_between_cluster_pods, affinity_between_role_pods, + commons::{ + affinity::{ + StackableAffinityFragment, affinity_between_cluster_pods, affinity_between_role_pods, + }, + opa::OpaConfig, }, k8s_openapi::api::core::v1::{PodAffinity, PodAntiAffinity}, }; @@ -11,18 +14,34 @@ pub fn get_affinity( cluster_name: &str, role: &HbaseRole, hdfs_discovery_cm_name: &str, + opa_config: Option<&OpaConfig>, ) -> StackableAffinityFragment { + // Masters and region servers load the OPA access controller coprocessor, so they are co-located + // with the OPA Pods. + let affinity_to_opa_pods = opa_config.map(|opa_config| { + affinity_between_role_pods( + "opa", + &opa_config.config_map_name, // The discovery cm has the same name as the OpaCluster itself + "server", + 50, + ) + }); let affinity_between_cluster_pods = affinity_between_cluster_pods(APP_NAME, cluster_name, 20); match role { HbaseRole::Master => StackableAffinityFragment { pod_affinity: Some(PodAffinity { - preferred_during_scheduling_ignored_during_execution: Some(vec![ - affinity_between_cluster_pods, - // We would like a affinity to the Zookeeper Pods, but the hbase CRD only contains a ZNode reference. - // We could look up the ZNode and extract the zk cluster from it but that causes network calls - // See https://github.com/stackabletech/zookeeper-operator/issues/644 - // Watch out: The zk can be in a different namespace, so the namespaceSelector must be used - ]), + preferred_during_scheduling_ignored_during_execution: Some( + vec![ + affinity_between_cluster_pods, + // We would like a affinity to the Zookeeper Pods, but the hbase CRD only contains a ZNode reference. + // We could look up the ZNode and extract the zk cluster from it but that causes network calls + // See https://github.com/stackabletech/zookeeper-operator/issues/644 + // Watch out: The zk can be in a different namespace, so the namespaceSelector must be used + ] + .into_iter() + .chain(affinity_to_opa_pods) + .collect(), + ), required_during_scheduling_ignored_during_execution: None, }), pod_anti_affinity: Some(PodAntiAffinity { @@ -36,15 +55,20 @@ pub fn get_affinity( }, HbaseRole::RegionServer => StackableAffinityFragment { pod_affinity: Some(PodAffinity { - preferred_during_scheduling_ignored_during_execution: Some(vec![ - affinity_between_cluster_pods, - affinity_between_role_pods( - "hdfs", - hdfs_discovery_cm_name, // The discovery cm has the same name as the HdfsCluster itself - "datanode", - 50, - ), - ]), + preferred_during_scheduling_ignored_during_execution: Some( + vec![ + affinity_between_cluster_pods, + affinity_between_role_pods( + "hdfs", + hdfs_discovery_cm_name, // The discovery cm has the same name as the HdfsCluster itself + "datanode", + 50, + ), + ] + .into_iter() + .chain(affinity_to_opa_pods) + .collect(), + ), required_during_scheduling_ignored_during_execution: None, }), pod_anti_affinity: Some(PodAntiAffinity { @@ -56,6 +80,8 @@ pub fn get_affinity( node_affinity: None, node_selector: None, }, + // The REST server does not load the OPA access controller coprocessor, so it gets no + // affinity to the OPA Pods. HbaseRole::RestServer => StackableAffinityFragment { pod_affinity: Some(PodAffinity { preferred_during_scheduling_ignored_during_execution: Some(vec![ @@ -111,6 +137,10 @@ mod tests { clusterConfig: hdfsConfigMapName: simple-hdfs zookeeperConfigMapName: simple-znode + authorization: + opa: + configMapName: simple-opa + package: hbase masters: roleGroups: default: @@ -183,6 +213,37 @@ mod tests { HbaseRole::RestServer => (), }; + // Masters and region servers load the OPA access controller coprocessor. + match role { + HbaseRole::Master | HbaseRole::RegionServer => { + expected_affinities.push(WeightedPodAffinityTerm { + pod_affinity_term: PodAffinityTerm { + label_selector: Some(LabelSelector { + match_expressions: None, + match_labels: Some(BTreeMap::from([ + ("app.kubernetes.io/name".to_string(), "opa".to_string()), + ( + "app.kubernetes.io/instance".to_string(), + "simple-opa".to_string(), + ), + ( + "app.kubernetes.io/component".to_string(), + "server".to_string(), + ), + ])), + }), + match_label_keys: None, + mismatch_label_keys: None, + namespace_selector: None, + namespaces: None, + topology_key: "kubernetes.io/hostname".to_string(), + }, + weight: 50, + }); + } + HbaseRole::RestServer => (), + }; + assert_eq!( affinity, StackableAffinity { diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index ff9d983e..f6404efe 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -6,6 +6,7 @@ use stackable_operator::{ commons::{ affinity::StackableAffinity, cluster_operation::ClusterOperation, + opa::OpaConfig, product_image_selection::ProductImage, resources::{ CpuLimitsFragment, MemoryLimitsFragment, NoRuntimeLimits, NoRuntimeLimitsFragment, @@ -315,6 +316,7 @@ impl HbaseConfigFragment { role: &HbaseRole, cluster_name: &str, hdfs_discovery_cm_name: &str, + opa_config: Option<&OpaConfig>, ) -> Self { let graceful_shutdown_timeout = match role { HbaseRole::Master => HbaseRole::DEFAULT_MASTER_GRACEFUL_SHUTDOWN_TIMEOUT, @@ -330,7 +332,7 @@ impl HbaseConfigFragment { hbase_rootdir: Some(default_hbase_rootdir()), resources: default_resources(role), logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, hdfs_discovery_cm_name), + affinity: get_affinity(cluster_name, role, hdfs_discovery_cm_name, opa_config), graceful_shutdown_timeout: Some(graceful_shutdown_timeout), requested_secret_lifetime: Some(requested_secret_lifetime), listener_class: Some( @@ -347,12 +349,13 @@ impl RegionServerConfigFragment { role: &HbaseRole, cluster_name: &str, hdfs_discovery_cm_name: &str, + opa_config: Option<&OpaConfig>, ) -> Self { RegionServerConfigFragment { hbase_rootdir: Some(default_hbase_rootdir()), resources: default_resources(role), logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, hdfs_discovery_cm_name), + affinity: get_affinity(cluster_name, role, hdfs_discovery_cm_name, opa_config), graceful_shutdown_timeout: Some( HbaseRole::DEFAULT_REGION_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, ), From c0a063c3a35c949573e4f970886fbec9c9d659f5 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 6 Oct 2026 12:06:49 +0200 Subject: [PATCH 2/2] Updating Changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be28a040..82455c99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - Support floating tags for product images via the new `spec.image.stackableVersionPolicy` field ([#809]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#812]). -- Masters and region servers now have a default affinity to the OPA Pods when OPA authorization is configured ([#XXX]). +- Masters and region servers now have a default affinity to the OPA Pods when OPA authorization is configured ([#816]). ### Changed @@ -57,6 +57,7 @@ [#803]: https://github.com/stackabletech/hbase-operator/pull/803 [#809]: https://github.com/stackabletech/hbase-operator/pull/809 [#812]: https://github.com/stackabletech/hbase-operator/pull/812 +[#816]: https://github.com/stackabletech/hbase-operator/pull/816 ## [26.7.0] - 2026-07-21