ROX-35145: Use roxie in long-running release cluster setup

.github/workflows/create-demo-clusters.yml

@@ -307,6 +307,7 @@ jobs:
     outputs:
       rox-password: ${{ steps.launch-central.outputs.rox-password }}
       central-ip: ${{ steps.launch-central.outputs.central-ip }}
+      ca-cert: ${{ steps.launch-central.outputs.ca-cert }}
     env:
       NAME: ${{ inputs.cluster-with-fake-load-name }}
       KUBECONFIG: artifacts/kubeconfig
@@ -342,6 +343,9 @@ jobs:
         id: artifacts
         run: |
           infractl artifacts "${NAME//./-}" -d artifacts >> "$GITHUB_STEP_SUMMARY"
+      - uses: ./.actions/roxie/install-cli
+        with:
+          version: v0.4.3
       - name: Launch central
         id: launch-central
         uses: ./.actions/release/start-acs
@@ -440,6 +444,9 @@ jobs:
           registry: quay.io
           username: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
           password: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
+      - uses: ./.actions/roxie/install-cli
+        with:
+          version: v0.4.3
       - name: Launch secured cluster
         id: launch-secured-cluster
         uses: ./.actions/release/start-secured-cluster
@@ -451,6 +458,7 @@ jobs:
           registry-password: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
           rox-admin-password: ${{ needs.start-acs.outputs.rox-password }}
           central-ip: ${{ needs.start-acs.outputs.central-ip }}
+          ca-cert: ${{ needs.start-acs.outputs.ca-cert }}
 
   start-kube-burner-for-secured-cluster:
     name: Start kube-burner for the secured cluster

release/start-acs/action.yml

@@ -22,7 +22,7 @@ inputs:
     required: true
     default: ""
   stackrox-dir:
-    description: Where the stackrox directory is located
+    description: Where the stackrox directory is located (needed for monitoring chart)
     required: true
     default: ""
   name:
@@ -42,24 +42,15 @@ runs:
 
     - id: launch-central
       env:
-        ROX_PRODUCT_BRANDING: RHACS_BRANDING
         MAIN_IMAGE_TAG: ${{ inputs.main-image-tag }}
-        API_ENDPOINT: localhost:8000
-        STORAGE: pvc # Backing storage
-        STORAGE_CLASS: faster # Runs on an SSD type
-        STORAGE_SIZE: "100" # 100G
-        MONITORING_SUPPORT: "true" # Runs monitoring
-        LOAD_BALANCER: lb
-        SECURED_CLUSTER_AUTO_LOCK_PROCESS_BASELINES: "true"
-        ROX_ADMIN_USERNAME: admin
+        MONITORING_SUPPORT: "true"
         PAGERDUTY_INTEGRATION_KEY: ${{ inputs.pagerduty-integration-key }}
+        ROX_ADMIN_USERNAME: admin
         REGISTRY_USERNAME: ${{ inputs.registry-username }}
         REGISTRY_PASSWORD: ${{ inputs.registry-password }}
         KUBECONFIG: ${{ inputs.kubeconfig }}
         STACKROX_DIR: ${{ inputs.stackrox-dir }}
         NAME: ${{ inputs.name }}
-        ROX_TELEMETRY_STORAGE_KEY_V1: R5fMyO9n0gibSGzOXtlP2qCFWCGb8uoW
-        ROX_SCANNER_V4: "true"
       run: |
         set -uo pipefail
         "${GITHUB_ACTION_PATH}/../../common/common.sh" \
@@ -73,3 +64,6 @@ outputs:
   central-ip:
     description: The ip address of central
     value: ${{steps.launch-central.outputs.central-ip}}
+  ca-cert:
+    description: Base64-encoded Central CA certificate
+    value: ${{steps.launch-central.outputs.ca-cert}}

release/start-acs/roxie-config.yaml

@@ -0,0 +1,127 @@
+# Resource requests, limits, and scaling settings below are tuned to match
+# the previous deploy-scripts-based flow. They may be adjusted in the future
+# to better resemble a typical customer deployment.
+central:
+  namespace: stackrox
+  pauseReconciliation: true
+  spec:
+    central:
+      exposure:
+        loadBalancer:
+          enabled: true
+      monitoring:
+        exposeEndpoint: Enabled
+      resources:
+        requests:
+          memory: "3Gi"
+          cpu: "2"
+        limits:
+          memory: "12Gi"
+          cpu: "4"
+      db:
+        resources:
+          requests:
+            cpu: "1"
+            memory: "1Gi"
+          limits:
+            cpu: "8"
+            memory: "16Gi"
+        persistence:
+          persistentVolumeClaim:
+            storageClassName: premium-rwo
+            size: "100Gi"
+    scanner:
+      scannerComponent: Enabled
+      analyzer:
+        scaling:
+          autoScaling: Enabled
+          minReplicas: 1
+          maxReplicas: 5
+          replicas: 1
+      monitoring:
+        exposeEndpoint: Enabled
+    customize:
+      envVars:
+        - name: MUTEX_WATCHDOG_TIMEOUT_SECS
+          value: "0"
+        - name: ROX_EXTERNAL_IPS
+          value: "true"
+        - name: ROX_NETWORK_GRAPH_EXTERNAL_IPS
+          value: "true"
+        - name: ROX_AUTO_LOCK_PROCESS_BASELINES
+          value: "true"
+        - name: ROX_BASELINE_GENERATION_DURATION
+          value: "3m"
+        - name: ROX_TELEMETRY_STORAGE_KEY_V1
+          value: "R5fMyO9n0gibSGzOXtlP2qCFWCGb8uoW"
+    scannerV4:
+      scannerComponent: Enabled
+      monitoring:
+        exposeEndpoint: Enabled
+      db:
+        resources:
+          requests:
+            cpu: "500m"
+      indexer:
+        resources:
+          requests:
+            cpu: "400m"
+        scaling:
+          autoScaling: Disabled
+          replicas: 1
+      matcher:
+        resources:
+          requests:
+            cpu: "400m"
+        scaling:
+          autoScaling: Disabled
+          replicas: 1
+securedCluster:
+  namespace: stackrox
+  pauseReconciliation: true
+  spec:
+    sensor:
+      resources:
+        requests:
+          memory: "3Gi"
+          cpu: "2"
+        limits:
+          memory: "12Gi"
+          cpu: "4"
+    scannerV4:
+      monitoring:
+        exposeEndpoint: Enabled
+    overlays:
+      - apiVersion: v1
+        kind: Service
+        name: sensor
+        patches:
+          - path: spec.ports[-1]
+            value: |
+              name: monitoring
+              port: 9090
+              protocol: TCP
+              targetPort: 9090
+      - apiVersion: networking.k8s.io/v1
+        kind: NetworkPolicy
+        name: sensor
+        patches:
+          - path: spec.ingress[-1]
+            value: |
+              ports:
+              - port: 9090
+                protocol: TCP
+    processBaselines:
+      autoLock: Enabled
+    customize:
+      envVars:
+        - name: MUTEX_WATCHDOG_TIMEOUT_SECS
+          value: "0"
+        - name: ROX_FAKE_KUBERNETES_WORKLOAD
+          value: "long-running"
+        - name: ROX_NETFLOW_BATCHING
+          value: "true"
+        - name: ROX_NETFLOW_CACHE_LIMITING
+          value: "true"
+        - name: ROX_FAKE_WORKLOAD_STORAGE
+          value: "/var/cache/stackrox/pebble.db"

release/start-acs/start-acs.sh

@@ -1,24 +1,21 @@
 #!/usr/bin/env bash
 #
-# Start ACS central and secured cluster and patch it so that it can be used for the long running cluster
+# Deploy ACS (Central + SecuredCluster) to a long-running cluster using roxie.
+# Most configuration is in roxie-config.yaml alongside this script.
 #
 # When running locally there should be a kubeconfig created.
 # When running locally there are some environment variables that should be set
 #
 # export NAME=<cluster name>
 # export KUBECONFIG=/tmp/${NAME}/kubeconfig
 #
-# export ROX_IMAGE_FLAVOR=RHACS_BRANDING
 # export MAIN_IMAGE_TAG=<tag>
-# export API_ENDPOINT=localhost:8000
-# export STORAGE=pvc # Backing storage
-# export STORAGE_CLASS=faster # Runs on an SSD type
-# export STORAGE_SIZE=100 # 100G
-# export MONITORING_SUPPORT=true # Runs monitoring
-# export LOAD_BALANCER=lb
+# export MONITORING_SUPPORT=true
 # export ROX_ADMIN_USERNAME=admin
 # export STACKROX_DIR=<stackrox dir>
 # export PAGERDUTY_INTEGRATION_KEY=<PagerDuty release engineering integration key>
+# export REGISTRY_USERNAME=<quay.io username>
+# export REGISTRY_PASSWORD=<quay.io password>
 #
 # export GITHUB_OUTPUT=delete-log-github-output.txt
 # export GITHUB_STEP_SUMMARY=delete-log-start-acs.txt
@@ -27,41 +24,43 @@ set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 
-# shellcheck source=/dev/null
-source "${STACKROX_DIR}"/deploy/common/deploy.sh
-
 if kubectl -n stackrox get deploy/central; then
   gh_log error "Central is already running. It means that you are trying to deploy ACS to a cluster where it is already deployed. This can happen if you try to create a long-running cluster for a release candicate when the such a cluster already exists. Try looking for another workflow that created the cluster."
   exit 1
 fi
 
-gh_log notice "Deploying central..."
-"${STACKROX_DIR}"/deploy/k8s/central.sh
-
-gh_log notice "Forwarding central port..."
-kubectl -n stackrox port-forward deploy/central 8000:8443 > /dev/null 2>&1 &
-sleep 20
-
-gh_log notice "Deploying sensor..."
-"${STACKROX_DIR}"/deploy/k8s/sensor.sh
-
-gh_log notice "Patching sensor deployment..."
-kubectl -n stackrox patch deploy/sensor --patch-file="${SCRIPT_DIR}/patch-sensor.json"
+gh_log notice "Deploying ACS with roxie..."
+roxie_envrc="$(mktemp)"
+roxie deploy \
+    --verbose \
+    --tag "$MAIN_IMAGE_TAG" \
+    --config "${SCRIPT_DIR}/roxie-config.yaml" \
+    --envrc "$roxie_envrc" \
+    --early-readiness
 
-gh_log notice "Patching central deployment..."
-kubectl -n stackrox patch deploy/central --patch-file="${SCRIPT_DIR}/patch-central.json"
-
-CENTRAL_IP=$(kubectl -n stackrox get svc/central-loadbalancer -o json | jq -r '.status.loadBalancer.ingress[0] | .ip // .hostname')
+# shellcheck source=/dev/null
+source "$roxie_envrc"
+rm -f "$roxie_envrc"
+CENTRAL_IP="${API_ENDPOINT%:*}"
 gh_log notice "CENTRAL_IP=$CENTRAL_IP"
 
-API_ENDPOINT="${CENTRAL_IP}:443"
-wait_for_central "${API_ENDPOINT}"
-
-ROX_ADMIN_PASSWORD=$(cat "${STACKROX_DIR}"/deploy/k8s/central-deploy/password)
+if [[ "${MONITORING_SUPPORT:-}" == "true" ]]; then
+    gh_log notice "Deploying monitoring stack..."
+    monitoring_values="$(mktemp)"
+    envsubst < "${STACKROX_DIR}/deploy/charts/monitoring/values.yaml" > "$monitoring_values"
+    helm dependency update "${STACKROX_DIR}/deploy/charts/monitoring"
+    helm upgrade -n stackrox --install stackrox-monitoring \
+        "${STACKROX_DIR}/deploy/charts/monitoring" \
+        --values "$monitoring_values" \
+        --set persistence.type=pvc \
+        --set exposure.type=none
+    rm -f "$monitoring_values"
+fi
 
 # Don't mask the password: masked values are not passed to the runner.
 gh_output rox-password "$ROX_ADMIN_PASSWORD"
 gh_output central-ip "$CENTRAL_IP"
+gh_output ca-cert "$(base64 -w0 < "$ROX_CA_CERT_FILE")"
 
 gh_log notice "Creating access-rhacs secret with the username and the password..."
 kubectl -n stackrox create secret generic access-rhacs \