From 2bbd67ab69f435407137537eb19381af40b0fc42 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 14 Oct 2026 17:47:01 +0200 Subject: [PATCH] feat: Support Vector 0.58.0 --- CHANGELOG.md | 6 ++--- .../spark-k8s/pages/usage-guide/logging.adoc | 2 +- .../src/product_logging/test-vector.sh | 24 ++++++++++++------- .../src/product_logging/vector-test.yaml | 5 ++++ .../src/product_logging/vector.yaml | 13 +++++++--- .../01-install-spark-vector-aggregator.yaml | 2 +- .../01-install-vector-aggregator.yaml.j2 | 2 +- 7 files changed, 37 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d3f2785..bbb19dac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ All notable changes to this project will be documented in this file. ### Changed -- Bump `stackable-operator` to 0.118.0 ([#732], [#753], [#773]). +- Bump `stackable-operator` to 0.121.0 ([#732], [#753], [#773], [#779], [#787]). - Replace the unmaintained `serde_yaml` with `yaml_serde` ([#787]). - All pods now set `securityContext.seccompProfile.type` to `RuntimeDefault` ([#787]). - BREAKING: `spec.image.stackableVersion` must now be a full, valid semver version, e.g. `26.7.1`. @@ -42,8 +42,7 @@ All notable changes to this project will be documented in this file. - BREAKING (behaviour): Keys of the `spark-env.sh` `configOverrides` must be valid shell identifiers (matching `[a-zA-Z_][a-zA-Z0-9_]*`) and are now rejected if they are not ([#761]). - Make operations infallible where dependent on static inputs ([#766], [#769]). -- Bump stackable-operator to 0.119.0 ([#779]). -- test: Bump vector-aggregator to 0.58.0 ([#784]). +- Support Vector 0.58.0 ([#784], [#788]). ### Fixed @@ -97,6 +96,7 @@ All notable changes to this project will be documented in this file. [#779]: https://github.com/stackabletech/spark-k8s-operator/pull/779 [#784]: https://github.com/stackabletech/spark-k8s-operator/pull/784 [#787]: https://github.com/stackabletech/spark-k8s-operator/pull/787 +[#788]: https://github.com/stackabletech/spark-k8s-operator/pull/788 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/spark-k8s/pages/usage-guide/logging.adoc b/docs/modules/spark-k8s/pages/usage-guide/logging.adoc index 9b2d2bf1..07edf9e5 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/logging.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/logging.adoc @@ -51,5 +51,5 @@ kind: ConfigMap metadata: name: spark-vector-aggregator-discovery data: - ADDRESS: spark-vector-aggregator:6123 + ADDRESS: http://spark-vector-aggregator:6123 ---- diff --git a/rust/operator-binary/src/product_logging/test-vector.sh b/rust/operator-binary/src/product_logging/test-vector.sh index fbf498f0..166f148c 100755 --- a/rust/operator-binary/src/product_logging/test-vector.sh +++ b/rust/operator-binary/src/product_logging/test-vector.sh @@ -1,11 +1,19 @@ #!/usr/bin/env sh -DATA_DIR=/stackable/log/_vector-state \ -LOG_DIR=/stackable/log \ -NAMESPACE=default \ -CLUSTER_NAME=spark \ -ROLE_NAME=node \ -ROLE_GROUP_NAME=default \ -VECTOR_AGGREGATOR_ADDRESS=vector-aggregator \ -VECTOR_FILE_LOG_LEVEL=info \ +set -eu + +export DATA_DIR=/stackable/log/_vector-state +export LOG_DIR=/stackable/log +export NAMESPACE=default +export CLUSTER_NAME=spark +export ROLE_NAME=node +export ROLE_GROUP_NAME=default +export VECTOR_AGGREGATOR_ADDRESS=http://vector-aggregator:6000 +export VECTOR_FILE_LOG_LEVEL=info + +# Required because vector.yaml contains ${...} placeholders +export VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION=true + +vector validate --no-environment vector.yaml + vector test vector.yaml vector-test.yaml diff --git a/rust/operator-binary/src/product_logging/vector-test.yaml b/rust/operator-binary/src/product_logging/vector-test.yaml index 54f4828a..e0df68e8 100644 --- a/rust/operator-binary/src/product_logging/vector-test.yaml +++ b/rust/operator-binary/src/product_logging/vector-test.yaml @@ -17,6 +17,7 @@ tests: timestamp: 2025-10-02T09:27:28.582Z outputs: - extract_from: extended_logs + expected_event_count: 1 conditions: - type: vrl source: | @@ -47,6 +48,7 @@ tests: timestamp: 2025-10-02T09:27:28.582Z outputs: - extract_from: extended_logs + expected_event_count: 1 conditions: - type: vrl source: | @@ -82,6 +84,7 @@ tests: timestamp: 2025-10-02T09:27:29.473487331Z outputs: - extract_from: extended_logs + expected_event_count: 1 conditions: - type: vrl source: | @@ -119,6 +122,7 @@ tests: version: 0.49.0 outputs: - extract_from: extended_logs + expected_event_count: 1 conditions: - type: vrl source: | @@ -147,6 +151,7 @@ tests: level: INFO outputs: - extract_from: filtered_logs_vector + expected_event_count: 1 conditions: - type: vrl source: | diff --git a/rust/operator-binary/src/product_logging/vector.yaml b/rust/operator-binary/src/product_logging/vector.yaml index e4d40b6e..2fd3d923 100644 --- a/rust/operator-binary/src/product_logging/vector.yaml +++ b/rust/operator-binary/src/product_logging/vector.yaml @@ -176,7 +176,9 @@ transforms: # Extends the processed files with the fields "container" and "file" extended_logs_files: inputs: - - processed_files_* + - processed_files_stdout + - processed_files_stderr + - processed_files_log4j2 type: remap source: | del(.source_type) @@ -214,7 +216,8 @@ transforms: # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs extended_logs: inputs: - - extended_logs_* + - extended_logs_files + - extended_logs_vector type: remap source: | .namespace = "${NAMESPACE}" @@ -228,4 +231,8 @@ sinks: inputs: - extended_logs type: vector - address: ${VECTOR_AGGREGATOR_ADDRESS} + routing: + endpoints: + - ${VECTOR_AGGREGATOR_ADDRESS} + compression: zstd + keepalive: {} diff --git a/tests/templates/kuttl/logging/01-install-spark-vector-aggregator.yaml b/tests/templates/kuttl/logging/01-install-spark-vector-aggregator.yaml index e90ef4c8..1968543d 100644 --- a/tests/templates/kuttl/logging/01-install-spark-vector-aggregator.yaml +++ b/tests/templates/kuttl/logging/01-install-spark-vector-aggregator.yaml @@ -14,4 +14,4 @@ kind: ConfigMap metadata: name: spark-vector-aggregator-discovery data: - ADDRESS: spark-vector-aggregator:6123 + ADDRESS: http://spark-vector-aggregator:6123 diff --git a/tests/templates/kuttl/spark-connect/01-install-vector-aggregator.yaml.j2 b/tests/templates/kuttl/spark-connect/01-install-vector-aggregator.yaml.j2 index 9e385148..219c2e33 100644 --- a/tests/templates/kuttl/spark-connect/01-install-vector-aggregator.yaml.j2 +++ b/tests/templates/kuttl/spark-connect/01-install-vector-aggregator.yaml.j2 @@ -19,5 +19,5 @@ kind: ConfigMap metadata: name: vector-aggregator-discovery data: - ADDRESS: spark-connect-vector-aggregator:6123 + ADDRESS: http://spark-connect-vector-aggregator:6123 {% endif %}