diff --git a/.github/workflows/jsonschema.yaml b/.github/workflows/jsonschema.yaml index cc027a4e..fda7c763 100644 --- a/.github/workflows/jsonschema.yaml +++ b/.github/workflows/jsonschema.yaml @@ -15,7 +15,7 @@ jobs: steps: - name: Checkout Code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: persist-credentials: false @@ -42,7 +42,7 @@ jobs: - name: Verify ClusterGroup values.schema.json against values-*yaml files run: | - set -e; for i in values-staging.yaml values-prod.yaml; do + set -e; for i in variants/staging/values-staging.yaml variants/prod/values-prod.yaml; do echo "$i" # disable shellcheck of single quotes in yq # shellcheck disable=2016 diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 242ef630..2dc64201 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -31,7 +31,7 @@ jobs: # Checkout the code base # ########################## - name: Checkout Code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: persist-credentials: false # Full git history is needed to get a proper list of changed files within `super-linter` diff --git a/.github/workflows/superlinter.yml b/.github/workflows/superlinter.yml index 83149bc9..859462c3 100644 --- a/.github/workflows/superlinter.yml +++ b/.github/workflows/superlinter.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Checkout Code - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: persist-credentials: false # Full git history is needed to get a proper list of changed files within `super-linter` diff --git a/.gitignore b/.gitignore index 8904b1c9..62c99755 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,8 @@ vault.init super-linter.log common/pattern-vault.init .cursor/* +.claude/* examples/* +tests/ci/.results/ +tests/ci/.pytest_cache/* +tests/ci/__pycache__/* diff --git a/Makefile-common b/Makefile-common index 537ca883..cf9b7bfe 100644 --- a/Makefile-common +++ b/Makefile-common @@ -1,6 +1,6 @@ MAKEFLAGS += --no-print-directory -ANSIBLE_STDOUT_CALLBACK ?= null # null silences all ansible output. Override this with default, minimal, oneline, etc. when debugging. -ANSIBLE_RUN := ANSIBLE_STDOUT_CALLBACK=$(ANSIBLE_STDOUT_CALLBACK) ansible-playbook $(EXTRA_PLAYBOOK_OPTS) +ANSIBLE_STDOUT_CALLBACK ?= rhvp.cluster_utils.readable +ANSIBLE_RUN ?= ANSIBLE_STDOUT_CALLBACK=$(ANSIBLE_STDOUT_CALLBACK) ansible-playbook $(EXTRA_PLAYBOOK_OPTS) DOCS_URL := https://validatedpatterns.io/blog/2025-08-29-new-common-makefile-structure/ .PHONY: help @@ -20,9 +20,9 @@ operator-deploy operator-upgrade: ## Installs/updates the pattern on a cluster ( .PHONY: install install: pattern-install ## Installs the pattern onto a cluster (Loads secrets as well if configured) -.PHONY: uninstall ## Prints a notice that patterns cannot currently be uninstalled -uninstall: - @echo "Uninstall is not possible at the moment so this target is empty. We are working to implement it as well as we can." +.PHONY: uninstall +uninstall: ## (EXPERIMENTAL) See https://validatedpatterns.io/blog/2026-02-16-pattern-uninstall/. + @$(ANSIBLE_RUN) rhvp.cluster_utils.uninstall .PHONY: pattern-install pattern-install: @@ -32,6 +32,11 @@ pattern-install: load-secrets: ## Loads secrets onto the cluster (unless explicitly disabled in values-global.yaml) @$(ANSIBLE_RUN) rhvp.cluster_utils.load_secrets +##@ Debug Tasks +.PHONY: display-secrets-info +display-secrets-info: ## Display your secret material on terminal or show secret loading error. Use with caution! + @$(ANSIBLE_RUN) rhvp.cluster_utils.display_secrets_info + ##@ Validation Tasks .PHONY: validate-prereq validate-prereq: ## verify pre-requisites @@ -52,3 +57,8 @@ validate-schema: ## validates values files against schema in common/clustergroup .PHONY: argo-healthcheck argo-healthcheck: ## Checks if all argo applications are synced @$(ANSIBLE_RUN) rhvp.cluster_utils.argo_healthcheck + +##@ Testing (CI) Tasks +.PHONY: run-ci-tests +run-ci-tests: ## To run ci-tests, set any needed env vars + @$(ANSIBLE_RUN) rhvp.cluster_utils.run_ci_tests diff --git a/ansible.cfg b/ansible.cfg index 516f8b84..528a8cb6 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,6 +1,15 @@ [defaults] localhost_warning=False retry_files_enabled=False -library=~/.ansible/plugins/modules:./ansible/plugins/modules:./common/ansible/plugins/modules:/usr/share/ansible/plugins/modules -roles_path=~/.ansible/roles:./ansible/roles:./common/ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles -filter_plugins=~/.ansible/plugins/filter:./ansible/plugins/filter:./common/ansible/plugins/filter:/usr/share/ansible/plugins/filter +# Retry files disabled to avoid cluttering CI/CD environments +interpreter_python=auto_silent +timeout=30 +library=~/.ansible/plugins/modules:./ansible/plugins/modules:/usr/share/ansible/plugins/modules +roles_path=~/.ansible/roles:./ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles +filter_plugins=~/.ansible/plugins/filter:./ansible/plugins/filter:/usr/share/ansible/plugins/filter +# use the collections from the util. container, +# change below if you want to test local collections +collections_path=/usr/share/ansible/collections + +[inventory] +inventory_unparsed_warning=False diff --git a/pattern.sh b/pattern.sh index d6daa15e..7d8937c5 100755 --- a/pattern.sh +++ b/pattern.sh @@ -1,19 +1,73 @@ #!/bin/bash +set -euo pipefail function is_available { - command -v $1 >/dev/null 2>&1 || { echo >&2 "$1 is required but it's not installed. Aborting."; exit 1; } + command -v "$1" >/dev/null 2>&1 || { echo >&2 "$1 is required but it's not installed. Aborting."; exit 1; } } function version { - echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }' + echo "$1" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }' } -if [ -z "$PATTERN_UTILITY_CONTAINER" ]; then +# We need this check mostly for CI testing, we do not want to run container in container +function is_container() { + [ -n "${KUBERNETES_SERVICE_HOST:-}" ] && return 0 + [ -f /.dockerenv ] && return 0 + [ -f /run/.containerenv ] && return 0 + return 1 +} + +function verify_image() { + local image="$1" + + case "${image}" in + quay.io/validatedpatterns/*|quay.io/hybridcloudpatterns/*) + ;; + *) + echo "Skipping image verification for third-party registry" + return 0 + ;; + esac + + if ! command -v cosign >/dev/null 2>&1; then + echo "WARNING: cosign is not installed, cannot verify image signature" + echo "Install cosign to enable image verification: https://docs.sigstore.dev/cosign/system_config/installation/" + return 0 + fi + + echo "Verifying image signature for ${image}..." + local output rc + local oidc_issuer="${VP_COSIGN_OIDC_ISSUER:-https://token.actions.githubusercontent.com}" + local cert_identity="${VP_COSIGN_CERT_IDENTITY:-https://github.com/validatedpatterns/utility-container/.*}" + output=$(cosign verify \ + --certificate-oidc-issuer "${oidc_issuer}" \ + --certificate-identity-regexp "${cert_identity}" \ + "${image}" 2>&1) && rc=$? || rc=$? + + if [ "${rc}" -eq 0 ]; then + echo "Image signature verified successfully" + elif [ "${rc}" -ge 10 ] && [ "${rc}" -le 13 ]; then + echo "ERROR: Image signature verification failed for ${image} (exit code ${rc})" + echo "${output}" + echo "Set VP_VERIFY_IMAGE=false to skip this check" + exit 1 + else + echo "WARNING: Could not verify image signature for ${image} (likely a network issue)" + echo "Set VP_VERIFY_IMAGE=false to skip this check" + fi +} + +if is_container; then + echo "Already running in a container" + exec "$@" +fi + +if [ -z "${PATTERN_UTILITY_CONTAINER:-}" ]; then PATTERN_UTILITY_CONTAINER="quay.io/validatedpatterns/utility-container" fi # If PATTERN_DISCONNECTED_HOME is set it will be used to populate both PATTERN_UTILITY_CONTAINER # and PATTERN_INSTALL_CHART automatically -if [ -n "${PATTERN_DISCONNECTED_HOME}" ]; then +if [ -n "${PATTERN_DISCONNECTED_HOME:-}" ]; then PATTERN_UTILITY_CONTAINER="${PATTERN_DISCONNECTED_HOME}/utility-container" PATTERN_INSTALL_CHART="oci://${PATTERN_DISCONNECTED_HOME}/pattern-install" echo "PATTERN_DISCONNECTED_HOME is set to ${PATTERN_DISCONNECTED_HOME}" @@ -23,10 +77,10 @@ if [ -n "${PATTERN_DISCONNECTED_HOME}" ]; then fi readonly commands=(podman) -for cmd in ${commands[@]}; do is_available "$cmd"; done +for cmd in "${commands[@]}"; do is_available "$cmd"; done UNSUPPORTED_PODMAN_VERSIONS="1.6 1.5" -PODMAN_VERSION_STR=$(podman --version) +PODMAN_VERSION_STR=$(podman --version) || { echo "Failed to get podman version"; exit 1; } for i in ${UNSUPPORTED_PODMAN_VERSIONS}; do # We add a space if echo "${PODMAN_VERSION_STR}" | grep -q -E "\b${i}"; then @@ -41,19 +95,20 @@ done PODMAN_VERSION=$(echo "${PODMAN_VERSION_STR}" | awk '{ print $NF }') # podman < 4.3.0 do not support keep-id:uid=... -if [ $(version "${PODMAN_VERSION}") -lt $(version "4.3.0") ]; then - PODMAN_ARGS="-v ${HOME}:/root" +PODMAN_ARGS=() +if [ "$(version "${PODMAN_VERSION}")" -lt "$(version "4.3.0")" ]; then + PODMAN_ARGS=(-v "${HOME}:/root") else # We do not rely on bash's $UID and $GID because on MacOSX $GID is not set MYNAME=$(id -n -u) MYUID=$(id -u) MYGID=$(id -g) - PODMAN_ARGS="--passwd-entry ${MYNAME}:x:${MYUID}:${MYGID}::/pattern-home:/bin/bash --user ${MYUID}:${MYGID} --userns keep-id:uid=${MYUID},gid=${MYGID}" - + PODMAN_ARGS=(--passwd-entry "${MYNAME}:x:${MYUID}:${MYGID}::/pattern-home:/bin/bash" --user "${MYUID}:${MYGID}" --userns "keep-id:uid=${MYUID},gid=${MYGID}") fi -if [ -n "$KUBECONFIG" ]; then - if [[ ! "${KUBECONFIG}" =~ ^$HOME* ]]; then +if [ -n "${KUBECONFIG:-}" ]; then + # Check if KUBECONFIG path starts with HOME directory + if [[ ! "${KUBECONFIG}" =~ ^"${HOME}" ]]; then echo "${KUBECONFIG} is pointing outside of the HOME folder, this will make it unavailable from the container." echo "Please move it somewhere inside your $HOME folder, as that is what gets bind-mounted inside the container" exit 1 @@ -62,20 +117,30 @@ fi # Detect if we use podman machine. If we do not then we bind mount local host ssl folders # if we are using podman machine then we do not bind mount anything (for now!) -REMOTE_PODMAN=$(podman system connection list | tail -n +2 | wc -l) -if [ $REMOTE_PODMAN -eq 0 ]; then # If we are not using podman machine we check the hosts folders +REMOTE_PODMAN=$(podman system connection list | tail -n +2 | wc -l) || REMOTE_PODMAN=0 +PKI_HOST_MOUNT_ARGS=() +if [ "${REMOTE_PODMAN}" -eq 0 ]; then # If we are not using podman machine we check the hosts folders # We check /etc/pki/tls because on ubuntu /etc/pki/fwupd sometimes # exists but not /etc/pki/tls and we do not want to bind mount in such a case # as it would find no certificates at all. if [ -d /etc/pki/tls ]; then - PKI_HOST_MOUNT_ARGS="-v /etc/pki:/etc/pki:ro" + PKI_HOST_MOUNT_ARGS=(-v /etc/pki:/etc/pki:ro) elif [ -d /etc/ssl ]; then - PKI_HOST_MOUNT_ARGS="-v /etc/ssl:/etc/ssl:ro" + PKI_HOST_MOUNT_ARGS=(-v /etc/ssl:/etc/ssl:ro) else - PKI_HOST_MOUNT_ARGS="-v /usr/share/ca-certificates:/usr/share/ca-certificates:ro" + PKI_HOST_MOUNT_ARGS=(-v /usr/share/ca-certificates:/usr/share/ca-certificates:ro) fi -else - PKI_HOST_MOUNT_ARGS="" +fi + +# Parse EXTRA_ARGS into an array if set +EXTRA_ARGS_ARRAY=() +if [ -n "${EXTRA_ARGS:-}" ]; then + # shellcheck disable=SC2206 + EXTRA_ARGS_ARRAY=(${EXTRA_ARGS}) +fi + +if [ "${VP_VERIFY_IMAGE:-true}" != "false" ]; then + verify_image "$PATTERN_UTILITY_CONTAINER" fi # Copy Kubeconfig from current environment. The utilities will pick up ~/.kube/config if set so it's not mandatory @@ -101,17 +166,19 @@ podman run -it --rm --pull=newer \ -e PATTERN_NAME \ -e TARGET_BRANCH \ -e TARGET_CLUSTERGROUP \ + -e TARGET_VARIANT \ -e TARGET_ORIGIN \ -e TOKEN_NAMESPACE \ -e TOKEN_SECRET \ -e UUID_FILE \ -e VALUES_SECRET \ - ${PKI_HOST_MOUNT_ARGS} \ + -e 'VP_*' \ + ${PKI_HOST_MOUNT_ARGS[@]+"${PKI_HOST_MOUNT_ARGS[@]}"} \ -v "$(pwd -P)":"$(pwd -P)" \ -v "${HOME}":"${HOME}" \ -v "${HOME}":/pattern-home \ - ${PODMAN_ARGS} \ - ${EXTRA_ARGS} \ + "${PODMAN_ARGS[@]}" \ + ${EXTRA_ARGS_ARRAY[@]+"${EXTRA_ARGS_ARRAY[@]}"} \ -w "$(pwd -P)" \ "$PATTERN_UTILITY_CONTAINER" \ - $@ + "$@" diff --git a/tests/ci/README.md b/tests/ci/README.md new file mode 100644 index 00000000..c62c7fc0 --- /dev/null +++ b/tests/ci/README.md @@ -0,0 +1,42 @@ +# CI Tests + +The requirements.txt file is just a placeholder to show the installed packages in the utility container: +[utility-container requirements.txt](https://github.com/validatedpatterns/utility-container/blob/main/requirements.txt) + +## The ci will run pytest based on file pattern + +pytest -lv test\_\.py --junit-xml .results/test\_\.xml + +## To run upstream tests locally + +Set the env variables pointing to the clusters needed to run the tests on:\ +`VP_HUBCONFIG` (`VP_SPOKECONFIG` if applicable) pointing to the kubconfig of hub (and spoke) clusters\ +(all VP\_\* env var will be available inside the container)\ +`TARGET_CLUSTERGROUP` if its different from the default (values-global.yaml main.clusterGroupName)\ +Test logs and junit-xml will be saved in ci/.results/\ +Run from root repository: + +```bash +./pattern.sh make run-ci-tests +``` + +## Writing additional tests + +The openshift_dyn_client fixture will return a DynamicClient which can be used inside test functions.\ +It requires only an env variable param to use it as a kubeconfig. + +```python +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +``` + +If your tests requires additional python packages, you might want to either run them fully locally in a venv or similar.\ +Or if you want to use the pattern framework (make/Ansible wrappers) you need to use your own util container + +```bash +PATTERN_UTILITY_CONTAINER="your_utility container" ./pattern.sh make run-ci-tests + +``` diff --git a/tests/ci/__init__.py b/tests/ci/__init__.py new file mode 100644 index 00000000..3dc1f76b --- /dev/null +++ b/tests/ci/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/tests/ci/conftest.py b/tests/ci/conftest.py new file mode 100644 index 00000000..12cc3c06 --- /dev/null +++ b/tests/ci/conftest.py @@ -0,0 +1 @@ +from validatedpatterns_tests.interop.conftest_openshift import * # noqa: F401,F403 diff --git a/tests/ci/requirements.txt b/tests/ci/requirements.txt new file mode 100644 index 00000000..0a68e1e2 --- /dev/null +++ b/tests/ci/requirements.txt @@ -0,0 +1,15 @@ +ansible-core==2.18.* +ansible-runner +awxkit +kubernetes +openshift +boto3>=1.21 +botocore>=1.24 +awscli>=1.22 +azure-cli>=2.34 +gcloud +humanize +pytz +pytest +junitparser +vp-qe-test-common @ git+https://github.com/validatedpatterns/vp-qe-test-common.git@v1 \ No newline at end of file diff --git a/tests/ci/route_utils.py b/tests/ci/route_utils.py new file mode 100644 index 00000000..a8df4517 --- /dev/null +++ b/tests/ci/route_utils.py @@ -0,0 +1,18 @@ +from ocp_resources.route import Route + + +def _route_url(openshift_dyn_client, namespace, name): + routes = [ + route + for route in Route.get( + dyn_client=openshift_dyn_client, namespace=namespace, name=name + ) + ] + + assert ( + len(routes) == 1 + ), f"Expected to find the route '{name}' in the namespace '{namespace}'" + + spec = routes[0].instance.spec + scheme = "https" if getattr(spec, "tls", None) else "http" + return f"{scheme}://{spec.host}" diff --git a/tests/ci/test_staging.py b/tests/ci/test_staging.py new file mode 100644 index 00000000..bb22335e --- /dev/null +++ b/tests/ci/test_staging.py @@ -0,0 +1,51 @@ +import pytest +from validatedpatterns_tests.interop import components, subscription + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_subscription_status(openshift_dyn_client): + expected_subs = { + "openshift-gitops-operator": ["openshift-gitops-operator"], + "multicluster-engine": ["multicluster-engine"], + "openshift-external-secrets-operator": ["external-secrets-operator"], + } + subscription.assert_subscription_status(openshift_dyn_client, expected_subs) + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_site_reachable_hub(openshift_dyn_client): + components.assert_site_reachable(openshift_dyn_client) + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_argocd_reachable(openshift_dyn_client): + components.assert_argocd_reachable(openshift_dyn_client) + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_pod_status(openshift_dyn_client): + projects = [ + "patterns-operator", + "vp-gitops", + "vault", + "external-secrets-operator", + "hypershift", + "multicluster-engine", + ] + components.assert_pod_status(openshift_dyn_client, projects, skip_check=[]) diff --git a/values-global.yaml b/values-global.yaml index 1622259e..14b41136 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -1,6 +1,7 @@ --- global: pattern: hypershift + singleArgoCD: true options: useCSV: false @@ -8,7 +9,7 @@ global: installPlanApproval: Automatic main: - clusterGroupName: staging + variant: staging multiSourceConfig: enabled: true clusterGroupChartVersion: 0.9.* diff --git a/values-prod.yaml b/variants/prod/values-prod.yaml similarity index 100% rename from values-prod.yaml rename to variants/prod/values-prod.yaml diff --git a/values-staging.yaml b/variants/staging/values-staging.yaml similarity index 98% rename from values-staging.yaml rename to variants/staging/values-staging.yaml index 81b0ada7..89ee0329 100644 --- a/values-staging.yaml +++ b/variants/staging/values-staging.yaml @@ -24,7 +24,7 @@ clusterGroup: mce: name: multicluster-engine namespace: multicluster-engine - channel: stable-2.11 + channel: stable-2.17 argoProjects: - hub