diff --git a/scripts/compare-releases.sh b/scripts/compare-releases.sh index 8297bc1..841bfaa 100755 --- a/scripts/compare-releases.sh +++ b/scripts/compare-releases.sh @@ -11,6 +11,10 @@ PROD_BASE="${PROD_BASE:-https://api.jetkvm.com}" # Together they exercise both eligible and ineligible rollout paths. DEFAULT_DEVICE_IDS=("compare-device-1" "compare-device-2") DEFAULT_SKUS=("__omit__" "jetkvm-v2" "jetkvm-v2-sdmmc") +# SKUs whose responses are allowed to differ from production until the SKU +# table is deployed there: the two Mini variants (no app artifact, no release +# yet) and one unregistered SKU (400 locally, 404 on the old code). +EXTRA_SKUS="${EXTRA_SKUS:-jetkvm-mini-ethernet jetkvm-mini-wireless jetkvm-v3}" TRISTATE_VALUES=("__omit__" "false" "true") TMP_DIR="$(mktemp -d)" @@ -43,6 +47,8 @@ Environment overrides: CURL_TIMEOUT Curl max time in seconds (default: 30) CURL_CONNECT_TIMEOUT Curl connect timeout in seconds (default: 10) FAIL_FAST Stop after first failed case (default: true) + EXTRA_SKUS Space-separated SKUs compared with the expected 400-vs-404 + deviation accepted (default: both Mini SKUs and jetkvm-v3) Examples: scripts/compare-releases.sh @@ -269,6 +275,7 @@ no_compat_patterns = [ re.compile(r'^Version .+ predates SKU support and cannot serve SKU "([^"]+)"$'), re.compile(r'^SKU "([^"]+)" is not available for version .+$'), re.compile(r'^No default (?:app|system|mini) release available for SKU "([^"]+)"$'), + re.compile(r'^No release found for type \S+ and SKU "([^"]+)"$'), ] def canonicalize(message): @@ -292,8 +299,9 @@ is_accepted_deviation() { local query="$1" local left_prefix="$2" local right_prefix="$3" - python3 - "$query" "${left_prefix}.meta" "${right_prefix}.meta" "${left_prefix}.normalized" "${right_prefix}.normalized" <<'PY' + EXTRA_SKUS="$EXTRA_SKUS" python3 - "$query" "${left_prefix}.meta" "${right_prefix}.meta" "${left_prefix}.normalized" "${right_prefix}.normalized" <<'PY' import json +import os import sys from pathlib import Path from urllib.parse import parse_qs @@ -324,6 +332,25 @@ right_meta = parse_meta(right_meta_path) left_body = load_json(left_body_path) right_body = load_json(right_body_path) +def accept(reason): + print(reason) + raise SystemExit(0) + +# Accepted behavior change: +# For SKUs in EXTRA_SKUS (unregistered, or registered but without releases or +# without the requested artifact) the SKU table answers 400 or 404 with its +# own wording; the code before it answered 400 or 404 from a different check. +# Both sides refusing with a client error means the same thing here. +if one("sku") in os.environ.get("EXTRA_SKUS", "").split(): + def refuses(meta, body): + return ( + meta.get("http_code") in ("400", "404") + and isinstance(body, dict) + and str(body.get("name", "")).endswith("Error") + ) + if refuses(left_meta, left_body) and refuses(right_meta, right_body): + accept("SKU table: both sides refuse this SKU with a 4xx (wording and status may differ)") + # Accepted behavior change: # Stable requests with prerelease/dev version constraints are DB-only locally. # Production still resolves those directly from S3. Local 404 vs prod 200 is expected. @@ -344,7 +371,7 @@ if not isinstance(left_body, dict) or left_body.get("name") != "NotFoundError": if not isinstance(right_body, dict) or not right_body.get("appVersion") or not right_body.get("systemVersion"): raise SystemExit(1) -raise SystemExit(0) +accept("stable dev/prerelease version constraints are DB-only locally") PY } @@ -500,11 +527,12 @@ write_case_result() { fi fi - if (( failed == 1 )) && is_accepted_deviation "$query" "$left_prefix" "$right_prefix"; then + if (( failed == 1 )) && accepted_reason="$(is_accepted_deviation "$query" "$left_prefix" "$right_prefix")"; then failed=0 - accepted_reason="stable dev/prerelease version constraints are DB-only locally" details="" mismatch_count=0 + else + accepted_reason="" fi { @@ -710,10 +738,24 @@ STABLE_SYSTEM_VERSION="${stable_versions[1]:-}" PRERELEASE_APP_VERSION="${prerelease_versions[0]:-}" PRERELEASE_SYSTEM_VERSION="${prerelease_versions[1]:-}" +# Range constraints around the live stable version: the caret range resolves +# to the newest patch of the current minor, "