chore: merge upstream cloud-custodian and stamp 0.9.54a1+0fa7d19f5 (ENG-9376)

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
   cancel-in-progress: true
 env:
-  UV_VERSION: "0.7.12"
+  UV_VERSION: "0.12.19"
   DEFAULT_PY_VERSION: "3.12"
 
 permissions: {}

.github/workflows/functional.yaml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch: {}
 
 env:
-  UV_VERSION: "0.7.12"
+  UV_VERSION: "0.12.19"
   DEFAULT_PY_VERSION: "3.12"
 
 permissions: {}

.github/workflows/release.yml

@@ -3,7 +3,7 @@ name: "Release"
 on:
   workflow_dispatch: {}
 env:
-  UV_VERSION: "0.7.12"
+  UV_VERSION: "0.12.19"
   DEFAULT_PY_VERSION: "3.12"
 
 permissions: {}

.gitignore

@@ -4,6 +4,7 @@
 .eggs/
 .terraform
 .terraform.lock.hcl
+*.tfstate*
 # c7n/version.py
 src/
 deps/

Makefile

@@ -45,6 +45,49 @@ test-coverage:
             --cov tools/c7n_oci/c7n_oci \
             tests tools $(ARGS)
 
+diff-coverage:
+	@echo "Running tests with coverage for changed files..."
+	@CHANGED_TEST_FILES=$$(git diff --name-only origin/main | grep -E "test_.*\.py$$" || true); \
+	if [ -z "$$CHANGED_TEST_FILES" ]; then \
+		echo "Error: No test files changed compared to origin/main"; \
+		echo "Use 'make diff-coverage-full' to run all tests for a provider, or specify TEST_PATH=<path>"; \
+		exit 1; \
+	fi; \
+	echo "Running changed test files:"; \
+	echo "$$CHANGED_TEST_FILES" | sed 's/^/  - /'; \
+	uv run pytest $$CHANGED_TEST_FILES -n auto \
+		--cov-config .coveragerc \
+		--cov-report xml \
+		--cov .
+	@echo ""
+	@echo "Generating diff coverage report..."
+	@uv tool run diff-cover coverage.xml \
+		--compare-branch=origin/main \
+		--src-roots . \
+		--exclude "tests/**" \
+		--exclude "tools/**/tests/**" \
+		--exclude "**/test_*.py" \
+		--html-report htmlcov/diff-coverage.html
+	@echo ""
+	@echo "HTML report generated: htmlcov/diff-coverage.html"
+	@echo "Run 'make view-diff-coverage-linux' to open the report in your browser"
+
+view-coverage-linux:
+	@if [ ! -f htmlcov/index.html ]; then \
+		echo "Error: htmlcov/index.html not found"; \
+		echo "Run 'make test-coverage' first to generate the full coverage report"; \
+		exit 1; \
+	fi
+	@xdg-open htmlcov/index.html 2>/dev/null || open htmlcov/index.html 2>/dev/null || echo "Please open htmlcov/index.html manually"
+
+view-diff-coverage-linux:
+	@if [ ! -f htmlcov/diff-coverage.html ]; then \
+		echo "Error: htmlcov/diff-coverage.html not found"; \
+		echo "Run 'make diff-coverage' first to generate the diff coverage report"; \
+		exit 1; \
+	fi
+	@xdg-open htmlcov/diff-coverage.html 2>/dev/null || open htmlcov/diff-coverage.html 2>/dev/null || echo "Please open htmlcov/diff-coverage.html manually"
+
 test-functional:
 # note this will provision real resources in a cloud environment
 	C7N_FUNCTIONAL=yes AWS_DEFAULT_REGION=us-east-2 pytest tests -m functional $(ARGS)

c7n/actions/invoke.py

@@ -81,7 +81,7 @@ def process(self, resources, event=None):
 
         params = dict(FunctionName=self.data['function'])
         if self.data.get('qualifier'):
-            params['Qualifier'] = self.data['Qualifier']
+            params['Qualifier'] = self.data['qualifier']
 
         if self.data.get('async', True):
             params['InvocationType'] = 'Event'

c7n/data/sagemaker_metrics.yaml

@@ -0,0 +1,171 @@
+# The metrics SageMaker publishes, and the dimensions that identify them.
+#
+# Maintained by hand/agent.
+# See tests/terraform/sagemaker_endpoint_metrics/probe_metrics.py
+#
+# Each section names the metric table it came from. The namespace is in the
+# prose above that table. A dimension entry is one row of the following
+# dimensions table: the names identifying a single time series.
+#
+# kind names a variety of the resource, for metrics only some varieties
+# publish; leave it out when every variety publishes the section's metrics.
+# Endpoints come in three kinds: a "classic" endpoint attaches a model to each
+# production variant, while an "inference-component" endpoint carries an
+# execution role and hosts models as components placed on a variant. The two
+# report invocations under different dimensions, which the documentation
+# lists together without saying which applies when. An "async" endpoint
+# carries an AsyncInferenceConfig; its metrics have a page of their own:
+# https://docs.aws.amazon.com/sagemaker/latest/dg/async-inference-monitor.html
+#
+# Only the metrics of resources c7n filters on are listed; the page also
+# covers jobs, Ground Truth, Feature Store and pipelines.
+#
+# The documentation's dimensions tables say which dimensions a metric can be
+# filtered by, not which sets exist, and CloudWatch identifies a series by
+# the exact set. Only sets observed to carry data are listed here -- see
+# tests/terraform/sagemaker_endpoint_metrics/probe_metrics.py. Notably
+# instance type is published only alongside AvailabilityZone and Region, so
+# the documented "EndpointName, VariantName, InstanceType" is absent.
+
+- title: Endpoint metrics
+  resource: sagemaker-endpoint
+  namespace: /aws/sagemaker/Endpoints
+  metrics:
+    - CPUUtilization
+    - DiskUtilization
+    - GPUMemoryUtilization
+    - GPUUtilization
+    - MemoryUtilization
+  dimensions:
+    - EndpointName, VariantName
+
+# Reservations exist only where a component reserves part of an instance,
+# and the normalized metrics report utilization against them. Observed for a
+# component-hosting endpoint and not for a classic one.
+- title: Endpoint metrics
+  resource: sagemaker-endpoint
+  kind: inference-component
+  namespace: /aws/sagemaker/Endpoints
+  metrics:
+    - CPUReservation
+    - CPUUtilizationNormalized
+    - GPUMemoryUtilizationNormalized
+    - GPUReservation
+    - GPUUtilizationNormalized
+    - MemoryReservation
+  dimensions:
+    - EndpointName, VariantName
+
+- title: Endpoint invocation metrics
+  resource: sagemaker-endpoint
+  kind: classic
+  namespace: AWS/SageMaker
+  metrics:
+    - ConcurrentRequestsPerModel
+    - Invocation4XXErrors
+    - Invocation5XXErrors
+    - InvocationModelErrors
+    - Invocations
+    - InvocationsPerInstance
+    - ModelLatency
+    - ModelSetupTime
+    - OverheadLatency
+    - MidStreamErrors
+    - FirstChunkLatency
+    - FirstChunkModelLatency
+    - FirstChunkOverheadLatency
+  dimensions:
+    - EndpointName, VariantName
+
+# A component-hosting endpoint reports its invocations per component, with
+# no EndpointName dimension.
+- title: Endpoint invocation metrics
+  resource: sagemaker-endpoint
+  kind: inference-component
+  namespace: AWS/SageMaker
+  metrics:
+    - ConcurrentRequestsPerCopy
+    - Invocation4XXErrors
+    - Invocation5XXErrors
+    - InvocationModelErrors
+    - Invocations
+    - InvocationsPerCopy
+    - ModelLatency
+    - ModelSetupTime
+    - OverheadLatency
+    - MidStreamErrors
+    - FirstChunkLatency
+    - FirstChunkModelLatency
+    - FirstChunkOverheadLatency
+  dimensions:
+    - InferenceComponentName
+
+- title: Multi-model endpoint model loading metrics
+  resource: sagemaker-endpoint
+  kind: classic
+  namespace: AWS/SageMaker
+  metrics:
+    - ModelLoadingWaitTime
+    - ModelUnloadingTime
+    - ModelDownloadingTime
+    - ModelLoadingTime
+    - ModelCacheHit
+  dimensions:
+    - EndpointName, VariantName
+
+- title: Multi-model endpoint model instance metrics
+  resource: sagemaker-endpoint
+  kind: classic
+  namespace: /aws/sagemaker/Endpoints
+  metrics:
+    - LoadedModelCount
+  dimensions:
+    - EndpointName, VariantName
+
+# The only real-time invocation metrics an async endpoint publishes. The page
+# gives them no dimensions; the error counts' are by analogy with ModelLatency,
+# since they weren't observed.
+- title: Common Endpoint Metrics
+  resource: sagemaker-endpoint
+  kind: async
+  namespace: AWS/SageMaker
+  metrics:
+    - Invocation4XXErrors
+    - Invocation5XXErrors
+    - ModelLatency
+  dimensions:
+    - EndpointName, VariantName
+
+# The queue belongs to the endpoint rather than to a variant.
+- title: Asynchronous Inference Endpoint Metrics
+  resource: sagemaker-endpoint
+  kind: async
+  namespace: AWS/SageMaker
+  metrics:
+    - ApproximateBacklogSize
+    - ApproximateBacklogSizePerInstance
+    - ApproximateAgeOfOldestRequest
+    - HasBacklogWithoutCapacity
+  dimensions:
+    - EndpointName
+
+# The page misspells InvocationsProcessed as InvocationsProcesssed.
+# NotificationFailures and ExpiredRequests weren't observed, so their
+# dimensions are the page's.
+- title: Asynchronous Inference Endpoint Metrics
+  resource: sagemaker-endpoint
+  kind: async
+  namespace: AWS/SageMaker
+  metrics:
+    - RequestDownloadFailures
+    - ResponseUploadFailures
+    - NotificationFailures
+    - RequestDownloadLatency
+    - ResponseUploadLatency
+    - ExpiredRequests
+    - InvocationFailures
+    - InvocationsProcessed
+    - TimeInBacklog
+    - TotalProcessingTime
+  dimensions:
+    - EndpointName, VariantName

c7n/filters/core.py

@@ -30,6 +30,7 @@
     type_schema,
     parse_cidr,
     parse_date,
+    parse_duration,
     jmespath_search,
     jmespath_compile
 )
@@ -111,10 +112,17 @@ def mod(x, y):
     'mod': mod}
 
 
+# Comparisons between two timedeltas. Other operators never match a
+# duration, and mod would divide by zero on a PT0S value.
+DURATION_OPERATORS = {
+    'eq', 'equal', 'ne', 'not-equal', 'gt', 'greater-than', 'ge', 'gte',
+    'le', 'lte', 'lt', 'less-than'}
+
+
 VALUE_TYPES = [
     'age', 'integer', 'expiration', 'normalize', 'size',
     'cidr', 'cidr_size', 'swap', 'resource_count', 'expr',
-    'unique_size', 'date', 'version', 'float']
+    'unique_size', 'date', 'version', 'float', 'duration']
 
 
 class FilterRegistry(PluginRegistry):
@@ -655,6 +663,21 @@ def validate(self):
                 raise PolicyValidationError(
                     "value_type: date with invalid date value:%s",
                     self.data.get('value', ''))
+        elif self.data.get('value_type') == 'duration':
+            if 'op' in self.data and self.data['op'] not in DURATION_OPERATORS:
+                raise PolicyValidationError(
+                    "value_type: duration can't be used with op: %s" % self.data['op'])
+            if ('value' not in self.data or
+                    'value_from' in self.data or
+                    'value_path' in self.data):
+                raise PolicyValidationError(
+                    "value_type: duration needs a literal value; "
+                    "value_from and value_path aren't supported")
+            if parse_duration(self.data['value']) is None:
+                raise PolicyValidationError(
+                    "value_type: duration needs an ISO 8601 duration in weeks, days, "
+                    "hours, minutes or seconds, such as PT1H (years and months aren't "
+                    "supported), got: %s" % self.data['value'])
         if 'key' not in self.data and 'key' in self.required_keys:
             raise PolicyValidationError(
                 "Missing 'key' in value filter %s" % self.data)
@@ -758,6 +781,9 @@ def match(self, i):
         # value type conversion
         if self.vtype is not None:
             v, r = self.process_value_type(self.v, r, i)
+            # a resource value that isn't a duration never matches, even for not-equal
+            if self.vtype == 'duration' and r is None:
+                return False
         else:
             v = self.v
 
@@ -813,6 +839,8 @@ def process_value_type(self, sentinel, value, resource):
             return value, sentinel
         elif self.vtype == 'date':
             return parse_date(sentinel), parse_date(value)
+        elif self.vtype == 'duration':
+            return parse_duration(sentinel), parse_duration(value)
         elif self.vtype == 'age':
             if not isinstance(sentinel, datetime.datetime):
                 sentinel = datetime.datetime.now(tz=tzutc()) - timedelta(sentinel)

c7n/filters/metrics.py

@@ -70,6 +70,9 @@ class MetricsFilter(Filter):
     Rather than skipping those resources, "missing-value: 0" causes the
     policy to treat their request counts as 0.
 
+    Missing values only make sense when the statistic represents a
+    total, such as a sum or count.
+
     Note the default statistic for metrics is Average.
 
     The ``period-start`` key allows you to align the metric window in two ways.
@@ -89,7 +92,38 @@ class MetricsFilter(Filter):
             value: 30
             op: less-than
             period-start: start-of-day
+
+    Cloudwatch metrics are time series identified by a metric name,
+    a metric namespace, and by key-value pairs called "dimensions".  A
+    "dimension set" identifies the possible keys and is often as
+    simple as a single element that is a resource identifier, like
+    BucketName.
+
+    Metric filters automatically supply a dimension that is the resource
+    identifier.
+
+    Additional dimensions can be provided to narrow the metric data
+    used:
+
+    .. code-block:: yaml
+
+      dimensions:
+        StorageType: StandardStorage
     """
+    #
+    # Note: metric filters are implemented by:
+    #
+    # - enumerating resources for a resource type.
+    #
+    # - For each resource:
+    #
+    #   - Fetch the metric for the resource, identified by the metric
+    #     name, namespace, and resource dimensions.
+    #
+    #   - Evaluate the condition against the data.
+    #     The filter matches if the condition holds for every data point.
+    #     If there are no data, the filter doesn't match.
+    #     A missing value can be supplied to be used when there are no data.
 
     schema = type_schema(
         'metrics',