diff --git a/c7n/lookup.py b/c7n/lookup.py index 123d5659e4b..46cf6d0aa21 100644 --- a/c7n/lookup.py +++ b/c7n/lookup.py @@ -6,28 +6,52 @@ import copy +class Skip: + """Sentinel returned by Lookup.resolve_value to omit a value entirely.""" + + __slots__ = () + + def __repr__(self): + return 'Lookup.SKIP' + + class Lookup: RESOURCE_SOURCE = 'resource' + SKIP = Skip() + + # A lookup needs a key, a fallback, or both. With a key it resolves against + # the resource and falls back to 'default-value' when the path yields + # nothing. With no key it is just the fallback -- which callers that pass a + # name to resolve_value treat as a conditional default, writing it only + # when that name is not already present on the resource. schema = { 'type': 'object', - 'oneOf': [ - { - 'properties': { - 'type': {'type': 'string', 'enum': [RESOURCE_SOURCE]}, - 'key': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['type', 'key'] - } + 'properties': { + 'type': {'type': 'string', 'enum': [RESOURCE_SOURCE]}, + 'key': {'type': 'string'}, + 'default-value': {}, + }, + 'additionalProperties': False, + 'required': ['type'], + 'anyOf': [ + {'required': ['key']}, + {'required': ['default-value']}, ] } @staticmethod - def lookup_type(schema): + def lookup_type(schema, default_schema=None): + """Schema for a value that may be given directly or looked up. + + ``default_schema`` types the lookup's fallback, and defaults to + ``schema``. Pass it when the fallback is narrower than the value -- + tag values, for instance, accept unquoted YAML scalars for backwards + compatibility, but a fallback is always written as a string. + """ lookup_schema = copy.deepcopy(Lookup.schema) - for lookup in lookup_schema['oneOf']: - lookup['properties']['default-value'] = schema + lookup_schema['properties']['default-value'] = ( + schema if default_schema is None else default_schema) return { 'oneOf': [ @@ -36,6 +60,30 @@ def lookup_type(schema): ] } + @staticmethod + def resolve_value(spec, resource, name=None, current_names=()): + """Resolve a value spec against a resource. + + Handles both lookup forms and plain scalars: + + - a scalar is returned unchanged + - ``{type: resource, key: }`` resolves against the resource, + falling back to ``default-value`` when the path yields nothing + - ``{type: resource, default-value: }`` (no key) is a conditional + default: it resolves to ``x`` only when ``name`` is absent from + ``current_names``, and to ``Lookup.SKIP`` otherwise. + """ + if not Lookup.is_lookup(spec): + return spec + if 'key' not in spec and name in current_names: + return Lookup.SKIP + return Lookup.extract(spec, resource) + + @staticmethod + def has_lookups(spec_map): + """True if any value in the mapping is a lookup rather than a scalar.""" + return any(Lookup.is_lookup(v) for v in spec_map.values()) + @staticmethod def extract(source, data=None): if Lookup.is_lookup(source): @@ -54,10 +102,11 @@ def get_value(source, data=None): @staticmethod def get_value_from_resource(source, resource): - value = jmespath_search(source['key'], resource) + if 'key' in source: + value = jmespath_search(source['key'], resource) + if value is not None: + return value - if value is not None: - return value if 'default-value' not in source: raise Exception('Lookup for key, {}, returned None'.format(source['key'])) else: diff --git a/c7n/resources/asg.py b/c7n/resources/asg.py index ca18a28e1b4..c6fc0bfcf61 100644 --- a/c7n/resources/asg.py +++ b/c7n/resources/asg.py @@ -20,9 +20,11 @@ from c7n.manager import resources from c7n import query from c7n.resources.securityhub import PostFinding -from c7n.tags import TagActionFilter, DEFAULT_TAG, TagCountFilter, TagTrim, TagDelayedAction +from c7n.tags import ( + TagActionFilter, DEFAULT_TAG, TagCountFilter, TagTrim, TagDelayedAction, + TagValueResolver, has_dynamic_tag_values, tag_value_schema) from c7n.utils import ( - FormatDate, local_session, type_schema, chunks, get_retry, select_keys) + local_session, type_schema, chunks, get_retry, select_keys) from .ec2 import deserialize_user_data @@ -1104,7 +1106,7 @@ def process_resource_set(self, client, asgs, tags): @ASG.action_registry.register('tag') @ASG.action_registry.register('mark') -class Tag(Action): +class Tag(TagValueResolver, Action): """Action to add a tag to an ASG The *propagate* parameter can be used to specify that the tag being added @@ -1125,13 +1127,32 @@ class Tag(Action): key: OwnerName value: OwnerName propagate: true + + Tag values may also be looked up per-asg from the asg's attributes: + + .. code-block:: yaml + + policies: + - name: asg-tag-owner + resource: asg + actions: + - type: tag + propagate: true + tags: + Owner: + type: resource + key: "Tags[?Key=='Team'].Value | [0]" # from the Team tag + default-value: unknown + CostCenter: + type: resource + default-value: unassigned # set only if CostCenter absent """ schema = type_schema( 'tag', key={'type': 'string'}, - value={'type': 'string'}, - tags={'type': 'object'}, + value=tag_value_schema(), + tags={'type': 'object', 'additionalProperties': tag_value_schema()}, # Backwards compatibility tag={'type': 'string'}, msg={'type': 'string'}, @@ -1141,34 +1162,57 @@ class Tag(Action): permissions = ('autoscaling:CreateOrUpdateTags',) batch_size = 1 - def get_tag_set(self): - tags = [] + def get_tag_spec_map(self): + spec_map = {} + tags = self.data.get('tags', {}) key = self.data.get('key', self.data.get('tag', DEFAULT_TAG)) - value = self.data.get( - 'value', self.data.get( - 'msg', 'AutoScaleGroup does not meet policy guidelines')) + value = self.data.get('value', self.data.get('msg')) + # The mark default only applies to the single-tag form. A policy that + # spells out a tags mapping gets that mapping and nothing else -- + # otherwise every asg picks up a marker tag it never asked for, and a + # conditional default can never resolve to an empty payload. + if value is None and not tags: + value = 'AutoScaleGroup does not meet policy guidelines' if key and value: - tags.append({'Key': key, 'Value': value}) + spec_map[key] = value + + spec_map.update(tags) - for k, v in self.data.get('tags', {}).items(): - tags.append({'Key': k, 'Value': v}) + return spec_map - return tags + def resolve_tag_sets(self, asgs, spec_map): + """Yield (asg_set, tags) pairs to submit. + + Static values are shared by every asg, so they go out in batches. + Dynamic values are resolved per asg and then regrouped, so asgs that + land on the same payload still batch together, and an asg whose tags + all resolve to skip drops out entirely. + """ + if not has_dynamic_tag_values(spec_map): + tags = [{'Key': k, 'Value': v} for k, v in spec_map.items()] + self.interpolate_values(tags) + for asg_set in chunks(asgs, self.batch_size): + yield asg_set, tags + return + + for asg_set, resolved in self.resolve_and_group(asgs, spec_map): + tags = [{'Key': k, 'Value': v} for k, v in resolved.items()] + for chunk in chunks(asg_set, self.batch_size): + yield chunk, tags def process(self, asgs): - tags = self.get_tag_set() + spec_map = self.get_tag_spec_map() error = None - self.interpolate_values(tags) - client = self.get_client() with self.executor_factory(max_workers=2) as w: futures = {} - for asg_set in chunks(asgs, self.batch_size): + for asg_set, tags in self.resolve_tag_sets(asgs, spec_map): futures[w.submit( - self.process_resource_set, client, asg_set, tags)] = asg_set + self.process_resource_set, client, asg_set, tags)] = ( + asg_set, tags) for f in as_completed(futures): - asg_set = futures[f] + asg_set, tags = futures[f] if f.exception(): self.log.exception( "Exception tagging tag:%s error:%s asg:%s" % ( @@ -1194,14 +1238,6 @@ def process_resource_set(self, client, asgs, tags): a.setdefault('Tags', []).append(atags) self.manager.retry(client.create_or_update_tags, Tags=tag_params) - def interpolate_values(self, tags): - params = { - 'account_id': self.manager.config.account_id, - 'now': FormatDate.utcnow(), - 'region': self.manager.config.region} - for t in tags: - t['Value'] = t['Value'].format(**params) - def get_client(self): return local_session(self.manager.session_factory).client('autoscaling') diff --git a/c7n/resources/cw.py b/c7n/resources/cw.py index 4bb205aa88f..ac99af30ef2 100644 --- a/c7n/resources/cw.py +++ b/c7n/resources/cw.py @@ -6,7 +6,9 @@ from concurrent.futures import as_completed from datetime import datetime, timedelta -from c7n.actions import BaseAction +from botocore.exceptions import ClientError + +from c7n.actions import AutoTagUser, BaseAction from c7n.exceptions import PolicyValidationError from c7n.filters import Filter, MetricsFilter from c7n.filters.core import parse_date, ValueFilter @@ -17,7 +19,8 @@ QueryResourceManager, TypeInfo, DescribeSource, ConfigSource, DescribeWithResourceTags) from c7n.resolver import ValuesFrom -from c7n.tags import universal_augment +from c7n.tags import ( + RemoveTag, Tag, TagActionFilter, TagDelayedAction, universal_augment) from c7n.utils import type_schema, local_session, chunks, get_retry, jmespath_search @@ -688,6 +691,32 @@ def process(self, resources): logGroupName=r['logGroupName'], **params) +class DescribeDashboard(DescribeSource): + + def augment(self, resources): + # Dashboards aren't supported by the resource groups tagging api, + # fetch their tags from cloudwatch directly. + resources = super().augment(resources) + client = local_session(self.manager.session_factory).client('cloudwatch') + results = [] + for r, arn in zip(resources, self.manager.get_arns(resources)): + try: + tags = self.manager.retry( + client.list_tags_for_resource, ResourceARN=arn).get('Tags', []) + except ClientError as e: + if e.response['Error']['Code'] not in ( + 'ResourceNotFound', 'ResourceNotFoundException'): + raise + # the dashboard was deleted between enumeration and augment + self.manager.log.warning( + "Resource not found: list_tags_for_resource using %s" % { + 'ResourceARN': arn}) + continue + r['Tags'] = tags + results.append(r) + return results + + @resources.register("cloudwatch-dashboard") class CloudWatchDashboard(QueryResourceManager): class resource_type(TypeInfo): @@ -698,14 +727,71 @@ class resource_type(TypeInfo): id = "DashboardName" name = "DashboardName" cfn_type = "AWS::CloudWatch::Dashboard" - universal_taggable = object() global_resource = True + permissions_augment = ("cloudwatch:ListTagsForResource",) source_mapping = { - "describe": DescribeWithResourceTags, + "describe": DescribeDashboard, } +@CloudWatchDashboard.action_registry.register('tag') +class DashboardTag(Tag): + """Add tags to a cloudwatch dashboard + + :example: + + .. code-block:: yaml + + policies: + - name: cloudwatch-dashboard-tag-owner + resource: aws.cloudwatch-dashboard + filters: + - "tag:Owner": absent + actions: + - type: tag + key: Owner + value: platform-team + """ + + permissions = ("cloudwatch:TagResource",) + + def process_resource_set(self, client, resource_set, tags): + for arn in self.manager.get_arns(resource_set): + self.manager.retry(client.tag_resource, ResourceARN=arn, Tags=tags) + + +@CloudWatchDashboard.action_registry.register('remove-tag') +class DashboardRemoveTag(RemoveTag): + """Remove tags from a cloudwatch dashboard + + :example: + + .. code-block:: yaml + + policies: + - name: cloudwatch-dashboard-remove-owner + resource: aws.cloudwatch-dashboard + filters: + - "tag:Owner": present + actions: + - type: remove-tag + tags: [Owner] + """ + + permissions = ("cloudwatch:UntagResource",) + + def process_resource_set(self, client, resource_set, tag_keys): + for arn in self.manager.get_arns(resource_set): + self.manager.retry( + client.untag_resource, ResourceARN=arn, TagKeys=tag_keys) + + +CloudWatchDashboard.filter_registry.register('marked-for-op', TagActionFilter) +CloudWatchDashboard.action_registry.register('mark-for-op', TagDelayedAction) +CloudWatchDashboard.action_registry.register('auto-tag-user', AutoTagUser) + + @resources.register("destination") class Destination(QueryResourceManager): class resource_type(TypeInfo): diff --git a/c7n/resources/ebs.py b/c7n/resources/ebs.py index 0058450d339..087c9508886 100644 --- a/c7n/resources/ebs.py +++ b/c7n/resources/ebs.py @@ -22,7 +22,7 @@ from c7n.resources.kms import ResourceKmsKeyAlias from c7n.resources.securityhub import PostFinding from c7n.query import QueryResourceManager, TypeInfo -from c7n.tags import Tag, coalesce_copy_user_tags +from c7n.tags import coalesce_copy_user_tags from c7n.utils import ( camelResource, chunks, @@ -94,16 +94,6 @@ def get_resources(self, ids, cache=True, augment=True): class ErrorHandler: - @staticmethod - def remove_snapshot(rid, resource_set): - found = None - for r in resource_set: - if r['SnapshotId'] == rid: - found = r - break - if found: - resource_set.remove(found) - @staticmethod def extract_bad_snapshot(e): """Handle various client side errors when describing snapshots""" @@ -132,16 +122,6 @@ def extract_bad_volume(e): log.warning("Volume id malformed %s" % e_vol_id) return e_vol_id - @staticmethod - def remove_volume(rid, resource_set): - found = None - for r in resource_set: - if r['VolumeId'] == rid: - found = r - break - if found: - resource_set.remove(found) - class SnapshotQueryParser(QueryParser): @@ -200,24 +180,6 @@ class VolumeQueryParser(QueryParser): type_name = 'EBS Volume' -@Snapshot.action_registry.register('tag') -class SnapshotTag(Tag): - - permissions = ('ec2:CreateTags',) - - def process_resource_set(self, client, resource_set, tags): - while resource_set: - try: - return super(SnapshotTag, self).process_resource_set( - client, resource_set, tags) - except ClientError as e: - bad_snap = ErrorHandler.extract_bad_snapshot(e) - if bad_snap: - ErrorHandler.remove_snapshot(bad_snap, resource_set) - continue - raise - - @Snapshot.filter_registry.register('age') class SnapshotAge(AgeFilter): """EBS Snapshot Age Filter @@ -739,24 +701,6 @@ def get_resources(self, ids, cache=True, augment=True): return [] -@EBS.action_registry.register('tag') -class VolumeTag(Tag): - - permissions = ('ec2:CreateTags',) - - def process_resource_set(self, client, resource_set, tags): - while resource_set: - try: - return super(VolumeTag, self).process_resource_set( - client, resource_set, tags) - except ClientError as e: - bad_vol = ErrorHandler.extract_bad_volume(e) - if bad_vol: - ErrorHandler.remove_volume(bad_vol, resource_set) - continue - raise - - @EBS.filter_registry.register('snapshots') class EBSSnapshotsFilter(ListItemFilter): """ diff --git a/c7n/resources/eks.py b/c7n/resources/eks.py index 74f0b9d70ad..9c174099c17 100644 --- a/c7n/resources/eks.py +++ b/c7n/resources/eks.py @@ -10,9 +10,11 @@ from c7n.manager import resources from c7n.resources.aws import shape_schema from c7n import tags, query +from c7n.tags import is_resource_gone from c7n.query import QueryResourceManager, TypeInfo, DescribeSource, \ ChildResourceManager, ChildDescribeSource from c7n.utils import local_session, type_schema, get_retry +from botocore.exceptions import ClientError from botocore.waiter import WaiterModel, create_waiter_with_client from .aws import shape_validate from .ecs import ContainerConfigSource @@ -339,7 +341,9 @@ def process_resource_set(self, client, resource_set, tags): client.tag_resource, resourceArn=r['arn'], tags={t['Key']: t['Value'] for t in tags}) - except client.exceptions.ResourceNotFoundException: + except ClientError as e: + if not is_resource_gone(e.response['Error']['Code']): + raise continue @@ -358,7 +362,9 @@ def process_resource_set(self, client, resource_set, tags): self.manager.retry( client.untag_resource, resourceArn=r['arn'], tagKeys=tags) - except client.exceptions.ResourceNotFoundException: + except ClientError as e: + if not is_resource_gone(e.response['Error']['Code']): + raise continue diff --git a/c7n/resources/kms.py b/c7n/resources/kms.py index 50113a5780f..91bedd1305e 100644 --- a/c7n/resources/kms.py +++ b/c7n/resources/kms.py @@ -14,6 +14,7 @@ from c7n.query import ( ConfigSource, DescribeSource, QueryResourceManager, RetryPageIterator, TypeInfo) from c7n.utils import local_session, type_schema, select_keys +from c7n import tags as tagmod from c7n.tags import universal_augment from .securityhub import PostFinding @@ -136,6 +137,46 @@ def alias_map(self): return alias_map +class SkipAwsManagedKey: + """Mixin for kms-key actions that cannot operate on AWS managed keys + (KeyManager == 'AWS', e.g. the keys backing alias/aws/s3, + alias/aws/ebs, etc). AWS owns the lifecycle, policy, and rotation of + these keys; ScheduleKeyDeletion, key policy changes, rotation + changes, and tag mutations are all rejected by the API for them. + Filters them out before delegating to the real action and emits a + warning so operators know why they were skipped.""" + + def process(self, resources): + excluded = [r for r in resources if r.get('KeyManager') == 'AWS'] + if excluded: + self.manager.log.warning( + "Skipping %d AWS managed key(s) which cannot be modified " + "by any customer account", + len(excluded)) + resources = [r for r in resources if r.get('KeyManager') != 'AWS'] + if not resources: + return + return super().process(resources) + + +@Key.action_registry.register('mark') +@Key.action_registry.register('tag') +class KeyTag(SkipAwsManagedKey, tagmod.UniversalTag): + pass + + +@Key.action_registry.register('unmark') +@Key.action_registry.register('untag') +@Key.action_registry.register('remove-tag') +class KeyRemoveTag(SkipAwsManagedKey, tagmod.UniversalUntag): + pass + + +@Key.action_registry.register('mark-for-op') +class KeyMarkForOp(SkipAwsManagedKey, tagmod.UniversalTagDelayedAction): + pass + + @Key.filter_registry.register('key-rotation-status') class KeyRotationStatus(ValueFilter): """Filters KMS keys by the rotation status @@ -313,7 +354,6 @@ def get_matching_aliases(self, resources, event=None): return matched -@Key.action_registry.register('remove-statements') @KeyAlias.action_registry.register('remove-statements') class RemovePolicyStatement(RemovePolicyBase): """Action to remove policy statements from KMS @@ -380,8 +420,12 @@ def process_resource(self, client, resource, key_id): 'Statements': found} -@Key.action_registry.register('set-rotation') -class KmsKeyRotation(BaseAction): +@Key.action_registry.register('remove-statements') +class KeyRemovePolicyStatement(SkipAwsManagedKey, RemovePolicyStatement): + pass + + +class _KmsKeyRotationBase(BaseAction): """Toggle KMS key rotation :example: @@ -411,6 +455,11 @@ def process(self, keys): client.disable_key_rotation(KeyId=k['KeyId']) +@Key.action_registry.register('set-rotation') +class KmsKeyRotation(SkipAwsManagedKey, _KmsKeyRotationBase): + pass + + @KeyAlias.action_registry.register('post-finding') @Key.action_registry.register('post-finding') class KmsPostFinding(PostFinding): @@ -607,8 +656,7 @@ def get_item_values(self, resource): return [result] -@Key.action_registry.register("schedule-deletion") -class KmsKeyScheduleDeletion(BaseAction): +class _KmsKeyScheduleDeletionBase(BaseAction): """Schedule KMS key deletion If the number of days is not specified, the default value of 30 days is used. @@ -641,3 +689,8 @@ def process(self, keys): client.schedule_key_deletion( KeyId=k["KeyId"], PendingWindowInDays=self.data.get("days", 30) ) + + +@Key.action_registry.register("schedule-deletion") +class KmsKeyScheduleDeletion(SkipAwsManagedKey, _KmsKeyScheduleDeletionBase): + pass diff --git a/c7n/resources/route53.py b/c7n/resources/route53.py index 40eb07d929a..1f37f7b366d 100644 --- a/c7n/resources/route53.py +++ b/c7n/resources/route53.py @@ -616,6 +616,46 @@ class resource_type(TypeInfo): } +class SkipAutoDefinedResolverRule: + """Mixin for ResolverRule actions that cannot operate on Route 53's + global auto-defined rules (e.g. rslvr-autodefined-rr-internet-resolver). + These have no owning account and cannot be tagged or modified by any + customer. Filters them out before delegating to the real action and + emits a warning so operators know why they were skipped.""" + + def process(self, resources): + excluded = [r for r in resources + if r.get('Id', '').startswith('rslvr-autodefined-rr-')] + if excluded: + self.manager.log.warning( + "Skipping %d AWS auto-defined resolver rule(s) which " + "have no owning account and cannot be modified", + len(excluded)) + resources = [r for r in resources + if not r.get('Id', '').startswith('rslvr-autodefined-rr-')] + if not resources: + return + return super().process(resources) + + +@ResolverRule.action_registry.register('mark') +@ResolverRule.action_registry.register('tag') +class ResolverRuleTag(SkipAutoDefinedResolverRule, tags.UniversalTag): + pass + + +@ResolverRule.action_registry.register('unmark') +@ResolverRule.action_registry.register('untag') +@ResolverRule.action_registry.register('remove-tag') +class ResolverRuleRemoveTag(SkipAutoDefinedResolverRule, tags.UniversalUntag): + pass + + +@ResolverRule.action_registry.register('mark-for-op') +class ResolverRuleMarkForOp(SkipAutoDefinedResolverRule, tags.UniversalTagDelayedAction): + pass + + @resources.register('resolver-logs') class ResolverQueryLogConfig(QueryResourceManager): diff --git a/c7n/resources/s3.py b/c7n/resources/s3.py index 6d34f12848c..517bb1e5a60 100644 --- a/c7n/resources/s3.py +++ b/c7n/resources/s3.py @@ -38,7 +38,8 @@ from botocore.client import Config from botocore.exceptions import ( - ClientError, ConnectTimeoutError, EndpointConnectionError, ReadTimeoutError) + ClientError, ConnectionClosedError, ConnectTimeoutError, EndpointConnectionError, + ReadTimeoutError) from collections import defaultdict from concurrent.futures import as_completed @@ -619,7 +620,8 @@ def assemble(self, bucket): "Bucket ssl error %s: %s %s", bucket['Name'], bucket.get('Location', 'unknown'), e) continue - except (ConnectTimeoutError, ReadTimeoutError, EndpointConnectionError) as e: + except (ConnectTimeoutError, ReadTimeoutError, EndpointConnectionError, + ConnectionClosedError) as e: # Endpoint is unreachable or hung - could be a degraded/ # unreachable region, or a bucket deleted between # list_buckets and here. @@ -705,6 +707,14 @@ def modify_bucket_tags(session_factory, buckets, add_tags=(), remove_tags=()): if e.response['Error']['Code'] != 'NoSuchTagSet': raise bucket['Tags'] = [] + except (ConnectTimeoutError, ReadTimeoutError, EndpointConnectionError, + ConnectionClosedError) as e: + # Endpoint is unreachable or hung - skip this bucket rather than + # abandoning the rest of the batch. + log.warning( + "Unable to get new set of bucket tags needed to modify tags, " + "skipping tag action for bucket: %s error: %s", bucket["Name"], e) + continue new_tags = {t['Key']: t['Value'] for t in add_tags} for t in bucket.get('Tags', ()): @@ -715,7 +725,8 @@ def modify_bucket_tags(session_factory, buckets, add_tags=(), remove_tags=()): try: client.put_bucket_tagging( Bucket=bucket['Name'], Tagging={'TagSet': tag_set}) - except ClientError as e: + except (ClientError, ConnectTimeoutError, ReadTimeoutError, + EndpointConnectionError, ConnectionClosedError) as e: log.exception( 'Exception tagging bucket %s: %s', bucket['Name'], e) continue diff --git a/c7n/tags.py b/c7n/tags.py index 2ecb97235d0..753fb594327 100644 --- a/c7n/tags.py +++ b/c7n/tags.py @@ -8,6 +8,8 @@ snapshots) and resources that support Amazon's Resource Groups Tagging API """ +import re + from collections import Counter from concurrent.futures import as_completed @@ -17,6 +19,8 @@ import time +from botocore.exceptions import ClientError + from c7n.manager import resources as aws_resources from c7n.actions import BaseAction as Action, AutoTagUser from c7n.exceptions import PolicyValidationError, PolicyExecutionError @@ -28,9 +32,57 @@ DEFAULT_TAG = "maid_status" +# Tag actions inherently race with resource deletion, we enumerate a +# resource set and then mutate it. ec2 fails the whole batch when any +# resource in it is unusable, but names the offending ids in the error +# message, so we can drop those and tag the rest. +EC2_BAD_RESOURCE_ID = re.compile(r'^Invalid.+\.(NotFound|Malformed)$') +QUOTED_RESOURCE_IDS = re.compile(r"['\"]([^'\"]+)['\"]") + +# Other services report a resource as already gone with a service specific +# error code - query protocol services use NotFound(Fault), +# json protocol services use (Resource)NotFoundException. +RESOURCE_GONE_ERROR = re.compile(r'NotFound(Fault|Exception)?$') + + +def is_resource_gone(error_code): + """Does an api error code mean the resource we wanted is already gone?""" + return bool(RESOURCE_GONE_ERROR.search(error_code)) + + +def extract_bad_resource_ids(e): + """Extract the resource ids an ec2 api call rejected from its error. + + ec2 names the ids it couldn't use in the error message, ie + ``The instance IDs 'i-0e8f4b9a' do not exist``. + """ + if not EC2_BAD_RESOURCE_ID.match(e.response['Error']['Code']): + return () + return tuple( + rid.strip(" []") + for quoted in QUOTED_RESOURCE_IDS.findall(e.response['Error']['Message']) + for rid in quoted.split(',')) + + +def retry_skip_bad_ids(retry, method, resource_ids, log, **kw): + """Invoke a batched ec2 tag api call, skipping unusable resource ids. + + A resource deleted between enumeration and tagging fails the entire + batch, so drop the ids ec2 rejected and tag the ones still around. + """ + while resource_ids: + try: + return retry(method, Resources=resource_ids, **kw) + except ClientError as e: + bad_ids = set(extract_bad_resource_ids(e)).intersection(resource_ids) + if not bad_ids: + raise + log.info("skipping missing resources %s", ", ".join(sorted(bad_ids))) + resource_ids = [r for r in resource_ids if r not in bad_ids] + # Sentinel: the resolver returns this to omit a tag from a resource's payload. -TAG_VALUE_SKIP = object() +TAG_VALUE_SKIP = Lookup.SKIP def tag_value_schema(): @@ -38,35 +90,18 @@ def tag_value_schema(): Accepts a scalar, a resource lookup by key (with optional fallback), or a conditional default with no key (write only when the tag is absent). + + Tag values are strings, but unquoted YAML scalars have always been + accepted here and coerced on the way out, so the scalar form stays wide. + A fallback has no such history and is held to a string. """ - return { - 'oneOf': [ - {'type': ['string', 'number', 'boolean']}, - { - 'type': 'object', - 'additionalProperties': False, - 'required': ['type', 'key'], - 'properties': { - 'type': {'enum': [Lookup.RESOURCE_SOURCE]}, - 'key': {'type': 'string'}, - 'default-value': {'type': 'string'}, - }, - }, - { - 'type': 'object', - 'additionalProperties': False, - 'required': ['type', 'default-value'], - 'properties': { - 'type': {'enum': [Lookup.RESOURCE_SOURCE]}, - 'default-value': {'type': 'string'}, - }, - }, - ] - } + return Lookup.lookup_type( + {'type': ['string', 'number', 'boolean']}, + default_schema={'type': 'string'}) def has_dynamic_tag_values(spec_map): - return any(Lookup.is_lookup(v) for v in spec_map.values()) + return Lookup.has_lookups(spec_map) def resource_tag_keys(resource): @@ -85,14 +120,76 @@ def resolve_tag_value(spec, tag_name, resource, current_tag_keys): Returns the resolved value, or TAG_VALUE_SKIP to omit the tag. """ - if not Lookup.is_lookup(spec): - return spec - if 'key' in spec: - return Lookup.extract(spec, resource) - # Conditional default (no key): write only if the tag is absent. - if tag_name in current_tag_keys: - return TAG_VALUE_SKIP - return spec['default-value'] + return Lookup.resolve_value(spec, resource, tag_name, current_tag_keys) + + +class TagValueResolver: + """Turn a mapping of tag value specs into concrete tags for a resource. + + A mixin rather than part of Tag itself because the asg tag action isn't a + Tag subclass, but resolves the same per-resource lookups and writes the + same {account_id}/{now}/{region} placeholders. + """ + + def get_interpolation_params(self): + """Placeholder values shared by every tag in a run. + + None of these depend on the resource being tagged, so a caller + resolving values per resource builds them once and passes them back + in rather than re-deriving a fresh {now} for each one. + """ + return { + 'account_id': self.manager.config.account_id, + 'now': utils.FormatDate.utcnow(), + 'region': self.manager.config.region} + + def resolve_and_group(self, resources, spec_map): + """Resolve tag values per resource and group by identical payload. + + Returns a list of (resource_set, resolved_dict). Resources whose + payload is empty (all tags conditionally skipped) are dropped. + """ + groups = {} + # {now} and friends don't vary per resource, so fix them once for the + # whole run -- otherwise a large resource set drifts across seconds and + # splits into a group per timestamp. + params = self.get_interpolation_params() + for r in resources: + resolved = self.resolve_resource_tags(spec_map, r, params) + if not resolved: + continue + sig = tuple(sorted(resolved.items())) + groups.setdefault(sig, ([], resolved))[0].append(r) + return list(groups.values()) + + def resolve_resource_tags(self, spec_map, resource, params): + """Resolve a spec mapping against one resource. + + Returns {name: value} of the tags to write - a conditional default the + resource already carries is left out, so the mapping comes back empty + when there's nothing to do for this resource. + """ + current = resource_tag_keys(resource) + resolved = {} + for name, spec in spec_map.items(): + value = resolve_tag_value(spec, name, resource, current) + if value is TAG_VALUE_SKIP: + continue + resolved[name] = self.interpolate_single_value(value, params) + return resolved + + def interpolate_single_value(self, value, params=None): + """Interpolate placeholders in a single tag value.""" + if params is None: + params = self.get_interpolation_params() + return str(value).format(**params) + + def interpolate_values(self, tags, params=None): + """Interpolate in a list of tags - 'old' ec2 format""" + if params is None: + params = self.get_interpolation_params() + for t in tags: + t['Value'] = self.interpolate_single_value(t['Value'], params) def register_ec2_tags(filters, actions): @@ -424,7 +521,7 @@ def __call__(self, i): return op(tag_count, count) -class Tag(Action): +class Tag(TagValueResolver, Action): """Tag an ec2 resource. Tag values may be looked up per-resource from resource attributes: @@ -499,56 +596,20 @@ def process(self, resources): self.process_resource_set, self.id_key, resources, tags, self.log) return - for resource_set, resolved in self._resolve_and_group(resources, spec_map): + for resource_set, resolved in self.resolve_and_group(resources, spec_map): tags = [{'Key': k, 'Value': v} for k, v in resolved.items()] _common_tag_processer( self.executor_factory, batch_size, self.concurrency, client, self.process_resource_set, self.id_key, resource_set, tags, self.log) - def _resolve_and_group(self, resources, spec_map): - """Resolve tag values per resource and group by identical payload. - - Returns a list of (resource_set, resolved_dict). Resources whose - payload is empty (all tags conditionally skipped) are dropped. - """ - groups = {} - for r in resources: - current = resource_tag_keys(r) - resolved = {} - for name, spec in spec_map.items(): - value = resolve_tag_value(spec, name, r, current) - if value is TAG_VALUE_SKIP: - continue - resolved[name] = self.interpolate_single_value(value) - if not resolved: - continue - sig = tuple(sorted(resolved.items())) - groups.setdefault(sig, ([], resolved))[0].append(r) - return list(groups.values()) - def process_resource_set(self, client, resource_set, tags): mid = self.manager.get_model().id - self.manager.retry( - client.create_tags, - Resources=[v[mid] for v in resource_set], + retry_skip_bad_ids( + self.manager.retry, client.create_tags, + [v[mid] for v in resource_set], self.log, Tags=tags, DryRun=self.manager.config.dryrun) - def interpolate_single_value(self, tag): - """Interpolate in a single tag value. - """ - params = { - 'account_id': self.manager.config.account_id, - 'now': utils.FormatDate.utcnow(), - 'region': self.manager.config.region} - return str(tag).format(**params) - - def interpolate_values(self, tags): - """Interpolate in a list of tags - 'old' ec2 format - """ - for t in tags: - t['Value'] = self.interpolate_single_value(t['Value']) - def get_client(self): return utils.local_session(self.manager.session_factory).client( self.manager.resource_type.service) @@ -584,9 +645,9 @@ def process(self, resources): self.process_resource_set, self.id_key, resources, tags, self.log) def process_resource_set(self, client, resource_set, tag_keys): - return self.manager.retry( - client.delete_tags, - Resources=[v[self.id_key] for v in resource_set], + return retry_skip_bad_ids( + self.manager.retry, client.delete_tags, + [v[self.id_key] for v in resource_set], self.log, Tags=[{'Key': k} for k in tag_keys], DryRun=self.manager.config.dryrun) @@ -986,7 +1047,7 @@ def process(self, resources): self.process_resource_set, self.id_key, resources, tags, self.log) return - for resource_set, resolved in self._resolve_and_group(resources, spec_map): + for resource_set, resolved in self.resolve_and_group(resources, spec_map): _common_tag_processer( self.executor_factory, batch_size, self.concurrency, client, self.process_resource_set, self.id_key, resource_set, resolved, @@ -999,11 +1060,13 @@ def process_resource_set(self, client, resource_set, tags): return universal_retry( client.tag_resources, ResourceARNList=arns, Tags=tags) - def interpolate_values(self, tags): + def interpolate_values(self, tags, params=None): """Interpolate in a list of tags - 'new' resourcegroupstaggingapi format """ + if params is None: + params = self.get_interpolation_params() for key in list(tags.keys()): - tags[key] = self.interpolate_single_value(tags[key]) + tags[key] = self.interpolate_single_value(tags[key], params) def get_client(self): # For global resources, manage tags from us-east-1 @@ -1409,7 +1472,7 @@ def universal_retry(method, ResourceARNList, **kw): error_code = failures[f_arn]['ErrorCode'] if error_code == 'ThrottlingException': throttles.add(f_arn) - elif error_code == 'ResourceNotFoundException': + elif is_resource_gone(error_code): continue else: errors[f_arn] = error_code @@ -1417,6 +1480,10 @@ def universal_retry(method, ResourceARNList, **kw): if errors: raise Exception("Resource Tag Errors %s" % (errors)) + if not throttles: + # everything else was skipped, the api rejects an empty arn list + return response + if idx == max_attempts - 1: raise Exception("Resource Tag Throttled %s" % (", ".join(throttles))) diff --git a/c7n/version.py b/c7n/version.py index e04cbc94a77..32ff6e9a53c 100644 --- a/c7n/version.py +++ b/c7n/version.py @@ -1,2 +1,2 @@ # Generated via tools/dev/devpkg.py -version = "0.9.52a5.post1+a83dda5c9" +version = "0.9.52a5.post2+a83dda5c9" diff --git a/pyproject.toml b/pyproject.toml index 43b6d397fed..3d5e20cc1b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ "urllib3>2", ] name = "c7n" -version = "0.9.52a5.post1+a83dda5c9" +version = "0.9.52a5.post2+a83dda5c9" description = "Cloud Custodian - Policy Rules Engine" readme = "README.md" classifiers = [ diff --git a/tests/common.py b/tests/common.py index 220a2efd92f..3fa93deaeb0 100644 --- a/tests/common.py +++ b/tests/common.py @@ -10,7 +10,7 @@ from c7n.testing import TestUtils, TextTestIO, functional # NOQA -from .zpill import PillTest, ACCOUNT_ID +from .zpill import PillTest, ACCOUNT_ID # NOQA logging.getLogger("placebo.pill").setLevel(logging.DEBUG) @@ -33,10 +33,6 @@ class BaseTest(TestUtils, PillTest): # custodian_schema = C7N_SCHEMA - @property - def account_id(self): - return ACCOUNT_ID - def _get_policy_config(self, **kw): if 'account_id' not in kw: kw['account_id'] = self.account_id diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListDashboards_1.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListDashboards_1.json new file mode 100644 index 00000000000..2b5e0228c20 --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListDashboards_1.json @@ -0,0 +1,23 @@ +{ + "status_code": 200, + "data": { + "DashboardEntries": [ + { + "DashboardName": "c7n-test-e3dd19005e27e4f1", + "DashboardArn": "arn:aws:cloudwatch::644160558196:dashboard/c7n-test-e3dd19005e27e4f1", + "LastModified": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 8, + "hour": 22, + "minute": 59, + "second": 16, + "microsecond": 0 + }, + "Size": 158 + } + ], + "ResponseMetadata": {} + } +} diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListDashboards_2.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListDashboards_2.json new file mode 100644 index 00000000000..9280faa7708 --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListDashboards_2.json @@ -0,0 +1,23 @@ +{ + "status_code": 200, + "data": { + "DashboardEntries": [ + { + "DashboardName": "c7n-test-e3dd19005e27e4f1", + "DashboardArn": "arn:aws:cloudwatch::644160558196:dashboard/c7n-test-e3dd19005e27e4f1", + "LastModified": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 8, + "hour": 22, + "minute": 59, + "second": 17, + "microsecond": 0 + }, + "Size": 346 + } + ], + "ResponseMetadata": {} + } +} diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_1.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_1.json new file mode 100644 index 00000000000..e41b16b3e98 --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_1.json @@ -0,0 +1,7 @@ +{ + "status_code": 200, + "data": { + "Tags": [], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_2.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_2.json new file mode 100644 index 00000000000..2ce4832506c --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_2.json @@ -0,0 +1,16 @@ +{ + "status_code": 200, + "data": { + "Tags": [ + { + "Key": "App", + "Value": "Custodian" + }, + { + "Key": "Env", + "Value": "Dev" + } + ], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_3.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_3.json new file mode 100644 index 00000000000..2ce4832506c --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_3.json @@ -0,0 +1,16 @@ +{ + "status_code": 200, + "data": { + "Tags": [ + { + "Key": "App", + "Value": "Custodian" + }, + { + "Key": "Env", + "Value": "Dev" + } + ], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_4.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_4.json new file mode 100644 index 00000000000..954290d0fc1 --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_4.json @@ -0,0 +1,12 @@ +{ + "status_code": 200, + "data": { + "Tags": [ + { + "Key": "App", + "Value": "Custodian" + } + ], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.TagResource_1.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.TagResource_1.json new file mode 100644 index 00000000000..5b2170a073c --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.TagResource_1.json @@ -0,0 +1,6 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.UntagResource_1.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.UntagResource_1.json new file mode 100644 index 00000000000..5b2170a073c --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.UntagResource_1.json @@ -0,0 +1,6 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_core_network_delete_error/.gitkeep b/tests/data/placebo/test_core_network_delete_error/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/placebo/test_eks_tag_deleted_cluster/eks.TagResource_1.json b/tests/data/placebo/test_eks_tag_deleted_cluster/eks.TagResource_1.json new file mode 100644 index 00000000000..b067a0dbc0b --- /dev/null +++ b/tests/data/placebo/test_eks_tag_deleted_cluster/eks.TagResource_1.json @@ -0,0 +1,11 @@ +{ + "status_code": 404, + "data": { + "Error": { + "Message": "Resource was not found", + "Code": "NotFoundException" + }, + "ResponseMetadata": {}, + "message": "Resource was not found" + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_eks_tag_deleted_cluster/eks.UntagResource_1.json b/tests/data/placebo/test_eks_tag_deleted_cluster/eks.UntagResource_1.json new file mode 100644 index 00000000000..b067a0dbc0b --- /dev/null +++ b/tests/data/placebo/test_eks_tag_deleted_cluster/eks.UntagResource_1.json @@ -0,0 +1,11 @@ +{ + "status_code": 404, + "data": { + "Error": { + "Message": "Resource was not found", + "Code": "NotFoundException" + }, + "ResponseMetadata": {}, + "message": "Resource was not found" + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.CreateTags_1.json b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.CreateTags_1.json new file mode 100644 index 00000000000..ebc15c522c0 --- /dev/null +++ b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.CreateTags_1.json @@ -0,0 +1,10 @@ +{ + "status_code": 400, + "data": { + "Error": { + "Code": "InvalidInstanceID.NotFound", + "Message": "The instance ID 'i-0023456789abcdef0' does not exist" + }, + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.CreateTags_2.json b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.CreateTags_2.json new file mode 100644 index 00000000000..5b2170a073c --- /dev/null +++ b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.CreateTags_2.json @@ -0,0 +1,6 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DeleteTags_1.json b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DeleteTags_1.json new file mode 100644 index 00000000000..5b2170a073c --- /dev/null +++ b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DeleteTags_1.json @@ -0,0 +1,6 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstanceImageMetadata_1.json b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstanceImageMetadata_1.json new file mode 100644 index 00000000000..77c70a527a7 --- /dev/null +++ b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstanceImageMetadata_1.json @@ -0,0 +1,132 @@ +{ + "status_code": 200, + "data": { + "InstanceImageMetadata": [ + { + "InstanceId": "i-09347f9d6785dbe53", + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 32, + "second": 26, + "microsecond": 0 + }, + "AvailabilityZone": "us-east-1f", + "ZoneId": "use1-az5", + "State": { + "Code": 48, + "Name": "terminated" + }, + "OwnerId": "644160558196", + "ImageMetadata": { + "ImageId": "ami-0757b090c88c85bab", + "Name": "amzn2-ami-hvm-2.0.20260908.0-x86_64-gp2", + "OwnerId": "644160558196", + "State": "available", + "ImageOwnerAlias": "amazon", + "CreationDate": "2026-09-08T17:09:15Z", + "DeprecationTime": "2026-12-07T17:11:00Z", + "IsPublic": true + } + }, + { + "InstanceId": "i-0fd2708167c1cdb5a", + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 32, + "second": 26, + "microsecond": 0 + }, + "AvailabilityZone": "us-east-1f", + "ZoneId": "use1-az5", + "State": { + "Code": 48, + "Name": "terminated" + }, + "OwnerId": "644160558196", + "ImageMetadata": { + "ImageId": "ami-0757b090c88c85bab", + "Name": "amzn2-ami-hvm-2.0.20260908.0-x86_64-gp2", + "OwnerId": "644160558196", + "State": "available", + "ImageOwnerAlias": "amazon", + "CreationDate": "2026-09-08T17:09:15Z", + "DeprecationTime": "2026-12-07T17:11:00Z", + "IsPublic": true + } + }, + { + "InstanceId": "i-0e6f711d31e6675f4", + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "AvailabilityZone": "us-east-1c", + "ZoneId": "use1-az2", + "State": { + "Code": 16, + "Name": "running" + }, + "OwnerId": "644160558196", + "ImageMetadata": { + "ImageId": "ami-0757b090c88c85bab", + "Name": "amzn2-ami-hvm-2.0.20260908.0-x86_64-gp2", + "OwnerId": "644160558196", + "State": "available", + "ImageOwnerAlias": "amazon", + "CreationDate": "2026-09-08T17:09:15Z", + "DeprecationTime": "2026-12-07T17:11:00Z", + "IsPublic": true + } + }, + { + "InstanceId": "i-042a559882c4907dc", + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "AvailabilityZone": "us-east-1c", + "ZoneId": "use1-az2", + "State": { + "Code": 16, + "Name": "running" + }, + "OwnerId": "644160558196", + "ImageMetadata": { + "ImageId": "ami-0757b090c88c85bab", + "Name": "amzn2-ami-hvm-2.0.20260908.0-x86_64-gp2", + "OwnerId": "644160558196", + "State": "available", + "ImageOwnerAlias": "amazon", + "CreationDate": "2026-09-08T17:09:15Z", + "DeprecationTime": "2026-12-07T17:11:00Z", + "IsPublic": true + } + } + ], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstances_1.json b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstances_1.json new file mode 100644 index 00000000000..737c83c3c3d --- /dev/null +++ b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstances_1.json @@ -0,0 +1,374 @@ +{ + "status_code": 200, + "data": { + "Reservations": [ + { + "ReservationId": "r-00cf274ff3ee020d9", + "OwnerId": "644160558196", + "Groups": [], + "Instances": [ + { + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 54, + "microsecond": 0 + }, + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-0e9414ef1d2eb432c", + "EbsCardIndex": 0 + } + } + ], + "ClientToken": "terraform-aHfqApEb7WCuXQ32mgMxEZBCP7", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "AttachmentId": "eni-attach-026ad86d2c3a449f5", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "Ipv6Addresses": [], + "MacAddress": "12:cd:c8:af:f4:41", + "NetworkInterfaceId": "eni-0816616bd448027b2", + "OwnerId": "644160558196", + "PrivateIpAddress": "10.0.1.200", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.0.1.200" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "InterfaceType": "interface", + "Operator": { + "Managed": false + } + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "SourceDestCheck": true, + "Tags": [ + { + "Key": "App", + "Value": "Custodian" + } + ], + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default", + "RebootMigration": "default" + }, + "CurrentInstanceBootMode": "legacy-bios", + "NetworkPerformanceOptions": { + "BandwidthWeighting": "default" + }, + "Operator": { + "Managed": false, + "HiddenByDefault": false + }, + "SecondaryInterfaces": [], + "InstanceId": "i-042a559882c4907dc", + "ImageId": "ami-0757b090c88c85bab", + "State": { + "Code": 16, + "Name": "running" + }, + "PrivateDnsName": "ip-10-0-1-200.ec2.internal", + "PublicDnsName": "", + "StateTransitionReason": "", + "AmiLaunchIndex": 0, + "ProductCodes": [], + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "Placement": { + "AvailabilityZoneId": "use1-az2", + "GroupName": "", + "Tenancy": "default", + "AvailabilityZone": "us-east-1c" + }, + "Monitoring": { + "State": "disabled" + }, + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "PrivateIpAddress": "10.0.1.200" + } + ] + }, + { + "ReservationId": "r-0bc85187826350a15", + "OwnerId": "644160558196", + "Groups": [], + "Instances": [ + { + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 54, + "microsecond": 0 + }, + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-039e696c5d3bd6f13", + "EbsCardIndex": 0 + } + } + ], + "ClientToken": "terraform-f8i4majTXcSBEqpc3kEVYJlrjG", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "AttachmentId": "eni-attach-074366427f4cb6808", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "Ipv6Addresses": [], + "MacAddress": "12:68:a3:b5:b6:99", + "NetworkInterfaceId": "eni-086f9944f87e7b575", + "OwnerId": "644160558196", + "PrivateIpAddress": "10.0.1.73", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.0.1.73" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "InterfaceType": "interface", + "Operator": { + "Managed": false + } + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "SourceDestCheck": true, + "Tags": [ + { + "Key": "App", + "Value": "Custodian" + } + ], + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default", + "RebootMigration": "default" + }, + "CurrentInstanceBootMode": "legacy-bios", + "NetworkPerformanceOptions": { + "BandwidthWeighting": "default" + }, + "Operator": { + "Managed": false, + "HiddenByDefault": false + }, + "SecondaryInterfaces": [], + "InstanceId": "i-0e6f711d31e6675f4", + "ImageId": "ami-0757b090c88c85bab", + "State": { + "Code": 16, + "Name": "running" + }, + "PrivateDnsName": "ip-10-0-1-73.ec2.internal", + "PublicDnsName": "", + "StateTransitionReason": "", + "AmiLaunchIndex": 0, + "ProductCodes": [], + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "Placement": { + "AvailabilityZoneId": "use1-az2", + "GroupName": "", + "Tenancy": "default", + "AvailabilityZone": "us-east-1c" + }, + "Monitoring": { + "State": "disabled" + }, + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "PrivateIpAddress": "10.0.1.73" + } + ] + } + ], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstances_2.json b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstances_2.json new file mode 100644 index 00000000000..773b61c1ae8 --- /dev/null +++ b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstances_2.json @@ -0,0 +1,362 @@ +{ + "status_code": 200, + "data": { + "Reservations": [ + { + "ReservationId": "r-00cf274ff3ee020d9", + "OwnerId": "644160558196", + "Groups": [], + "Instances": [ + { + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 54, + "microsecond": 0 + }, + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-0e9414ef1d2eb432c", + "EbsCardIndex": 0 + } + } + ], + "ClientToken": "terraform-aHfqApEb7WCuXQ32mgMxEZBCP7", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "AttachmentId": "eni-attach-026ad86d2c3a449f5", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "Ipv6Addresses": [], + "MacAddress": "12:cd:c8:af:f4:41", + "NetworkInterfaceId": "eni-0816616bd448027b2", + "OwnerId": "644160558196", + "PrivateIpAddress": "10.0.1.200", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.0.1.200" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "InterfaceType": "interface", + "Operator": { + "Managed": false + } + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "SourceDestCheck": true, + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default", + "RebootMigration": "default" + }, + "CurrentInstanceBootMode": "legacy-bios", + "NetworkPerformanceOptions": { + "BandwidthWeighting": "default" + }, + "Operator": { + "Managed": false, + "HiddenByDefault": false + }, + "SecondaryInterfaces": [], + "InstanceId": "i-042a559882c4907dc", + "ImageId": "ami-0757b090c88c85bab", + "State": { + "Code": 16, + "Name": "running" + }, + "PrivateDnsName": "ip-10-0-1-200.ec2.internal", + "PublicDnsName": "", + "StateTransitionReason": "", + "AmiLaunchIndex": 0, + "ProductCodes": [], + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "Placement": { + "AvailabilityZoneId": "use1-az2", + "GroupName": "", + "Tenancy": "default", + "AvailabilityZone": "us-east-1c" + }, + "Monitoring": { + "State": "disabled" + }, + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "PrivateIpAddress": "10.0.1.200" + } + ] + }, + { + "ReservationId": "r-0bc85187826350a15", + "OwnerId": "644160558196", + "Groups": [], + "Instances": [ + { + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 54, + "microsecond": 0 + }, + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-039e696c5d3bd6f13", + "EbsCardIndex": 0 + } + } + ], + "ClientToken": "terraform-f8i4majTXcSBEqpc3kEVYJlrjG", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "AttachmentId": "eni-attach-074366427f4cb6808", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "Ipv6Addresses": [], + "MacAddress": "12:68:a3:b5:b6:99", + "NetworkInterfaceId": "eni-086f9944f87e7b575", + "OwnerId": "644160558196", + "PrivateIpAddress": "10.0.1.73", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.0.1.73" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "InterfaceType": "interface", + "Operator": { + "Managed": false + } + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "SourceDestCheck": true, + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default", + "RebootMigration": "default" + }, + "CurrentInstanceBootMode": "legacy-bios", + "NetworkPerformanceOptions": { + "BandwidthWeighting": "default" + }, + "Operator": { + "Managed": false, + "HiddenByDefault": false + }, + "SecondaryInterfaces": [], + "InstanceId": "i-0e6f711d31e6675f4", + "ImageId": "ami-0757b090c88c85bab", + "State": { + "Code": 16, + "Name": "running" + }, + "PrivateDnsName": "ip-10-0-1-73.ec2.internal", + "PublicDnsName": "", + "StateTransitionReason": "", + "AmiLaunchIndex": 0, + "ProductCodes": [], + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "Placement": { + "AvailabilityZoneId": "use1-az2", + "GroupName": "", + "Tenancy": "default", + "AvailabilityZone": "us-east-1c" + }, + "Monitoring": { + "State": "disabled" + }, + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "PrivateIpAddress": "10.0.1.73" + } + ] + } + ], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_universal_tag_skips_gone_snapshot/tagging.TagResources_1.json b/tests/data/placebo/test_universal_tag_skips_gone_snapshot/tagging.TagResources_1.json new file mode 100644 index 00000000000..b3fea6b6c33 --- /dev/null +++ b/tests/data/placebo/test_universal_tag_skips_gone_snapshot/tagging.TagResources_1.json @@ -0,0 +1,13 @@ +{ + "status_code": 200, + "data": { + "FailedResourcesMap": { + "arn:aws:elasticache:us-east-1:644160558196:snapshot:c7n-deleted": { + "StatusCode": 404, + "ErrorCode": "SnapshotNotFoundFault", + "ErrorMessage": "c7n-deleted is either not present or not available" + } + }, + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/terraform/cloudwatch_dashboard_tag/main.tf b/tests/terraform/cloudwatch_dashboard_tag/main.tf new file mode 100644 index 00000000000..6d48460fe8b --- /dev/null +++ b/tests/terraform/cloudwatch_dashboard_tag/main.tf @@ -0,0 +1,26 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +provider "aws" {} + +resource "random_id" "id" { + byte_length = 8 +} + +resource "aws_cloudwatch_dashboard" "test" { + dashboard_name = "c7n-test-${random_id.id.hex}" + + dashboard_body = jsonencode({ + widgets = [ + { + type = "text" + x = 0 + y = 0 + width = 6 + height = 6 + properties = { + markdown = "cloud custodian test dashboard" + } + } + ] + }) +} diff --git a/tests/terraform/cloudwatch_dashboard_tag/tf_resources.json b/tests/terraform/cloudwatch_dashboard_tag/tf_resources.json new file mode 100644 index 00000000000..61561c8fbc4 --- /dev/null +++ b/tests/terraform/cloudwatch_dashboard_tag/tf_resources.json @@ -0,0 +1,27 @@ +{ + "pytest-terraform": 1, + "outputs": {}, + "resources": { + "aws_cloudwatch_dashboard": { + "test": { + "dashboard_arn": "arn:aws:cloudwatch::644160558196:dashboard/c7n-test-e3dd19005e27e4f1", + "dashboard_body": "{\"widgets\":[{\"height\":6,\"properties\":{\"markdown\":\"cloud custodian test dashboard\"},\"type\":\"text\",\"width\":6,\"x\":0,\"y\":0}]}", + "dashboard_name": "c7n-test-e3dd19005e27e4f1", + "id": "c7n-test-e3dd19005e27e4f1", + "region": "us-east-2" + } + }, + "random_id": { + "id": { + "b64_std": "490ZAF4n5PE=", + "b64_url": "490ZAF4n5PE", + "byte_length": 8, + "dec": "16419307305833063665", + "hex": "e3dd19005e27e4f1", + "id": "490ZAF4n5PE", + "keepers": null, + "prefix": null + } + } + } +} \ No newline at end of file diff --git a/tests/terraform/ec2_instance_tag/main.tf b/tests/terraform/ec2_instance_tag/main.tf new file mode 100644 index 00000000000..61796ec1aac --- /dev/null +++ b/tests/terraform/ec2_instance_tag/main.tf @@ -0,0 +1,36 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +provider "aws" {} + +data "aws_ami" "amazon_linux" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["amzn2-ami-hvm-*-x86_64-gp2"] + } +} + +resource "aws_vpc" "example" { + cidr_block = "10.0.0.0/16" +} + +resource "aws_subnet" "example" { + vpc_id = aws_vpc.example.id + cidr_block = "10.0.1.0/24" +} + +# two instances, so a batch that loses one id still has more than one +# survivor to prove the retry keeps all of them +resource "aws_instance" "first" { + ami = data.aws_ami.amazon_linux.id + instance_type = "t2.micro" + subnet_id = aws_subnet.example.id +} + +resource "aws_instance" "second" { + ami = data.aws_ami.amazon_linux.id + instance_type = "t2.micro" + subnet_id = aws_subnet.example.id +} diff --git a/tests/terraform/ec2_instance_tag/tf_resources.json b/tests/terraform/ec2_instance_tag/tf_resources.json new file mode 100644 index 00000000000..06b1ea31101 --- /dev/null +++ b/tests/terraform/ec2_instance_tag/tf_resources.json @@ -0,0 +1,392 @@ +{ + "pytest-terraform": 1, + "outputs": {}, + "resources": { + "aws_ami": { + "amazon_linux": { + "allow_unsafe_filter": null, + "architecture": "x86_64", + "arn": "arn:aws:ec2:us-east-1::image/ami-0757b090c88c85bab", + "block_device_mappings": [ + { + "device_name": "/dev/xvda", + "ebs": { + "delete_on_termination": "true", + "encrypted": "false", + "iops": "0", + "snapshot_id": "snap-0b2af24c3d555bc22", + "throughput": "0", + "volume_initialization_rate": "0", + "volume_size": "8", + "volume_type": "gp2" + }, + "no_device": "", + "virtual_name": "" + } + ], + "boot_mode": "", + "creation_date": "2026-09-08T17:09:15.000Z", + "deprecation_time": "2026-12-07T17:11:00.000Z", + "description": "Amazon Linux 2 AMI 2.0.20260908.0 x86_64 HVM gp2", + "ena_support": true, + "executable_users": null, + "filter": [ + { + "name": "name", + "values": [ + "amzn2-ami-hvm-*-x86_64-gp2" + ] + } + ], + "hypervisor": "xen", + "id": "ami-0757b090c88c85bab", + "image_id": "ami-0757b090c88c85bab", + "image_location": "amazon/amzn2-ami-hvm-2.0.20260908.0-x86_64-gp2", + "image_owner_alias": "amazon", + "image_type": "machine", + "imds_support": "", + "include_deprecated": false, + "kernel_id": "", + "last_launched_time": "", + "most_recent": true, + "name": "amzn2-ami-hvm-2.0.20260908.0-x86_64-gp2", + "name_regex": null, + "owner_id": "644160558196", + "owners": [ + "amazon" + ], + "platform": "", + "platform_details": "Linux/UNIX", + "product_codes": [], + "public": true, + "ramdisk_id": "", + "region": "us-east-1", + "root_device_name": "/dev/xvda", + "root_device_type": "ebs", + "root_snapshot_id": "snap-0b2af24c3d555bc22", + "sriov_net_support": "simple", + "state": "available", + "state_reason": { + "code": "UNSET", + "message": "UNSET" + }, + "tags": {}, + "timeouts": null, + "tpm_support": "", + "uefi_data": null, + "usage_operation": "RunInstances", + "virtualization_type": "hvm" + } + }, + "aws_instance": { + "first": { + "ami": "ami-0757b090c88c85bab", + "arn": "arn:aws:ec2:us-east-1:644160558196:instance/i-0e6f711d31e6675f4", + "associate_public_ip_address": false, + "availability_zone": "us-east-1c", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_options": [ + { + "amd_sev_snp": "", + "core_count": 1, + "nested_virtualization": "", + "threads_per_core": 1 + } + ], + "credit_specification": [ + { + "cpu_credits": "standard" + } + ], + "disable_api_stop": false, + "disable_api_termination": false, + "ebs_block_device": [], + "ebs_optimized": false, + "enable_primary_ipv6": null, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "force_destroy": false, + "get_password_data": false, + "hibernation": false, + "host_id": "", + "host_resource_group_arn": null, + "iam_instance_profile": "", + "id": "i-0e6f711d31e6675f4", + "instance_initiated_shutdown_behavior": "stop", + "instance_lifecycle": "", + "instance_market_options": [], + "instance_state": "running", + "instance_type": "t2.micro", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "", + "launch_template": [], + "maintenance_options": [ + { + "auto_recovery": "default" + } + ], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_protocol_ipv6": "disabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional", + "instance_metadata_tags": "disabled" + } + ], + "monitoring": false, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_group_id": "", + "placement_partition_number": 0, + "primary_network_interface": [ + { + "delete_on_termination": true, + "network_interface_id": "eni-086f9944f87e7b575" + } + ], + "primary_network_interface_id": "eni-086f9944f87e7b575", + "private_dns": "ip-10-0-1-73.ec2.internal", + "private_dns_name_options": [ + { + "enable_resource_name_dns_a_record": false, + "enable_resource_name_dns_aaaa_record": false, + "hostname_type": "ip-name" + } + ], + "private_ip": "10.0.1.73", + "public_dns": "", + "public_ip": "", + "region": "us-east-1", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": true, + "iops": 100, + "kms_key_id": "arn:aws:kms:us-east-1:644160558196:key/04fa09fb-afe3-46c8-ad3c-410b8047f7b9", + "tags": {}, + "tags_all": {}, + "throughput": 0, + "volume_id": "vol-039e696c5d3bd6f13", + "volume_size": 8, + "volume_type": "gp2" + } + ], + "secondary_network_interface": [], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "spot_instance_request_id": "", + "subnet_id": "subnet-01fd2d48d6f162150", + "tags": null, + "tags_all": {}, + "tenancy": "default", + "timeouts": null, + "user_data": null, + "user_data_base64": null, + "user_data_replace_on_change": false, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-0034c1e69b8e6bcc5" + ] + }, + "second": { + "ami": "ami-0757b090c88c85bab", + "arn": "arn:aws:ec2:us-east-1:644160558196:instance/i-042a559882c4907dc", + "associate_public_ip_address": false, + "availability_zone": "us-east-1c", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_options": [ + { + "amd_sev_snp": "", + "core_count": 1, + "nested_virtualization": "", + "threads_per_core": 1 + } + ], + "credit_specification": [ + { + "cpu_credits": "standard" + } + ], + "disable_api_stop": false, + "disable_api_termination": false, + "ebs_block_device": [], + "ebs_optimized": false, + "enable_primary_ipv6": null, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "force_destroy": false, + "get_password_data": false, + "hibernation": false, + "host_id": "", + "host_resource_group_arn": null, + "iam_instance_profile": "", + "id": "i-042a559882c4907dc", + "instance_initiated_shutdown_behavior": "stop", + "instance_lifecycle": "", + "instance_market_options": [], + "instance_state": "running", + "instance_type": "t2.micro", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "", + "launch_template": [], + "maintenance_options": [ + { + "auto_recovery": "default" + } + ], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_protocol_ipv6": "disabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional", + "instance_metadata_tags": "disabled" + } + ], + "monitoring": false, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_group_id": "", + "placement_partition_number": 0, + "primary_network_interface": [ + { + "delete_on_termination": true, + "network_interface_id": "eni-0816616bd448027b2" + } + ], + "primary_network_interface_id": "eni-0816616bd448027b2", + "private_dns": "ip-10-0-1-200.ec2.internal", + "private_dns_name_options": [ + { + "enable_resource_name_dns_a_record": false, + "enable_resource_name_dns_aaaa_record": false, + "hostname_type": "ip-name" + } + ], + "private_ip": "10.0.1.200", + "public_dns": "", + "public_ip": "", + "region": "us-east-1", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": true, + "iops": 100, + "kms_key_id": "arn:aws:kms:us-east-1:644160558196:key/04fa09fb-afe3-46c8-ad3c-410b8047f7b9", + "tags": {}, + "tags_all": {}, + "throughput": 0, + "volume_id": "vol-0e9414ef1d2eb432c", + "volume_size": 8, + "volume_type": "gp2" + } + ], + "secondary_network_interface": [], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "spot_instance_request_id": "", + "subnet_id": "subnet-01fd2d48d6f162150", + "tags": null, + "tags_all": {}, + "tenancy": "default", + "timeouts": null, + "user_data": null, + "user_data_base64": null, + "user_data_replace_on_change": false, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-0034c1e69b8e6bcc5" + ] + } + }, + "aws_subnet": { + "example": { + "arn": "arn:aws:ec2:us-east-1:644160558196:subnet/subnet-01fd2d48d6f162150", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-1c", + "availability_zone_id": "use1-az2", + "cidr_block": "10.0.1.0/24", + "customer_owned_ipv4_pool": "", + "enable_dns64": false, + "enable_lni_at_device_index": 0, + "enable_resource_name_dns_a_record_on_launch": false, + "enable_resource_name_dns_aaaa_record_on_launch": false, + "id": "subnet-01fd2d48d6f162150", + "ipv4_ipam_pool_id": null, + "ipv4_netmask_length": null, + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "ipv6_ipam_pool_id": null, + "ipv6_native": false, + "ipv6_netmask_length": null, + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "644160558196", + "private_dns_hostname_type_on_launch": "ip-name", + "region": "us-east-1", + "tags": null, + "tags_all": {}, + "timeouts": null, + "vpc_id": "vpc-0977a508b64394d7e" + } + }, + "aws_vpc": { + "example": { + "arn": "arn:aws:ec2:us-east-1:644160558196:vpc/vpc-0977a508b64394d7e", + "assign_generated_ipv6_cidr_block": false, + "cidr_block": "10.0.0.0/16", + "default_network_acl_id": "acl-05bc7b1ba43105f96", + "default_route_table_id": "rtb-02919e3e487a62814", + "default_security_group_id": "sg-0034c1e69b8e6bcc5", + "dhcp_options_id": "dopt-05763bf1939671093", + "enable_dns_hostnames": false, + "enable_dns_support": true, + "enable_network_address_usage_metrics": false, + "id": "vpc-0977a508b64394d7e", + "instance_tenancy": "default", + "ipv4_ipam_pool_id": null, + "ipv4_netmask_length": null, + "ipv6_association_id": "", + "ipv6_cidr_block": "", + "ipv6_cidr_block_network_border_group": "", + "ipv6_ipam_pool_id": "", + "ipv6_netmask_length": 0, + "main_route_table_id": "rtb-02919e3e487a62814", + "owner_id": "644160558196", + "region": "us-east-1", + "tags": null, + "tags_all": {} + } + } + } +} \ No newline at end of file diff --git a/tests/test_asg.py b/tests/test_asg.py index a03374051a2..76eb35a5e36 100644 --- a/tests/test_asg.py +++ b/tests/test_asg.py @@ -11,9 +11,9 @@ from .common import BaseTest from c7n.exceptions import PolicyValidationError -from c7n.resources.asg import LaunchInfo +from c7n.resources.asg import LaunchInfo, Tag as AsgTag from c7n.resources.aws import shape_validate -from c7n.utils import jmespath_search +from c7n.utils import FormatDate, jmespath_search class LaunchConfigTest(BaseTest): @@ -1178,3 +1178,173 @@ def test_asg_scaling_policy_enabled(self): resources = p.run() self.assertEqual(len(resources), 1) self.assertTrue(resources[0].get('Enabled')) + + +class AsgDynamicTagTest(BaseTest): + """Dynamic tag values on the asg-specific tag action. + + The asg tag action is not a c7n.tags.Tag subclass, so it needs its own + wiring for the resource lookup form of a tag value. + """ + + def _run(self, asgs, action): + mock_factory = mock.MagicMock() + mock_factory.region = 'us-east-1' + create_tags = mock_factory().client('autoscaling').create_or_update_tags + create_tags.return_value = {} + policy = self.load_policy( + {"name": "d", "resource": "asg", "actions": [action]}, + session_factory=mock_factory) + policy.resource_manager.actions[0].process(asgs) + return create_tags + + def test_lookup_key_hit(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-1", + "Tags": [{"Key": "Team", "Value": "platform"}]}], + {"type": "tag", "tags": {"Owner": { + "type": "resource", "key": "Tags[?Key=='Team'].Value | [0]"}}}) + create_tags.assert_called_once_with( + Tags=[{"Key": "Owner", "Value": "platform", + "PropagateAtLaunch": False, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-1"}]) + + def test_lookup_key_miss_uses_default(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-1"}], + {"type": "tag", "tags": {"Owner": { + "type": "resource", "key": "Nope", "default-value": "unknown"}}}) + create_tags.assert_called_once_with( + Tags=[{"Key": "Owner", "Value": "unknown", + "PropagateAtLaunch": False, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-1"}]) + + def test_conditional_default_skips_asg_with_tag_present(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-has", + "Tags": [{"Key": "Owner", "Value": "keep-me"}]}], + {"type": "tag", "tags": {"Owner": { + "type": "resource", "default-value": "unassigned"}}}) + create_tags.assert_not_called() + + def test_conditional_default_writes_per_asg(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-has", + "Tags": [{"Key": "Owner", "Value": "keep-me"}]}, + {"AutoScalingGroupName": "asg-missing"}], + {"type": "tag", "tags": {"Owner": { + "type": "resource", "default-value": "unassigned"}}}) + create_tags.assert_called_once_with( + Tags=[{"Key": "Owner", "Value": "unassigned", + "PropagateAtLaunch": False, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-missing"}]) + + def test_lookup_in_single_value_form(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-1", + "Tags": [{"Key": "Team", "Value": "platform"}]}], + {"type": "tag", "key": "Owner", "propagate": True, + "value": {"type": "resource", "key": "Tags[?Key=='Team'].Value | [0]"}}) + create_tags.assert_called_once_with( + Tags=[{"Key": "Owner", "Value": "platform", + "PropagateAtLaunch": True, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-1"}]) + + def test_placeholder_interpolated_in_lookup_default(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-1"}], + {"type": "tag", "tags": {"Stamp": { + "type": "resource", "key": "Nope", + "default-value": "made-in-{region}"}}}) + create_tags.assert_called_once_with( + Tags=[{"Key": "Stamp", "Value": "made-in-us-east-1", + "PropagateAtLaunch": False, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-1"}]) + + def test_dynamic_values_batch_by_payload(self): + # batch_size is 1 today, so raise it to show the dynamic path batches + # asgs that resolve to the same payload rather than going one at a time + asgs = [{"AutoScalingGroupName": "asg-1", + "Tags": [{"Key": "Env", "Value": "prod"}]}, + {"AutoScalingGroupName": "asg-2", + "Tags": [{"Key": "Env", "Value": "dev"}]}, + {"AutoScalingGroupName": "asg-3", + "Tags": [{"Key": "Env", "Value": "prod"}]}] + with mock.patch.object(AsgTag, "batch_size", 2): + create_tags = self._run( + asgs, + {"type": "tag", "tags": {"Tier": { + "type": "resource", "key": "Tags[?Key=='Env'].Value | [0]"}}}) + self.assertEqual(create_tags.call_count, 2) + by_value = {c.kwargs["Tags"][0]["Value"]: {t["ResourceId"] for t in c.kwargs["Tags"]} + for c in create_tags.call_args_list} + self.assertEqual(by_value["prod"], {"asg-1", "asg-3"}) + self.assertEqual(by_value["dev"], {"asg-2"}) + + def test_now_resolved_once_for_the_whole_run(self): + # a clock that moves on every read - the run should only read it once, + # so both asgs get stamped alike even though batch_size sends them + # in separate calls + ticks = [FormatDate(datetime(2022, 6, 27, 12, 34, 56)), + FormatDate(datetime(2022, 6, 27, 12, 34, 57))] + with mock.patch.object(FormatDate, "utcnow", side_effect=ticks) as utcnow: + create_tags = self._run( + [{"AutoScalingGroupName": "asg-1"}, + {"AutoScalingGroupName": "asg-2"}], + {"type": "tag", "tags": {"Stamp": { + "type": "resource", "key": "Nope", + "default-value": "at-{now}"}}}) + self.assertEqual(utcnow.call_count, 1) + stamped = [c.kwargs["Tags"][0]["Value"] for c in create_tags.call_args_list] + self.assertEqual(stamped, ["at-2022-06-27 12:34:56"] * 2) + + def test_malformed_tag_value_rejected(self): + with self.assertRaises(PolicyValidationError): + self.load_policy( + {"name": "d", "resource": "asg", + "actions": [{"type": "tag", "tags": {"Owner": {"foo": "bar"}}}]}, + validate=True) + + def test_tags_mapping_does_not_add_default_marker(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-1"}], + {"type": "tag", "tags": {"Owner": "platform"}}) + create_tags.assert_called_once_with( + Tags=[{"Key": "Owner", "Value": "platform", + "PropagateAtLaunch": False, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-1"}]) + + def test_bare_mark_still_writes_default_marker(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-1"}], {"type": "mark"}) + create_tags.assert_called_once_with( + Tags=[{"Key": "maid_status", + "Value": "AutoScaleGroup does not meet policy guidelines", + "PropagateAtLaunch": False, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-1"}]) + + def test_auto_tag_user_does_not_add_default_marker(self): + mock_factory = mock.MagicMock() + mock_factory.region = 'us-east-1' + create_tags = mock_factory().client('autoscaling').create_or_update_tags + create_tags.return_value = {} + policy = self.load_policy( + {"name": "d", "resource": "asg", + "mode": {"type": "cloudtrail", "events": ["CreateAutoScalingGroup"]}, + "actions": [{"type": "auto-tag-user", "tag": "CreatorName"}]}, + session_factory=mock_factory) + action = policy.resource_manager.actions[0] + action.set_resource_tags( + {"CreatorName": "alice"}, [{"AutoScalingGroupName": "asg-1"}]) + create_tags.assert_called_once_with( + Tags=[{"Key": "CreatorName", "Value": "alice", + "PropagateAtLaunch": False, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-1"}]) diff --git a/tests/test_cw_dashboard.py b/tests/test_cw_dashboard.py new file mode 100644 index 00000000000..03052c7d6f9 --- /dev/null +++ b/tests/test_cw_dashboard.py @@ -0,0 +1,101 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +from unittest import mock + +from botocore.exceptions import ClientError +from pytest_terraform import terraform + +from c7n.resources import cw + +from .common import BaseTest + + +@terraform('cloudwatch_dashboard_tag') +def test_cloudwatch_dashboard_tag(test, cloudwatch_dashboard_tag): + # Dashboards are tagged through the native cloudwatch apis, the + # resource groups tagging api rejects their region-less arns. + aws_region = 'us-east-1' + session_factory = test.replay_flight_data( + 'test_cloudwatch_dashboard_tag', region=aws_region) + + dashboard_name = cloudwatch_dashboard_tag[ + 'aws_cloudwatch_dashboard.test.dashboard_name'] + + p = test.load_policy({ + 'name': 'dashboard-tag', + 'resource': 'aws.cloudwatch-dashboard', + 'filters': [{'DashboardName': dashboard_name}], + 'actions': [ + {'type': 'tag', 'tags': {'App': 'Custodian', 'Env': 'Dev'}}]}, + session_factory=session_factory, config={'region': aws_region}) + resources = p.run() + test.assertEqual(len(resources), 1) + + client = session_factory().client('cloudwatch') + arn = resources[0]['DashboardArn'] + test.assertEqual( + {t['Key']: t['Value'] + for t in client.list_tags_for_resource(ResourceARN=arn)['Tags']}, + {'App': 'Custodian', 'Env': 'Dev'}) + + # filtering on the tag we just set exercises the read path - these + # tags aren't visible through the resource groups tagging api. + p = test.load_policy({ + 'name': 'dashboard-remove-tag', + 'resource': 'aws.cloudwatch-dashboard', + 'filters': [ + {'DashboardName': dashboard_name}, + {'tag:App': 'Custodian'}], + 'actions': [{'type': 'remove-tag', 'tags': ['Env']}]}, + session_factory=session_factory, config={'region': aws_region}) + resources = p.run() + test.assertEqual(len(resources), 1) + + test.assertEqual( + client.list_tags_for_resource(ResourceARN=arn)['Tags'], + [{'Key': 'App', 'Value': 'Custodian'}]) + + +def _client_error(code): + return ClientError({'Error': {'Code': code, 'Message': code}}, 'Operation') + + +class CloudWatchDashboardTest(BaseTest): + + def get_source(self): + p = self.load_policy({ + 'name': 'dashboard-augment', + 'resource': 'aws.cloudwatch-dashboard'}) + return p.resource_manager.source + + def test_augment_skips_deleted_dashboard(self): + source = self.get_source() + client = mock.MagicMock() + client.list_tags_for_resource.side_effect = [ + _client_error('ResourceNotFoundException'), + {'Tags': [{'Key': 'App', 'Value': 'Custodian'}]}, + ] + dashboards = [ + {'DashboardName': 'gone', + 'DashboardArn': 'arn:aws:cloudwatch::644160558196:dashboard/gone'}, + {'DashboardName': 'here', + 'DashboardArn': 'arn:aws:cloudwatch::644160558196:dashboard/here'}, + ] + with mock.patch.object(cw, 'local_session') as ls: + ls.return_value.client.return_value = client + resources = source.augment(dashboards) + self.assertEqual( + [r['DashboardName'] for r in resources], ['here']) + self.assertEqual( + resources[0]['Tags'], [{'Key': 'App', 'Value': 'Custodian'}]) + + def test_augment_error(self): + source = self.get_source() + client = mock.MagicMock() + client.list_tags_for_resource.side_effect = _client_error('AccessDeniedException') + with mock.patch.object(cw, 'local_session') as ls: + ls.return_value.client.return_value = client + with self.assertRaises(ClientError): + source.augment([ + {'DashboardName': 'here', + 'DashboardArn': 'arn:aws:cloudwatch::644160558196:dashboard/here'}]) diff --git a/tests/test_ebs.py b/tests/test_ebs.py index 2b74aef54d2..7a2abb7e0ff 100644 --- a/tests/test_ebs.py +++ b/tests/test_ebs.py @@ -177,16 +177,6 @@ def test_tag_error(self): tagger.process_resource_set(client, snaps, [{'Key': 'bar', 'Value': 'foo'}]) client.create_tags.assert_called_once() - def test_remove_snapshot(self): - snaps = [{'SnapshotId': 'a'}, {'SnapshotId': 'b'}, {'SnapshotId': 'c'}] - - t1 = list(snaps) - ErrorHandler.remove_snapshot('c', t1) - self.assertEqual([t['SnapshotId'] for t in t1], ['a', 'b']) - - ErrorHandler.remove_snapshot('d', snaps) - self.assertEqual(len(snaps), 3) - def test_get_bad_snapshot_malformed(self): operation_name = "DescribeSnapshots" error_response = { @@ -235,16 +225,6 @@ def test_get_bad_volume_notfound(self): vol = ErrorHandler.extract_bad_volume(e) self.assertEqual(vol, "vol-notfound") - def test_remove_volume(self): - vols = [{'VolumeId': 'a'}, {'VolumeId': 'b'}, {'VolumeId': 'c'}] - - t1 = list(vols) - ErrorHandler.remove_volume('c', t1) - self.assertEqual([t['VolumeId'] for t in t1], ['a', 'b']) - - ErrorHandler.remove_volume('d', vols) - self.assertEqual(len(vols), 3) - def test_volume_tag_error(self): vols = [{'VolumeId': 'vol-aa'}] error_response = { diff --git a/tests/test_eks.py b/tests/test_eks.py index 92b6d86564d..1b4d83bb5ff 100644 --- a/tests/test_eks.py +++ b/tests/test_eks.py @@ -183,6 +183,35 @@ def test_tag_and_remove_tag(self): name='devx')['cluster']['tags'], {'App': 'Custodian'}) + def test_tag_deleted_cluster(self): + # eks raises NotFoundException (not ResourceNotFoundException) when + # a cluster is gone by the time we tag it. No terraform fixture + # here - the cluster's absence is the whole point, so we tag an + # arn that isn't there rather than spend 20m creating a cluster + # to delete it. + aws_region = 'us-east-1' + factory = self.replay_flight_data( + 'test_eks_tag_deleted_cluster', region=aws_region) + + p = self.load_policy({ + 'name': 'eks-tag-deleted', + 'resource': 'aws.eks', + 'actions': [ + {'type': 'tag', 'tags': {'App': 'Custodian'}}, + {'type': 'remove-tag', 'tags': ['Env']}]}, + session_factory=factory, config={'region': aws_region}) + + gone = { + 'name': 'c7n-deleted', + 'arn': 'arn:aws:eks:%s:%s:cluster/c7n-deleted' % ( + aws_region, self.account_id), + 'tags': {'Env': 'Dev'}} + tag, remove_tag = p.resource_manager.actions + + # neither raises, the cluster is simply skipped + tag.process([gone]) + remove_tag.process([gone]) + def test_kms_filter(self): factory = self.replay_flight_data('test_eks_kms_filter') kms = factory().client('kms') diff --git a/tests/test_kms.py b/tests/test_kms.py index f033e0660c1..0077eab7cb4 100644 --- a/tests/test_kms.py +++ b/tests/test_kms.py @@ -3,7 +3,7 @@ import json import time from datetime import datetime, timedelta -from unittest.mock import patch +from unittest.mock import patch, MagicMock import boto3 import moto @@ -1002,3 +1002,75 @@ def test_schedule_deletion_default_days(self): deletion_date = key_meta["DeletionDate"].astimezone(tzutc()) assert deletion_date > thirty_days_away assert deletion_date < (thirty_days_away + timedelta(days=1)) + + +class KMSSkipAwsManagedKeyTest(BaseTest): + # https://github.com/cloud-custodian/cloud-custodian/issues/11027 + # AWS managed keys (KeyManager == 'AWS') can't have their policy, + # rotation, or tags modified by any customer account - actions + # should skip them rather than let the API call fail. + + aws_key = { + 'KeyId': 'aws-key-1', + 'Arn': 'arn:aws:kms:us-east-1:111111111111:key/aws-key-1', + 'KeyManager': 'AWS', + } + customer_key = { + 'KeyId': 'cust-key-1', + 'Arn': 'arn:aws:kms:us-east-1:111111111111:key/cust-key-1', + 'KeyManager': 'CUSTOMER', + } + + def _policy(self, action, client_name='kms'): + mock_factory = MagicMock() + mock_factory.region = 'us-east-1' + client = mock_factory().client(client_name) + policy = self.load_policy( + {'name': 'k', 'resource': 'kms-key', 'actions': [action]}, + session_factory=mock_factory) + return policy, client + + def test_schedule_deletion_skips_aws_managed(self): + policy, client = self._policy({'type': 'schedule-deletion', 'days': 7}) + policy.resource_manager.actions[0].process([self.aws_key, self.customer_key]) + client.schedule_key_deletion.assert_called_once_with( + KeyId='cust-key-1', PendingWindowInDays=7) + + def test_schedule_deletion_all_aws_managed_noop(self): + policy, client = self._policy({'type': 'schedule-deletion'}) + policy.resource_manager.actions[0].process([self.aws_key]) + client.schedule_key_deletion.assert_not_called() + + def test_set_rotation_skips_aws_managed(self): + policy, client = self._policy({'type': 'set-rotation', 'state': True}) + policy.resource_manager.actions[0].process([self.aws_key, self.customer_key]) + client.enable_key_rotation.assert_called_once_with(KeyId='cust-key-1') + + def test_remove_statements_skips_aws_managed(self): + policy, client = self._policy({'type': 'remove-statements', 'statement_ids': 'matched'}) + policy.resource_manager.actions[0].process([self.aws_key]) + client.get_key_policy.assert_not_called() + + def _tag_policy(self, action): + mock_factory = MagicMock() + mock_factory.region = 'us-east-1' + tag_resources = mock_factory().client('resourcegroupstaggingapi').tag_resources + tag_resources.return_value = {} + policy = self.load_policy( + {'name': 'k', 'resource': 'kms-key', 'actions': [action]}, + session_factory=mock_factory) + return policy, tag_resources + + def test_tag_skips_aws_managed(self): + policy, tag_resources = self._tag_policy( + {'type': 'tag', 'tags': {'Owner': 'platform'}}) + policy.resource_manager.actions[0].process([self.aws_key, self.customer_key]) + tag_resources.assert_called_once_with( + ResourceARNList=['arn:aws:kms:us-east-1:111111111111:key/cust-key-1'], + Tags={'Owner': 'platform'}) + + def test_tag_all_aws_managed_noop(self): + policy, tag_resources = self._tag_policy( + {'type': 'tag', 'tags': {'Owner': 'platform'}}) + policy.resource_manager.actions[0].process([self.aws_key]) + tag_resources.assert_not_called() diff --git a/tests/test_lookup.py b/tests/test_lookup.py index 6033633b989..c5649d09d15 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -2,6 +2,8 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 +import jsonschema + from .common import BaseTest from c7n.lookup import Lookup @@ -18,12 +20,12 @@ def test_lookup_type(self): self.assertEqual(number_schema, lookup_default_number['oneOf'][1]) self.assertEqual(number_schema, - lookup_default_number['oneOf'][0]['oneOf'][0] + lookup_default_number['oneOf'][0] ['properties']['default-value']) self.assertEqual(string_schema, lookup_default_string['oneOf'][1]) self.assertEqual(string_schema, - lookup_default_string['oneOf'][0]['oneOf'][0] + lookup_default_string['oneOf'][0] ['properties']['default-value']) def test_extract_no_lookup(self): @@ -76,6 +78,15 @@ def test_get_value_from_resource_value_not_exists(self): value = Lookup.get_value_from_resource(source, resource) self.assertEqual(value, 'value_2') + def test_extract_lookup_without_key_returns_default(self): + source = {'type': Lookup.RESOURCE_SOURCE, 'default-value': 'value_2'} + self.assertEqual(Lookup.extract(source, {'name': 'x'}), 'value_2') + + def test_get_value_from_resource_without_key_returns_default(self): + source = {'type': Lookup.RESOURCE_SOURCE, 'default-value': 'value_2'} + self.assertEqual( + Lookup.get_value_from_resource(source, {'name': 'x'}), 'value_2') + def test_get_value_from_resource_value_not_exists_exception(self): resource = { 'field_level_1': { @@ -89,3 +100,114 @@ def test_get_value_from_resource_value_not_exists_exception(self): with self.assertRaises(Exception): Lookup.get_value_from_resource(source, resource) + + +class LookupSchemaTest(BaseTest): + """A lookup needs a key, a fallback, or both.""" + + string_schema = {'type': 'string'} + + def validates(self, doc, schema=None): + try: + jsonschema.validate(doc, schema or Lookup.lookup_type(self.string_schema)) + return True + except jsonschema.ValidationError: + return False + + def test_key_only_accepted(self): + self.assertTrue(self.validates({'type': 'resource', 'key': 'name'})) + + def test_key_with_default_accepted(self): + self.assertTrue( + self.validates( + {'type': 'resource', 'key': 'name', 'default-value': 'x'})) + + def test_default_only_accepted(self): + self.assertTrue(self.validates({'type': 'resource', 'default-value': 'x'})) + + def test_scalar_accepted(self): + self.assertTrue(self.validates('plain')) + + def test_neither_key_nor_default_rejected(self): + self.assertFalse(self.validates({'type': 'resource'})) + + def test_unknown_property_rejected(self): + self.assertFalse(self.validates({'type': 'resource', 'ky': 'name'})) + + def test_non_lookup_dict_rejected(self): + self.assertFalse(self.validates({'foo': 'bar'})) + + def test_default_value_is_typed_by_caller(self): + self.assertFalse( + self.validates({'type': 'resource', 'default-value': 5})) + self.assertTrue( + self.validates({'type': 'resource', 'default-value': 5}, + Lookup.lookup_type({'type': 'number'}))) + + def test_default_value_may_be_typed_separately_from_the_value(self): + schema = Lookup.lookup_type( + {'type': ['string', 'number']}, default_schema={'type': 'string'}) + # the bare scalar keeps the wider type + self.assertTrue(self.validates(5, schema)) + self.assertTrue(self.validates('plain', schema)) + # the fallback is narrowed + self.assertTrue( + self.validates( + {'type': 'resource', 'key': 'k', 'default-value': 'd'}, schema)) + self.assertFalse( + self.validates( + {'type': 'resource', 'key': 'k', 'default-value': 5}, schema)) + + def test_default_schema_defaults_to_the_value_schema(self): + one_arg = Lookup.lookup_type({'type': 'number'}) + explicit = Lookup.lookup_type( + {'type': 'number'}, default_schema={'type': 'number'}) + self.assertEqual(one_arg, explicit) + + +class LookupResolveValueTest(BaseTest): + + def test_static_value_passthrough(self): + self.assertEqual(Lookup.resolve_value('plain', {}), 'plain') + + def test_key_hit(self): + resource = {'State': {'Name': 'running'}} + spec = {'type': 'resource', 'key': 'State.Name'} + self.assertEqual(Lookup.resolve_value(spec, resource), 'running') + + def test_key_miss_uses_default(self): + spec = {'type': 'resource', 'key': 'nope', 'default-value': 'dv'} + self.assertEqual(Lookup.resolve_value(spec, {}), 'dv') + + def test_key_miss_without_default_raises(self): + spec = {'type': 'resource', 'key': 'nope'} + with self.assertRaises(Exception) as cm: + Lookup.resolve_value(spec, {}) + self.assertIn('nope', str(cm.exception)) + + def test_conditional_default_writes_when_name_absent(self): + spec = {'type': 'resource', 'default-value': 'dv'} + self.assertEqual( + Lookup.resolve_value(spec, {}, 'Owner', set()), 'dv') + + def test_conditional_default_skips_when_name_present(self): + spec = {'type': 'resource', 'default-value': 'dv'} + self.assertIs( + Lookup.resolve_value(spec, {}, 'Owner', {'Owner'}), Lookup.SKIP) + + def test_skip_sentinel_is_distinct_from_none_and_empty_string(self): + self.assertIsNot(Lookup.SKIP, None) + self.assertNotEqual(Lookup.SKIP, '') + + +class LookupHasLookupsTest(BaseTest): + + def test_all_static(self): + self.assertFalse(Lookup.has_lookups({'A': 'x', 'B': 'y'})) + + def test_empty(self): + self.assertFalse(Lookup.has_lookups({})) + + def test_one_dynamic(self): + self.assertTrue( + Lookup.has_lookups({'A': 'x', 'B': {'type': 'resource', 'key': 'k'}})) diff --git a/tests/test_networkmanager.py b/tests/test_networkmanager.py index 0c0072eac31..bd0b02b0c9f 100644 --- a/tests/test_networkmanager.py +++ b/tests/test_networkmanager.py @@ -203,7 +203,7 @@ def test_mark_for_op_global_network(self): def test_core_network_delete_error(self): invalid_network_id = 'core-network-7a6b617270696e736b69' - factory = self.record_flight_data("test_core_network_delete_error") + factory = self.replay_flight_data("test_core_network_delete_error") client = factory().client("networkmanager") mock_factory = MagicMock() mock_factory.region = 'us-east-1' diff --git a/tests/test_route53.py b/tests/test_route53.py index 66c65acbeda..2771aa0e120 100644 --- a/tests/test_route53.py +++ b/tests/test_route53.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import time import logging +from unittest.mock import MagicMock import pytest from pytest_terraform import terraform @@ -698,3 +699,37 @@ def test_resolver_rule_query(self): resources = p.run() self.assertEqual(len(resources), 1) self.assertEqual(resources[0]["RuleType"], "FORWARD") + + # https://github.com/cloud-custodian/cloud-custodian/issues/11027 + # AWS's global auto-defined resolver rules (e.g. + # rslvr-autodefined-rr-internet-resolver) have no owning account and + # can't be tagged/modified by any customer - actions should skip them. + + auto_defined_rule = {'Id': 'rslvr-autodefined-rr-internet-resolver'} + customer_rule = {'Id': 'rslvr-rr-real1'} + + def _tag_policy(self, action): + mock_factory = MagicMock() + mock_factory.region = 'us-east-1' + tag_resources = mock_factory().client('resourcegroupstaggingapi').tag_resources + tag_resources.return_value = {} + policy = self.load_policy( + {'name': 'r', 'resource': 'resolver-rule', 'actions': [action]}, + session_factory=mock_factory) + return policy, tag_resources + + def test_tag_skips_auto_defined_rule(self): + policy, tag_resources = self._tag_policy( + {'type': 'tag', 'tags': {'Owner': 'platform'}}) + policy.resource_manager.actions[0].process( + [self.auto_defined_rule, self.customer_rule]) + tag_resources.assert_called_once() + arns = tag_resources.call_args.kwargs['ResourceARNList'] + self.assertEqual(len(arns), 1) + self.assertIn('rslvr-rr-real1', arns[0]) + + def test_tag_all_auto_defined_noop(self): + policy, tag_resources = self._tag_policy( + {'type': 'tag', 'tags': {'Owner': 'platform'}}) + policy.resource_manager.actions[0].process([self.auto_defined_rule]) + tag_resources.assert_not_called() diff --git a/tests/test_s3.py b/tests/test_s3.py index b54b668c79b..eef077c2e1e 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -14,7 +14,9 @@ from unittest import TestCase from contextlib import suppress -from botocore.exceptions import ClientError, ConnectTimeoutError, EndpointConnectionError +from botocore.exceptions import ( + ClientError, ConnectionClosedError, ConnectTimeoutError, EndpointConnectionError, + ReadTimeoutError) from dateutil.tz import tzutc import pytest from pytest_terraform import terraform @@ -36,6 +38,12 @@ ) +# every transport failure botocore raises that we mean to survive +TRANSPORT_ERRORS = pytest.mark.parametrize('error_class', [ + ConnectTimeoutError, ReadTimeoutError, EndpointConnectionError, + ConnectionClosedError]) + + def test_s3_assembly_validate(test): for tcase in ('all', 'none', 'detect', ['Website', 'Replication']): @@ -166,6 +174,125 @@ def test_s3_assembly_endpoint_connection_error(test): assert 'Location' not in bucket +def test_s3_assembly_connection_closed(test): + # a connection closed mid response is as unreachable as a connect + # timeout - a degraded endpoint can accept and then drop + policy = test.load_policy({'name': 's3-attrs', 'resource': 's3'}) + assembly = s3.BucketAssembly(policy.resource_manager) + assembly.initialize() + + client = mock.MagicMock() + client.meta.region_name = assembly.default_region + client.get_bucket_location.side_effect = ConnectionClosedError(endpoint_url='x') + assembly.region_clients[assembly.default_region] = client + + bucket = assembly.assemble({'Name': 'hung-bucket'}) + assert bucket['Name'] == 'hung-bucket' + assert 'Location' not in bucket + + +@TRANSPORT_ERRORS +def test_s3_modify_tags_continues_past_unreachable_bucket(test, error_class): + # one hung bucket shouldn't cost us the rest of the batch + client = mock.MagicMock() + client.get_bucket_tagging.side_effect = [ + error_class(endpoint_url='x'), + {'TagSet': [{'Key': 'Existing', 'Value': 'keep'}]}] + test.patch(s3, 'bucket_client', lambda session, b, kms=False: client) + + s3.modify_bucket_tags( + lambda: mock.MagicMock(), + [{'Name': 'hung'}, {'Name': 'healthy'}], + add_tags=[{'Key': 'Owner', 'Value': 'devs'}]) + + # the healthy bucket is still tagged, and keeps the tags it had + client.put_bucket_tagging.assert_called_once_with( + Bucket='healthy', + Tagging={'TagSet': [{'Key': 'Owner', 'Value': 'devs'}, + {'Key': 'Existing', 'Value': 'keep'}]}) + + +@TRANSPORT_ERRORS +def test_s3_modify_tags_continues_past_failed_write(test, error_class): + # a bucket that answers the read and then hangs on the write is skipped + # too, and the rest of the batch still gets tagged + client = mock.MagicMock() + client.get_bucket_tagging.return_value = {'TagSet': []} + client.put_bucket_tagging.side_effect = [ + error_class(endpoint_url='x'), None] + test.patch(s3, 'bucket_client', lambda session, b, kms=False: client) + + s3.modify_bucket_tags( + lambda: mock.MagicMock(), + [{'Name': 'hung'}, {'Name': 'healthy'}], + add_tags=[{'Key': 'Owner', 'Value': 'devs'}]) + + assert client.put_bucket_tagging.call_count == 2 + + +def test_s3_hung_bucket_doesnt_cost_the_rest_of_its_batch(test): + """one bucket whose endpoint hangs, end to end, over a batch boundary. + + the tag action chunks at 25 and walks the buckets within a chunk + serially, so a hung bucket that raises takes every bucket after it in + its own chunk down with it, and fails the run. + """ + hung, healthy = 'hung-bucket', ['healthy-%02d' % i for i in range(29)] + buckets = [{'Name': hung}] + [{'Name': n} for n in healthy] + + def make_client(region): + client = mock.MagicMock() + client.meta.region_name = region + if region == 'elsewhere-1': + for method_name, _, _, _, _ in s3.S3_AUGMENT_TABLE: + getattr(client, method_name).side_effect = ConnectTimeoutError( + endpoint_url='https://s3.elsewhere-1.amazonaws.com') + return client + client.get_bucket_location.side_effect = lambda Bucket: { + 'LocationConstraint': 'elsewhere-1' if Bucket == hung else None} + client.get_bucket_tagging.return_value = {'TagSet': []} + return client + + clients = {} + + def session(*args, **kw): + s = mock.MagicMock() + + def client(svc, region_name=None, **kw): + region = region_name or 'us-east-1' + if region not in clients: + clients[region] = make_client(region) + return clients[region] + + s.client.side_effect = client + return s + + policy = test.load_policy({ + 'name': 's3-tag', 'resource': 's3', + 'actions': [{'type': 'tag', 'key': 'Owner', 'value': 'devs'}]}) + test.patch(s3, 'local_session', session) + + resources = s3.DescribeS3(policy.resource_manager).augment(buckets) + assert len(resources) == len(buckets) + + # the endpoint is just as unreachable for the action as for collection + written = {} + + def action_client(sess, b, kms=False): + if b['Name'] not in written: + written[b['Name']] = make_client( + 'elsewhere-1' if b['Name'] == hung else 'us-east-1') + return written[b['Name']] + + test.patch(s3, 'bucket_client', action_client) + policy.resource_manager.actions[0].process(resources) + + tagged = {n for n, c in written.items() if c.put_bucket_tagging.called} + assert hung not in tagged + assert tagged == set(healthy), "the hung bucket cost %d of its batch" % ( + len(set(healthy) - tagged)) + + def test_s3_express(test): session_factory = test.replay_flight_data('test_s3_express') p = test.load_policy( diff --git a/tests/test_tags.py b/tests/test_tags.py index 5aa746c3b28..29cbc536ab3 100644 --- a/tests/test_tags.py +++ b/tests/test_tags.py @@ -4,13 +4,17 @@ module to test some universal tagging infrastructure not directly exposed. """ import time +import jsonschema +from datetime import datetime from freezegun import freeze_time -from unittest.mock import MagicMock, call +from unittest.mock import MagicMock, call, patch + +from botocore.exceptions import ClientError from c7n import tags as tagmod from c7n.tags import universal_retry, coalesce_copy_user_tags from c7n.exceptions import PolicyExecutionError, PolicyValidationError -from c7n.utils import yaml_load +from c7n.utils import FormatDate, yaml_load from .common import BaseTest @@ -606,6 +610,40 @@ def test_copy_related_tag_validate_aws_prefix_fake_resource(self): self.assertRaises(PolicyValidationError, self.load_policy, policy) +class TagValueSchemaTest(BaseTest): + """Tag values are strings, but unquoted YAML scalars have always been + accepted and coerced. The fallback is a newer field and stays strict. + """ + + def validates(self, value): + try: + jsonschema.validate( + {'X': value}, + {'type': 'object', + 'additionalProperties': tagmod.tag_value_schema()}) + return True + except jsonschema.ValidationError: + return False + + def test_scalar_accepts_number_and_boolean(self): + self.assertTrue(self.validates('plain')) + self.assertTrue(self.validates(12345)) + self.assertTrue(self.validates(True)) + + def test_default_value_must_be_a_string(self): + self.assertTrue( + self.validates({'type': 'resource', 'key': 'K', 'default-value': 'd'})) + self.assertFalse( + self.validates({'type': 'resource', 'key': 'K', 'default-value': 5})) + self.assertFalse( + self.validates({'type': 'resource', 'default-value': True})) + + def test_non_scalar_values_rejected(self): + self.assertFalse(self.validates(['a'])) + self.assertFalse(self.validates(None)) + self.assertFalse(self.validates({'foo': 'bar'})) + + class ResolveTagValueTest(BaseTest): def test_static_passthrough(self): self.assertEqual( @@ -715,6 +753,22 @@ def test_placeholder_interpolated_in_default_value(self): Tags=[{"Key": "Stamp", "Value": "created-2022-06-27 12:34:56-in-us-east-1"}], DryRun=False) + def test_now_resolved_once_for_the_whole_run(self): + # a clock that moves on every read - the run should only read it once, + # so every resource lands in one group with one timestamp + ticks = [FormatDate(datetime(2022, 6, 27, 12, 34, 56)), + FormatDate(datetime(2022, 6, 27, 12, 34, 57))] + with patch.object(FormatDate, "utcnow", side_effect=ticks) as utcnow: + create_tags = self._run( + [{"InstanceId": "i-1"}, {"InstanceId": "i-2"}], + {"Stamp": {"type": "resource", "key": "Nope", + "default-value": "at-{now}"}}) + self.assertEqual(utcnow.call_count, 1) + create_tags.assert_called_once_with( + Resources=["i-1", "i-2"], + Tags=[{"Key": "Stamp", "Value": "at-2022-06-27 12:34:56"}], + DryRun=False) + class DynamicUniversalTagTest(BaseTest): def _run(self, resources, tags, resource='rds'): @@ -743,3 +797,183 @@ def test_universal_static_fast_path(self): {"K": "v"}) tag_resources.assert_called_once_with( ResourceARNList=["arn:x"], Tags={"K": "v"}) + + +# a syntactically valid instance id that doesn't exist, so ec2 answers the +# tag call with its real "resource gone" error rather than a malformed id one +GONE_INSTANCE_ID = 'i-0023456789abcdef0' + + +@terraform('ec2_instance_tag') +def test_native_tag_skips_gone_instance(test, ec2_instance_tag): + """A terminated instance in the batch shouldn't cost the rest their tags. + + ec2 fails the whole create_tags call when any resource in it is gone, + so we drop the ids it named and tag the rest. aws.ec2-instance-ami is + the resource from #10999 - it tags via instance id. + """ + aws_region = 'us-east-1' + session_factory = test.replay_flight_data( + 'test_native_tag_skips_gone_instance', region=aws_region) + + instance_ids = [ + ec2_instance_tag['aws_instance.first.id'], + ec2_instance_tag['aws_instance.second.id'], + ] + + p = test.load_policy({ + 'name': 'ec2-instance-ami-tag', + 'resource': 'aws.ec2-instance-ami', + 'actions': [ + {'type': 'tag', 'tags': {'App': 'Custodian'}}, + {'type': 'remove-tag', 'tags': ['App']}]}, + session_factory=session_factory, config={'region': aws_region}) + + resources = [r for r in p.resource_manager.resources() + if r['InstanceId'] in instance_ids] + test.assertEqual(len(resources), 2) + + # the batch ec2 rejects, plus the ones it should still tag + resource_set = resources + [{'InstanceId': GONE_INSTANCE_ID}] + tag, remove_tag = p.resource_manager.actions + + tag.process(resource_set) + client = session_factory().client('ec2') + tags = { + i['InstanceId']: {t['Key']: t['Value'] for t in i.get('Tags', [])} + for r in client.describe_instances( + InstanceIds=instance_ids)['Reservations'] + for i in r['Instances']} + for instance_id in instance_ids: + test.assertEqual(tags[instance_id].get('App'), 'Custodian') + + # and the same salvage on the way back out + remove_tag.id_key = 'InstanceId' + remove_tag.process(resource_set) + tags = { + i['InstanceId']: {t['Key']: t['Value'] for t in i.get('Tags', [])} + for r in client.describe_instances( + InstanceIds=instance_ids)['Reservations'] + for i in r['Instances']} + for instance_id in instance_ids: + test.assertNotIn('App', tags[instance_id]) + + +def test_universal_tag_skips_gone_snapshot(test): + """The rgta path passes through the service's own resource gone fault. + + elasticache reports a deleted snapshot as SnapshotNotFoundFault, which + universal_retry used to raise on - it only skipped + ResourceNotFoundException. No fixture, a snapshot that isn't there is + the point. + """ + aws_region = 'us-east-1' + session_factory = test.replay_flight_data( + 'test_universal_tag_skips_gone_snapshot', region=aws_region) + + p = test.load_policy({ + 'name': 'cache-snapshot-tag', + 'resource': 'aws.cache-snapshot', + 'actions': [{'type': 'tag', 'tags': {'App': 'Custodian'}}]}, + session_factory=session_factory, + config={'region': aws_region, 'account_id': test.account_id}) + + # doesn't raise, the snapshot is skipped + p.resource_manager.actions[0].process([{'SnapshotName': 'c7n-deleted'}]) + + +class ResourceGoneTest(BaseTest): + """Tag actions race with resource deletion, a resource which is gone by + the time we tag it should be skipped, not abort the whole resource set. + + The recorded tests above cover what aws actually answers with. These + cover our own control flow around it, which needs no api. + """ + + def client_error(self, code, message, op='CreateTags'): + return ClientError( + {'Error': {'Code': code, 'Message': message}}, op) + + def get_tag_action(self, name): + p = self.load_policy({ + 'name': 'tag-instances', + 'resource': 'aws.ec2', + 'actions': [ + {'type': 'tag', 'tags': {'Env': 'Dev'}}, + {'type': 'remove-tag', 'tags': ['Env']}]}) + action = {a.type: a for a in p.resource_manager.actions}[name] + action.id_key = 'InstanceId' + return action + + def test_extract_bad_resource_ids(self): + # message shapes here are the ones in our recorded api data, see + # tests/data/placebo - ec2 words them per resource type + for code, message, expected in ( + ('InvalidNetworkInterfaceID.NotFound', + "The networkInterface ID 'eni-d834cdcf' does not exist", + ('eni-d834cdcf',)), + ('InvalidVolume.NotFound', + "The volume 'vol-notfound' does not exist.", + ('vol-notfound',)), + ('InvalidSnapshotID.Malformed', + 'Invalid id: "snap-malformedsnap"', + ('snap-malformedsnap',)), + # not an ec2 bad id error, nothing to skip + ('UnauthorizedOperation', "You are not authorized", ()), + ('SnapshotNotFoundFault', "'snap-1' not found", ())): + self.assertEqual( + tagmod.extract_bad_resource_ids( + self.client_error(code, message)), + expected) + + def test_native_tag_raises_other_errors(self): + action = self.get_tag_action('tag') + client = MagicMock() + client.create_tags.side_effect = self.client_error( + 'UnauthorizedOperation', "You are not authorized") + self.assertRaises( + ClientError, + action.process_resource_set, + client, [{'InstanceId': 'i-1'}], [{'Key': 'Env', 'Value': 'Dev'}]) + + def test_native_tag_raises_on_unknown_bad_id(self): + # we only skip the resources ec2 actually named, an id we don't + # recognize means we don't understand the error - don't swallow it + action = self.get_tag_action('tag') + client = MagicMock() + client.create_tags.side_effect = self.client_error( + 'InvalidInstanceID.NotFound', + "The instance IDs 'i-9' do not exist") + self.assertRaises( + ClientError, + action.process_resource_set, + client, [{'InstanceId': 'i-1'}], [{'Key': 'Env', 'Value': 'Dev'}]) + + def test_universal_retry_skips_gone_and_retries_throttle(self): + # a throttle alongside a skip still retries the throttled arn + self.patch(time, "sleep", MagicMock()) + method = MagicMock() + method.side_effect = [ + {"FailedResourcesMap": { + "arn:abc": {"ErrorCode": "ThrottlingException"}, + "arn:def": {"ErrorCode": "NotFoundException"}}}, + {"Result": 32}, + ] + self.assertEqual( + universal_retry(method, ["arn:abc", "arn:def"]), {"Result": 32}) + self.assertEqual( + method.call_args_list, + [call(ResourceARNList=["arn:abc", "arn:def"]), + call(ResourceARNList=["arn:abc"])]) + + def test_universal_retry_all_skipped_stops(self): + # nothing left to retry, and the api rejects an empty arn list + sleep = MagicMock() + self.patch(time, "sleep", sleep) + method = MagicMock() + method.side_effect = [{ + "FailedResourcesMap": { + "arn:abc": {"ErrorCode": "SnapshotNotFoundFault"}}}] + universal_retry(method, ["arn:abc"]) + method.assert_called_once() + sleep.assert_not_called() diff --git a/tests/zpill.py b/tests/zpill.py index 8b526643b1d..267dd230f57 100644 --- a/tests/zpill.py +++ b/tests/zpill.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import base64 import fnmatch +import functools from io import BytesIO import json import os @@ -17,6 +18,7 @@ from placebo import pill from c7n.testing import CustodianTestCore +from c7n.utils import get_account_id_from_sts # Custodian Test Account. This is used only for testing. @@ -267,6 +269,11 @@ def save_response(self, service, operation, response_data, http_response=200): super(RedPill, self).save_response(service, operation, response_data, http_response) +@functools.lru_cache(maxsize=None) +def get_recording_account_id(): + return get_account_id_from_sts(boto3.Session()) + + class PillTest(CustodianTestCore): archive_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "placebo_data.zip") @@ -276,6 +283,19 @@ class PillTest(CustodianTestCore): recording = False + @property + def account_id(self): + """The account id to construct arns with. + + Recorded api data is sanitized to ACCOUNT_ID, but while recording + (or running functionally) we're talking to a real account and need + its real id, which is what the aws provider's initialization would + have resolved for us. + """ + if not self.recording: + return ACCOUNT_ID + return get_recording_account_id() + def cleanUp(self): self.pill = None diff --git a/tools/c7n_awscc/pyproject.toml b/tools/c7n_awscc/pyproject.toml index 65f3ae02a7a..551d6777c8d 100644 --- a/tools/c7n_awscc/pyproject.toml +++ b/tools/c7n_awscc/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "jsonpatch<2.0,>=1.32", ] name = "c7n_awscc" -version = "0.1.37a5.post1+a83dda5c9" +version = "0.1.37a5.post2+a83dda5c9" readme = "readme.md" description = "Cloud Custodian - AWS Cloud Control Provider" classifiers = [ diff --git a/tools/c7n_azure/c7n_azure/actions/tagging.py b/tools/c7n_azure/c7n_azure/actions/tagging.py index 05370fde165..32397896760 100644 --- a/tools/c7n_azure/c7n_azure/actions/tagging.py +++ b/tools/c7n_azure/c7n_azure/actions/tagging.py @@ -40,18 +40,33 @@ class Tag(AzureBaseAction): tag: Environment value: Test - - name: azure-tag-from-resource-attr + :example: + + Tag values may also be resolved per resource. A value of the form + ``{type: resource, key: }`` is looked up against the resource, + falling back to ``default-value`` when the path yields nothing. Omitting + ``key`` gives a conditional default: the tag is written only when it is + not already present, leaving existing values untouched. + + .. code-block:: yaml + + policies: + - name: azure-vm-set-default-tags resource: azure.vm - description: | - Tag all existing virtual machines with values taken from the resource + filters: + - or: + - "tag:Env": empty + - "tag:Owner": empty actions: - type: tag tags: - Environment: Test - ResourceName: - type: resource - key: name - default-value: name_not_found + Env: + type: resource + key: tags.Environment # seed from an existing tag if present + default-value: unknown # otherwise fall back + Owner: + type: resource + default-value: platform@example.com # only set if Owner absent """ schema = utils.type_schema( @@ -61,7 +76,9 @@ class Tag(AzureBaseAction): 'tag': Lookup.lookup_type({'type': 'string'}), 'tags': { 'type': 'object', - 'additionalProperties': Lookup.lookup_type({'type': 'string'}) + 'additionalProperties': Lookup.lookup_type( + {'type': ['string', 'number', 'boolean']}, + default_schema={'type': 'string'}), } } ) @@ -88,10 +105,18 @@ def _process_resource(self, resource): def _get_tags(self, resource): tags = self.data.get('tags') - if tags: - return {k: Lookup.extract(v, resource) for k, v in tags.items()} - return {Lookup.extract( - self.data.get('tag'), resource): Lookup.extract(self.data.get('value'), resource)} + if not tags: + return {Lookup.extract(self.data.get('tag'), resource): + Lookup.extract(self.data.get('value'), resource)} + + current = set(resource.get('tags') or ()) + resolved = {} + for name, spec in tags.items(): + value = Lookup.resolve_value(spec, resource, name, current) + if value is Lookup.SKIP: + continue + resolved[name] = value + return resolved class RemoveTag(AzureBaseAction): diff --git a/tools/c7n_azure/pyproject.toml b/tools/c7n_azure/pyproject.toml index b088403cf44..fe422ebe95d 100644 --- a/tools/c7n_azure/pyproject.toml +++ b/tools/c7n_azure/pyproject.toml @@ -87,7 +87,7 @@ dependencies = [ "requests<3.0.0,>=2.22.0", ] name = "c7n_azure" -version = "0.7.51a5.post1+a83dda5c9" +version = "0.7.51a5.post2+a83dda5c9" description = "Cloud Custodian - Azure Support" readme = "readme.md" classifiers = [ diff --git a/tools/c7n_azure/tests_azure/test_actions_tag.py b/tools/c7n_azure/tests_azure/test_actions_tag.py index 8ddbd51de49..d38893ed42d 100644 --- a/tools/c7n_azure/tests_azure/test_actions_tag.py +++ b/tools/c7n_azure/tests_azure/test_actions_tag.py @@ -195,58 +195,162 @@ def test_add_or_update_tags(self, update_resource_tags): self.assertEqual(tags, expected_tags) + +class ActionsTagMappingLookupTest(BaseTest): + """Resource lookups used as values inside the ``tags`` mapping.""" + + existing_tags = {'pre-existing-1': 'unmodified', 'pre-existing-2': 'unmodified'} + + def _get_action(self, data): + # build through a real policy so the action is registry-registered and + # the mapping is run through schema validation on the way in + policy = self.load_policy( + tools.get_policy([dict(data, type='tag')]), validate=True) + return policy.resource_manager.actions[0] + @patch('c7n_azure.tags.TagHelper.update_resource_tags') - def test_add_or_update_tags_from_resource(self, update_resource_tags): - """Verifies values in the tags dictionary are resolved from the resource - rather than passed through as the raw lookup directive - """ + def test_mapping_lookup_by_key_resolves(self, update_resource_tags): + action = self._get_action({ + 'tags': {'from_name': {'type': 'resource', 'key': 'name'}}}) + resource = tools.get_resource(self.existing_tags) - action = self._get_action( - { - 'tags': { - 'tag1': 'value1', - 'tag2': { - 'type': 'resource', - 'key': 'name' - } - } - }) + action.process([resource]) + + expected_tags = self.existing_tags.copy() + expected_tags.update({'from_name': resource['name']}) + self.assertEqual(tools.get_tags_parameter(update_resource_tags), expected_tags) + @patch('c7n_azure.tags.TagHelper.update_resource_tags') + def test_mapping_lookup_miss_uses_default_value(self, update_resource_tags): + action = self._get_action({ + 'tags': {'env': {'type': 'resource', + 'key': 'doesnotexist', + 'default-value': 'production'}}}) resource = tools.get_resource(self.existing_tags) action.process([resource]) - tags = tools.get_tags_parameter(update_resource_tags) + expected_tags = self.existing_tags.copy() + expected_tags.update({'env': 'production'}) + self.assertEqual(tools.get_tags_parameter(update_resource_tags), expected_tags) + + @patch('c7n_azure.tags.TagHelper.update_resource_tags') + def test_mapping_lookup_preserves_existing_tag_value(self, update_resource_tags): + """key pointing at an existing tag keeps that tag's current value. + + A second, changing tag forces the write so the preserved value is + visible in the payload. + """ + action = self._get_action({ + 'tags': {'pre-existing-1': {'type': 'resource', + 'key': 'tags."pre-existing-1"', + 'default-value': 'fallback'}, + 'new': 'added'}}) + resource = tools.get_resource(self.existing_tags) + + action.process([resource]) expected_tags = self.existing_tags.copy() - expected_tags.update({'tag1': 'value1', 'tag2': resource['name']}) + expected_tags.update({'new': 'added'}) + self.assertEqual( + tools.get_tags_parameter(update_resource_tags), expected_tags) - self.assertEqual(tags, expected_tags) + @patch('c7n_azure.tags.TagHelper.update_resource_tags') + def test_no_write_when_lookup_resolves_to_current_value(self, update_resource_tags): + """Resolving a tag to the value it already has is a no-op.""" + action = self._get_action({ + 'tags': {'pre-existing-1': {'type': 'resource', + 'key': 'tags."pre-existing-1"', + 'default-value': 'fallback'}}}) + resource = tools.get_resource(self.existing_tags) + + action.process([resource]) + + update_resource_tags.assert_not_called() @patch('c7n_azure.tags.TagHelper.update_resource_tags') - def test_add_or_update_tags_from_resource_default(self, update_resource_tags): - """Verifies values in the tags dictionary fall back to default-value - when the looked up key does not exist on the resource - """ + def test_mapping_mixes_static_and_lookup_values(self, update_resource_tags): + action = self._get_action({ + 'tags': {'static': 'plain', + 'dynamic': {'type': 'resource', 'key': 'name'}}}) + resource = tools.get_resource(self.existing_tags) - action = self._get_action( - { - 'tags': { - 'tag1': { - 'type': 'resource', - 'key': 'doesnotexist', - 'default-value': 'default_value' - } - } - }) + action.process([resource]) + + expected_tags = self.existing_tags.copy() + expected_tags.update({'static': 'plain', 'dynamic': resource['name']}) + self.assertEqual(tools.get_tags_parameter(update_resource_tags), expected_tags) + @patch('c7n_azure.tags.TagHelper.update_resource_tags') + def test_conditional_default_writes_when_tag_absent(self, update_resource_tags): + action = self._get_action({ + 'tags': {'owner': {'type': 'resource', 'default-value': 'platform'}}}) resource = tools.get_resource(self.existing_tags) action.process([resource]) - tags = tools.get_tags_parameter(update_resource_tags) + expected_tags = self.existing_tags.copy() + expected_tags.update({'owner': 'platform'}) + self.assertEqual(tools.get_tags_parameter(update_resource_tags), expected_tags) + + @patch('c7n_azure.tags.TagHelper.update_resource_tags') + def test_conditional_default_skipped_when_tag_present(self, update_resource_tags): + action = self._get_action({ + 'tags': {'pre-existing-1': {'type': 'resource', + 'default-value': 'should-not-be-written'}}}) + resource = tools.get_resource(self.existing_tags) + + action.process([resource]) + + # nothing left to write, so no update call is made at all + update_resource_tags.assert_not_called() + + @patch('c7n_azure.tags.TagHelper.update_resource_tags') + def test_conditional_default_partial_skip(self, update_resource_tags): + action = self._get_action({ + 'tags': {'pre-existing-1': {'type': 'resource', 'default-value': 'skipped'}, + 'owner': {'type': 'resource', 'default-value': 'platform'}}}) + resource = tools.get_resource(self.existing_tags) + + action.process([resource]) expected_tags = self.existing_tags.copy() - expected_tags.update({'tag1': 'default_value'}) + expected_tags.update({'owner': 'platform'}) + self.assertEqual(tools.get_tags_parameter(update_resource_tags), expected_tags) - self.assertEqual(tags, expected_tags) + @patch('c7n_azure.tags.TagHelper.update_resource_tags') + def test_lookup_resolved_per_resource(self, update_resource_tags): + """Two resources with different values each get their own tag value.""" + action = self._get_action({ + 'tags': {'from_name': {'type': 'resource', 'key': 'name'}}}) + first = tools.get_resource({}) + second = tools.get_resource({}) + second['name'] = 'other-vm' + + action.process([first, second]) + + written = [c[0][2]['from_name'] for c in update_resource_tags.call_args_list] + self.assertEqual(written, ['cctestvm', 'other-vm']) + + def test_schema_accepts_lookup_forms_in_mapping(self): + self.assertTrue(self.load_policy( + tools.get_policy([ + {'type': 'tag', + 'tags': { + 'by_key': {'type': 'resource', 'key': 'name'}, + 'by_key_default': {'type': 'resource', + 'key': 'name', + 'default-value': 'dv'}, + 'conditional': {'type': 'resource', 'default-value': 'dv'}, + 'static': 'plain'}} + ]), validate=True)) + + def test_schema_rejects_malformed_mapping_values(self): + for bad in ({'type': 'resource'}, + {'type': 'resource', 'ky': 'name'}, + {'foo': 'bar'}, + ['a']): + with self.assertRaises(Exception): + self.load_policy( + tools.get_policy([{'type': 'tag', 'tags': {'X': bad}}]), + validate=True) diff --git a/tools/c7n_gcp/c7n_gcp/actions/labels.py b/tools/c7n_gcp/c7n_gcp/actions/labels.py index 28797476661..07b4477eef2 100644 --- a/tools/c7n_gcp/c7n_gcp/actions/labels.py +++ b/tools/c7n_gcp/c7n_gcp/actions/labels.py @@ -115,6 +115,18 @@ class SetLabelsAction(BaseLabelAction): key: name default-value: name_not_found + - name: gcp-add-default-label + resource: gcp.instance + description: | + Omitting key gives a conditional default - the label is set only + when it is absent, leaving an existing value untouched + actions: + - type: set-labels + labels: + owner: + type: resource + default-value: platform + - name: gcp-remove-label resource: gcp.instance description: | @@ -135,7 +147,14 @@ def validate(self): raise FilterValidationError("Must specify one of labels or remove") def get_labels_to_add(self, resource): - return {k: Lookup.extract(v, resource) for k, v in self.data.get('labels', {}).items()} + current = set(self._get_current_labels(resource)) + labels = {} + for name, spec in self.data.get('labels', {}).items(): + value = Lookup.resolve_value(spec, resource, name, current) + if value is Lookup.SKIP: + continue + labels[name] = value + return labels def get_labels_to_delete(self, resource): return self.data.get('remove') diff --git a/tools/c7n_gcp/c7n_gcp/resources/cloudrun.py b/tools/c7n_gcp/c7n_gcp/resources/cloudrun.py index 54f479b60d0..7941f7b746e 100644 --- a/tools/c7n_gcp/c7n_gcp/resources/cloudrun.py +++ b/tools/c7n_gcp/c7n_gcp/resources/cloudrun.py @@ -1,13 +1,64 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 -import copy - from c7n_gcp.provider import resources from c7n_gcp.query import QueryResourceManager, TypeInfo from c7n_gcp.filters import IamPolicyFilter from c7n_gcp.filters.iampolicy import IamPolicyValueFilter +from c7n.exceptions import PolicyExecutionError from c7n.utils import local_session, jmespath_search +# The only fields the Knative schema defines at the root of a Service or a Job. +KNATIVE_FIELDS = ('apiVersion', 'kind', 'metadata', 'spec', 'status') + +# Prefixes of the annotations c7n keeps on a resource. Only the colon form is +# written to a nested object; the dotted `c7n.metrics` is set at the root alone. +# Nesting matches on the colon form only, because kubernetes annotation keys - +# user controlled, and carried in `metadata.annotations` - may contain a dot but +# never a colon, and this is a whole object replace: a key dropped from the body +# is a key deleted from the service. +C7N_ANNOTATION = 'c7n:' +C7N_ROOT_ANNOTATIONS = (C7N_ANNOTATION, 'c7n.') + + +def _without_annotations(value): + "Copy a value, dropping any c7n annotation held anywhere beneath it." + if isinstance(value, dict): + return {k: _without_annotations(v) for k, v in value.items() + if not k.startswith(C7N_ANNOTATION)} + if isinstance(value, list): + return [_without_annotations(v) for v in value] + # everything else in a decoded api response is an immutable scalar + return value + + +def knative_body(resource): + """Copy a resource into a replace request body, dropping non schema fields. + + Cloud Run replaces the whole object, so the body is the resource itself + rather than a fragment, and the api rejects the request with a 400 on any + field it can't place. A resource carries two kinds it can't: the top level + `labels` key that augment() flattens out of the metadata, and the + annotations the policy's filters leave behind - at the root, and nested + within `spec` when a list-item filter matched on something there. + + A root field that is neither raises rather than being dropped. Under a + whole object replace a field left out of the body is deleted from the + service, so a field this doesn't recognize is one it cannot safely send + or safely omit. + """ + unrecognized = { + k for k in resource + if k not in KNATIVE_FIELDS and k != 'labels' + and not k.startswith(C7N_ROOT_ANNOTATIONS) + } + if unrecognized: + raise PolicyExecutionError( + '%s %s has fields absent from the cloud run schema: %s' % ( + resource.get('kind', 'resource'), + resource.get('metadata', {}).get('name'), + ', '.join(sorted(unrecognized)))) + return {k: _without_annotations(v) for k, v in resource.items() if k in KNATIVE_FIELDS} + @resources.register("cloud-run-service") class CloudRunService(QueryResourceManager): @@ -35,7 +86,7 @@ def get_label_params(resource, all_labels): location = metadata['labels']['cloud.googleapis.com/location'] namespace = metadata['namespace'] svc_name = metadata['name'] - body = copy.deepcopy(resource) + body = knative_body(resource) body['metadata']['labels'] = all_labels return { 'name': 'projects/{}/locations/{}/services/{}'.format( @@ -98,7 +149,7 @@ def get_label_params(resource, all_labels): metadata = resource['metadata'] namespace = metadata['namespace'] job_name = metadata['name'] - body = copy.deepcopy(resource) + body = knative_body(resource) body['metadata']['labels'] = all_labels return { 'name': 'namespaces/{}/jobs/{}'.format(namespace, job_name), diff --git a/tools/c7n_gcp/pyproject.toml b/tools/c7n_gcp/pyproject.toml index e9c2c6d2cbe..26bc32a6e1a 100644 --- a/tools/c7n_gcp/pyproject.toml +++ b/tools/c7n_gcp/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "retrying<2.0.0,>=1.3.3", ] name = "c7n_gcp" -version = "0.4.51a5.post1+a83dda5c9" +version = "0.4.51a5.post2+a83dda5c9" description = "Cloud Custodian - Google Cloud Provider" readme = "readme.md" classifiers = [ diff --git a/tools/c7n_gcp/tests/data/flights/gcp-cloud-run-service-mark-for-op/get-v1-projects-cloud-custodian-locations---services_1.json b/tools/c7n_gcp/tests/data/flights/gcp-cloud-run-service-mark-for-op/get-v1-projects-cloud-custodian-locations---services_1.json new file mode 100644 index 00000000000..998083a29fb --- /dev/null +++ b/tools/c7n_gcp/tests/data/flights/gcp-cloud-run-service-mark-for-op/get-v1-projects-cloud-custodian-locations---services_1.json @@ -0,0 +1,57 @@ +{ + "headers": { + "content-type": "application/json; charset=UTF-8", + "status": "200" + }, + "body": { + "apiVersion": "serving.knative.dev/v1", + "kind": "ServiceList", + "items": [ + { + "apiVersion": "serving.knative.dev/v1", + "kind": "Service", + "metadata": { + "name": "hello", + "namespace": "cloud-custodian", + "selfLink": "/apis/serving.knative.dev/v1/namespaces/cloud-custodian/services/hello", + "uid": "46b28549-6c5c-4245-bc8d-9cec06d262b6", + "resourceVersion": "AAX4MAndusc", + "generation": 1, + "labels": { + "cloud.googleapis.com/location": "us-central1" + }, + "annotations": { + "run.googleapis.com/ingress": "all" + }, + "creationTimestamp": "2023-03-31T10:53:45.882066Z" + }, + "spec": { + "template": { + "metadata": { + "name": "hello-00001-tiv", + "annotations": {} + }, + "spec": { + "containerConcurrency": 80, + "timeoutSeconds": 300, + "containers": [ + { + "image": "us-docker.pkg.dev/cloudrun/container/hello", + "ports": [{"name": "http1", "containerPort": 8080}] + } + ] + } + }, + "traffic": [{"percent": 100, "latestRevision": true}] + }, + "status": { + "observedGeneration": 1, + "conditions": [ + {"type": "Ready", "status": "True"} + ], + "url": "https://hello-g3kdvpt64a-uc.a.run.app" + } + } + ] + } +} diff --git a/tools/c7n_gcp/tests/data/flights/gcp-cloud-run-service-mark-for-op/put-v1-projects-cloud-custodian-locations-us-central1-services-hello_1.json b/tools/c7n_gcp/tests/data/flights/gcp-cloud-run-service-mark-for-op/put-v1-projects-cloud-custodian-locations-us-central1-services-hello_1.json new file mode 100644 index 00000000000..b64a30335e2 --- /dev/null +++ b/tools/c7n_gcp/tests/data/flights/gcp-cloud-run-service-mark-for-op/put-v1-projects-cloud-custodian-locations-us-central1-services-hello_1.json @@ -0,0 +1,52 @@ +{ + "headers": { + "content-type": "application/json; charset=UTF-8", + "status": "200" + }, + "body": { + "apiVersion": "serving.knative.dev/v1", + "kind": "Service", + "metadata": { + "name": "hello", + "namespace": "cloud-custodian", + "selfLink": "/apis/serving.knative.dev/v1/namespaces/cloud-custodian/services/hello", + "uid": "46b28549-6c5c-4245-bc8d-9cec06d262b6", + "resourceVersion": "AAX4MBndusc", + "generation": 2, + "labels": { + "cloud.googleapis.com/location": "us-central1", + "custodian_status": "resource_policy-notify-2026_08_21__0_0" + }, + "annotations": { + "run.googleapis.com/ingress": "all" + }, + "creationTimestamp": "2023-03-31T10:53:45.882066Z" + }, + "spec": { + "template": { + "metadata": { + "name": "hello-00001-tiv", + "annotations": {} + }, + "spec": { + "containerConcurrency": 80, + "timeoutSeconds": 300, + "containers": [ + { + "image": "us-docker.pkg.dev/cloudrun/container/hello", + "ports": [{"name": "http1", "containerPort": 8080}] + } + ] + } + }, + "traffic": [{"percent": 100, "latestRevision": true}] + }, + "status": { + "observedGeneration": 2, + "conditions": [ + {"type": "Ready", "status": "True"} + ], + "url": "https://hello-g3kdvpt64a-uc.a.run.app" + } + } +} diff --git a/tools/c7n_gcp/tests/test_cloudrun.py b/tools/c7n_gcp/tests/test_cloudrun.py index 7483c0c7a95..f4f7061a1ef 100644 --- a/tools/c7n_gcp/tests/test_cloudrun.py +++ b/tools/c7n_gcp/tests/test_cloudrun.py @@ -1,9 +1,153 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 +import pytest + +from freezegun import freeze_time + from gcp_common import BaseTest +from c7n.exceptions import PolicyExecutionError from c7n.utils import yaml_load +from c7n_gcp.actions.core import MethodAction +from c7n_gcp.resources.cloudrun import CloudRunJob, CloudRunService, knative_body + +# The root fields the api accepts, independent of the module under test. +SCHEMA_FIELDS = {'apiVersion', 'kind', 'metadata', 'spec', 'status'} +LOCATION = {'cloud.googleapis.com/location': 'us-central1'} + + +def listed_resource(api_version, kind, name): + "A resource as augment() and a value filter leave it once listed." + return { + 'apiVersion': api_version, + 'kind': kind, + 'metadata': { + 'name': name, + 'namespace': 'cloud-custodian', + 'labels': dict(LOCATION), + }, + 'spec': {'template': {}}, + 'status': {'observedGeneration': 1}, + 'labels': dict(LOCATION), + 'c7n:MatchedFilters': ['metadata.name'], + } + + +def capture_api_params(test): + """Record the params of every api call the policy's actions make.""" + captured = [] + invoke_api = MethodAction.invoke_api + + def record(action, client, op_name, params): + captured.append((op_name, params)) + return invoke_api(action, client, op_name, params) + + test.patch(MethodAction, 'invoke_api', record) + return captured + + +def assert_replace_body(body, labels): + "A replace body holds the schema fields and the merged labels, nothing else." + assert set(body) == SCHEMA_FIELDS + assert 'labels' not in body + assert 'c7n:MatchedFilters' not in body + assert body['metadata']['labels'] == labels + + +def assert_label_params(model, resource, expected_name): + "get_label_params addresses the resource and leaves it fit to read again." + all_labels = dict(LOCATION, environment='test') + params = model.get_label_params(resource, all_labels) + + assert params['name'] == expected_name + assert_replace_body(params['body'], all_labels) + # the action reads its current label state from the resource, so both the + # synthetic key and the metadata it came from have to survive the call + assert resource['labels'] == LOCATION + assert resource['metadata']['labels'] == LOCATION + + +class KnativeBodyTest(BaseTest): + + def test_keeps_only_the_schema_fields(self): + body = knative_body({ + 'apiVersion': 'serving.knative.dev/v1', + 'kind': 'Service', + 'metadata': {'name': 'hello'}, + 'spec': {}, + 'status': {}, + 'labels': {'env': 'test'}, + 'c7n:MatchedFilters': ['metadata.name'], + }) + assert set(body) == SCHEMA_FIELDS + + def test_drops_annotations_nested_in_spec(self): + # a list-item filter annotates the objects it matched in place + body = knative_body({ + 'metadata': {'name': 'hello'}, + 'spec': { + 'template': { + 'spec': { + 'containers': [{ + 'image': 'us-docker.pkg.dev/cloudrun/container/hello', + 'c7n:MatchedFilters': ['image'], + }], + }, + }, + }, + 'c7n.metrics': {'run.googleapis.com/request_count': []}, + }) + assert body['spec'] == { + 'template': { + 'spec': { + 'containers': [{ + 'image': 'us-docker.pkg.dev/cloudrun/container/hello'}], + }, + }, + } + + def test_keeps_user_annotations(self): + # a kubernetes annotation key may contain a dot, so only the colon form + # is recognized below the root - a replace deletes what the body omits + annotations = { + 'run.googleapis.com/ingress': 'all', + 'c7n.example.com/owner': 'platform-team', + } + body = knative_body({ + 'metadata': {'name': 'hello', 'annotations': dict(annotations)}, + 'spec': {'template': {'metadata': {'annotations': dict(annotations)}}}, + }) + assert body['metadata']['annotations'] == annotations + assert body['spec']['template']['metadata']['annotations'] == annotations + + def test_copies_rather_than_mutates(self): + resource = { + 'metadata': {'name': 'hello'}, + 'spec': {'containers': [{'c7n:MatchedFilters': ['image']}]}, + } + knative_body(resource)['metadata']['name'] = 'changed' + assert resource['metadata']['name'] == 'hello' + assert resource['spec']['containers'][0] == {'c7n:MatchedFilters': ['image']} + + def test_raises_on_fields_outside_the_schema(self): + with pytest.raises(PolicyExecutionError) as raised: + knative_body({ + 'kind': 'Service', + 'metadata': {'name': 'hello'}, + 'somethingNew': 'unrecognized', + }) + assert 'somethingNew' in str(raised.value) + assert 'hello' in str(raised.value) + + def test_accepts_the_synthetic_fields(self): + knative_body({ + 'metadata': {'name': 'hello'}, + 'labels': {'env': 'test'}, + 'c7n:MatchedFilters': ['metadata.name'], + 'c7n.metrics': {'run.googleapis.com/request_count': []}, + }) + class RunServiceTest(BaseTest): def test_query(self): @@ -28,11 +172,18 @@ def test_query_empty(self): resources = p.run() assert resources == [] + def test_label_params(self): + assert_label_params( + CloudRunService.resource_type, + listed_resource('serving.knative.dev/v1', 'Service', 'hello'), + 'projects/cloud-custodian/locations/us-central1/services/hello') + def test_set_labels(self): project_id = 'cloud-custodian' factory = self.replay_flight_data( "gcp-cloud-run-service-set-labels", project_id=project_id ) + captured = capture_api_params(self) p = self.load_policy( { 'name': 'cloud-run-svc-set-labels', @@ -53,6 +204,49 @@ def test_set_labels(self): assert len(resources) == 1 assert resources[0]['metadata']['name'] == 'hello' + assert len(captured) == 1 + op_name, params = captured.pop() + assert op_name == 'replaceService' + assert_replace_body(params['body'], { + 'cloud.googleapis.com/location': 'us-central1', + 'environment': 'test', + }) + + @freeze_time("2026-08-19") + def test_mark_for_op(self): + project_id = 'cloud-custodian' + factory = self.replay_flight_data( + "gcp-cloud-run-service-mark-for-op", project_id=project_id + ) + captured = capture_api_params(self) + p = self.load_policy( + { + 'name': 'cloud-run-svc-mark-for-op', + 'resource': 'gcp.cloud-run-service', + 'filters': [ + {'type': 'value', + 'key': 'metadata.name', + 'value': 'hello'} + ], + 'actions': [ + {'type': 'mark-for-op', + 'op': 'notify', + 'days': 2} + ] + }, + session_factory=factory, + ) + resources = p.run() + assert len(resources) == 1 + + assert len(captured) == 1 + op_name, params = captured.pop() + assert op_name == 'replaceService' + assert_replace_body(params['body'], { + 'cloud.googleapis.com/location': 'us-central1', + 'custodian_status': 'resource_policy-notify-2026_08_21__0_0', + }) + def test_filter(self): factory = self.replay_flight_data("gcp-cloud-run-service") @@ -103,11 +297,18 @@ def test_query(self): assert len(resources) == 1 assert resources[0]["metadata"]["name"] == "job" + def test_label_params(self): + assert_label_params( + CloudRunJob.resource_type, + listed_resource('run.googleapis.com/v1', 'Job', 'job'), + 'namespaces/cloud-custodian/jobs/job') + def test_set_labels(self): project_id = 'cloud-custodian' factory = self.replay_flight_data( "gcp-cloud-run-job-set-labels", project_id=project_id ) + captured = capture_api_params(self) p = self.load_policy( { 'name': 'cloud-run-job-set-labels', @@ -128,6 +329,14 @@ def test_set_labels(self): assert len(resources) == 1 assert resources[0]['metadata']['name'] == 'job' + assert len(captured) == 1 + op_name, params = captured.pop() + assert op_name == 'replaceJob' + assert_replace_body(params['body'], { + 'cloud.googleapis.com/location': 'us-central1', + 'environment': 'test', + }) + class RevisionServiceTest(BaseTest): def test_query(self): diff --git a/tools/c7n_gcp/tests/test_gcp_labels.py b/tools/c7n_gcp/tests/test_gcp_labels.py index 6c019515bf8..cc4c813898a 100644 --- a/tools/c7n_gcp/tests/test_gcp_labels.py +++ b/tools/c7n_gcp/tests/test_gcp_labels.py @@ -47,6 +47,78 @@ def test_schema_validate(self): ])) +class SetLabelsLookupTest(BaseTest): + """Resource lookups used as values inside the ``labels`` mapping.""" + + def _get_action(self, labels): + policy = self.load_policy( + get_policy([{'type': 'set-labels', 'labels': labels}])) + return policy.resource_manager.actions[0] + + def test_lookup_by_key_resolves(self): + action = self._get_action({'env': {'type': 'resource', 'key': 'name'}}) + self.assertEqual( + action.get_labels_to_add({'name': 'instance-1', 'labels': {}}), + {'env': 'instance-1'}) + + def test_lookup_miss_uses_default_value(self): + action = self._get_action( + {'env': {'type': 'resource', + 'key': 'doesnotexist', + 'default-value': 'production'}}) + self.assertEqual( + action.get_labels_to_add({'name': 'instance-1', 'labels': {}}), + {'env': 'production'}) + + def test_static_value_passthrough(self): + action = self._get_action({'env': 'test'}) + self.assertEqual( + action.get_labels_to_add({'name': 'instance-1', 'labels': {}}), + {'env': 'test'}) + + def test_conditional_default_writes_when_label_absent(self): + action = self._get_action( + {'owner': {'type': 'resource', 'default-value': 'platform'}}) + self.assertEqual( + action.get_labels_to_add({'name': 'instance-1', 'labels': {}}), + {'owner': 'platform'}) + + def test_conditional_default_skipped_when_label_present(self): + action = self._get_action( + {'owner': {'type': 'resource', 'default-value': 'platform'}}) + self.assertEqual( + action.get_labels_to_add( + {'name': 'instance-1', 'labels': {'owner': 'keep-me'}}), + {}) + + def test_conditional_default_leaves_existing_label_intact(self): + """The skipped label must survive the merge with current labels.""" + action = self._get_action( + {'owner': {'type': 'resource', 'default-value': 'platform'}, + 'env': 'test'}) + resource = {'name': 'instance-1', 'labels': {'owner': 'keep-me'}} + merged = action._merge_labels( + action._get_current_labels(resource), + action.get_labels_to_add(resource), + action.get_labels_to_delete(resource)) + self.assertEqual(merged, {'owner': 'keep-me', 'env': 'test'}) + + def test_schema_accepts_conditional_default(self): + self.assertTrue(self.load_policy(get_policy([ + {'type': 'set-labels', + 'labels': {'owner': {'type': 'resource', 'default-value': 'x'}}} + ]))) + + def test_schema_rejects_malformed_label_values(self): + for bad in ({'type': 'resource'}, + {'type': 'resource', 'ky': 'name'}, + {'foo': 'bar'}): + with self.assertRaises(Exception): + self.load_policy( + get_policy([{'type': 'set-labels', 'labels': {'x': bad}}]), + validate=True) + + class LabelDelayedActionTest(BaseTest): def test_schema_validate(self): diff --git a/tools/c7n_kube/pyproject.toml b/tools/c7n_kube/pyproject.toml index 744c9beb1b0..39d3db3786b 100644 --- a/tools/c7n_kube/pyproject.toml +++ b/tools/c7n_kube/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "kubernetes>=10.0.1", ] name = "c7n_kube" -version = "0.2.51a5.post1+a83dda5c9" +version = "0.2.51a5.post2+a83dda5c9" description = "Cloud Custodian - Kubernetes Provider" readme = "readme.md" classifiers = [ diff --git a/tools/c7n_left/pyproject.toml b/tools/c7n_left/pyproject.toml index f5a2e88ddff..8f1dfe3c8f6 100644 --- a/tools/c7n_left/pyproject.toml +++ b/tools/c7n_left/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "tfparse<1.0,>=0.6", ] name = "c7n_left" -version = "0.3.38a5.post1+a83dda5c9" +version = "0.3.38a5.post2+a83dda5c9" description = "Custodian policies for IAAC definitions" readme = "README.md" classifiers = [ diff --git a/tools/c7n_mailer/pyproject.toml b/tools/c7n_mailer/pyproject.toml index 4f63b5407b7..1a9b4f2fd20 100644 --- a/tools/c7n_mailer/pyproject.toml +++ b/tools/c7n_mailer/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "types-six<2.0.0,>=1.16.10", ] name = "c7n_mailer" -version = "0.6.51a5.post1+a83dda5c9" +version = "0.6.51a5.post2+a83dda5c9" description = "Cloud Custodian - Reference Mailer" readme = "README.md" classifiers = [ diff --git a/tools/c7n_oci/pyproject.toml b/tools/c7n_oci/pyproject.toml index fb7f8e8ecd8..6e5731cbce7 100644 --- a/tools/c7n_oci/pyproject.toml +++ b/tools/c7n_oci/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "pyopenssl>=24" ] name = "c7n_oci" -version = "0.1.26a5.post1+a83dda5c9" +version = "0.1.26a5.post2+a83dda5c9" description = "Cloud Custodian - OCI Support" readme = "readme.md" classifiers = [ diff --git a/tools/c7n_openstack/pyproject.toml b/tools/c7n_openstack/pyproject.toml index ac1e7c6f42e..d5ba37ece40 100644 --- a/tools/c7n_openstack/pyproject.toml +++ b/tools/c7n_openstack/pyproject.toml @@ -9,7 +9,7 @@ dependencies = [ "openstacksdk==0.52.0", ] name = "c7n_openstack" -version = "0.1.42a5.post1+a83dda5c9" +version = "0.1.42a5.post2+a83dda5c9" description = "Cloud Custodian - OpenStack Provider" readme = "readme.md" classifiers = [ diff --git a/tools/c7n_org/pyproject.toml b/tools/c7n_org/pyproject.toml index 4b22cb92c9e..72d594dc30c 100644 --- a/tools/c7n_org/pyproject.toml +++ b/tools/c7n_org/pyproject.toml @@ -9,7 +9,7 @@ dependencies = [ "click>=8.0", ] name = "c7n_org" -version = "0.6.51a5.post1+a83dda5c9" +version = "0.6.51a5.post2+a83dda5c9" description = "Cloud Custodian - Parallel Execution" readme = "README.md" classifiers = [ diff --git a/tools/c7n_policystream/pyproject.toml b/tools/c7n_policystream/pyproject.toml index c155712dcbe..e3d170f2924 100644 --- a/tools/c7n_policystream/pyproject.toml +++ b/tools/c7n_policystream/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "requests<3.0.0,>=2.22.0", ] name = "c7n_policystream" -version = "0.4.51a5.post1+a83dda5c9" +version = "0.4.51a5.post2+a83dda5c9" description = "Cloud Custodian - Git Commits as Logical Policy Changes" readme = "README.md" classifiers = [ diff --git a/tools/c7n_sphinxext/pyproject.toml b/tools/c7n_sphinxext/pyproject.toml index 19e304aad68..4fbbdef03c6 100644 --- a/tools/c7n_sphinxext/pyproject.toml +++ b/tools/c7n_sphinxext/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "sphinx-rtd-theme>=1.0.0", ] name = "c7n_sphinxext" -version = "1.1.51a5.post1+a83dda5c9" +version = "1.1.51a5.post2+a83dda5c9" description = "Cloud Custodian - Sphinx Extensions" classifiers = [ "License :: OSI Approved :: Apache Software License", diff --git a/tools/c7n_tencentcloud/pyproject.toml b/tools/c7n_tencentcloud/pyproject.toml index 152bbdf15f1..0916532ed4e 100644 --- a/tools/c7n_tencentcloud/pyproject.toml +++ b/tools/c7n_tencentcloud/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "tencentcloud-sdk-python<4.0.0,>=3.0.783", ] name = "c7n_tencentcloud" -version = "0.1.34a5.post1+a83dda5c9" +version = "0.1.34a5.post2+a83dda5c9" description = "Cloud Custodian - Tencent Cloud Provider" readme = "readme.md" diff --git a/uv.lock b/uv.lock index e095edb72f4..275ec2a2199 100644 --- a/uv.lock +++ b/uv.lock @@ -1488,7 +1488,7 @@ wheels = [ [[package]] name = "c7n" -version = "0.9.52a5.post1+a83dda5c9" +version = "0.9.52a5.post2+a83dda5c9" source = { editable = "." } dependencies = [ { name = "argcomplete" }, @@ -1577,7 +1577,7 @@ lint = [ [[package]] name = "c7n-awscc" -version = "0.1.37a5.post1+a83dda5c9" +version = "0.1.37a5.post2+a83dda5c9" source = { editable = "tools/c7n_awscc" } dependencies = [ { name = "boto3" }, @@ -1602,7 +1602,7 @@ dev = [{ name = "pytest" }] [[package]] name = "c7n-azure" -version = "0.7.51a5.post1+a83dda5c9" +version = "0.7.51a5.post2+a83dda5c9" source = { editable = "tools/c7n_azure" } dependencies = [ { name = "adal" }, @@ -1785,7 +1785,7 @@ dev = [ [[package]] name = "c7n-gcp" -version = "0.4.51a5.post1+a83dda5c9" +version = "0.4.51a5.post2+a83dda5c9" source = { editable = "tools/c7n_gcp" } dependencies = [ { name = "c7n" }, @@ -1826,7 +1826,7 @@ dev = [ [[package]] name = "c7n-kube" -version = "0.2.51a5.post1+a83dda5c9" +version = "0.2.51a5.post2+a83dda5c9" source = { editable = "tools/c7n_kube" } dependencies = [ { name = "c7n" }, @@ -1855,7 +1855,7 @@ dev = [ [[package]] name = "c7n-left" -version = "0.3.38a5.post1+a83dda5c9" +version = "0.3.38a5.post2+a83dda5c9" source = { editable = "tools/c7n_left" } dependencies = [ { name = "c7n" }, @@ -1888,7 +1888,7 @@ dev = [ [[package]] name = "c7n-mailer" -version = "0.6.51a5.post1+a83dda5c9" +version = "0.6.51a5.post2+a83dda5c9" source = { editable = "tools/c7n_mailer" } dependencies = [ { name = "boto3" }, @@ -1954,7 +1954,7 @@ dev = [ [[package]] name = "c7n-oci" -version = "0.1.26a5.post1+a83dda5c9" +version = "0.1.26a5.post2+a83dda5c9" source = { editable = "tools/c7n_oci" } dependencies = [ { name = "c7n" }, @@ -1979,7 +1979,7 @@ dev = [{ name = "pytest" }] [[package]] name = "c7n-openstack" -version = "0.1.42a5.post1+a83dda5c9" +version = "0.1.42a5.post2+a83dda5c9" source = { editable = "tools/c7n_openstack" } dependencies = [ { name = "c7n" }, @@ -2006,7 +2006,7 @@ dev = [ [[package]] name = "c7n-org" -version = "0.6.51a5.post1+a83dda5c9" +version = "0.6.51a5.post2+a83dda5c9" source = { editable = "tools/c7n_org" } dependencies = [ { name = "c7n" }, @@ -2029,7 +2029,7 @@ dev = [{ name = "pytest" }] [[package]] name = "c7n-policystream" -version = "0.4.51a5.post1+a83dda5c9" +version = "0.4.51a5.post2+a83dda5c9" source = { editable = "tools/c7n_policystream" } dependencies = [ { name = "boto3" }, @@ -2065,7 +2065,7 @@ dev = [ [[package]] name = "c7n-sphinxext" -version = "1.1.51a5.post1+a83dda5c9" +version = "1.1.51a5.post2+a83dda5c9" source = { editable = "tools/c7n_sphinxext" } dependencies = [ { name = "c7n" }, @@ -2092,7 +2092,7 @@ requires-dist = [ [[package]] name = "c7n-tencentcloud" -version = "0.1.34a5.post1+a83dda5c9" +version = "0.1.34a5.post2+a83dda5c9" source = { editable = "tools/c7n_tencentcloud" } dependencies = [ { name = "c7n" },