From 1dde3003cfcdbb8473a347045cc9b886e9d90780 Mon Sep 17 00:00:00 2001 From: AJ Kerrigan Date: Mon, 14 Sep 2026 20:59:43 -0400 Subject: [PATCH 1/7] aws - tagging - tolerate resources deleted mid-run (#11089) --------- Co-authored-by: Claude Opus 5 (1M context) --- c7n/resources/ebs.py | 58 +-- c7n/resources/eks.py | 10 +- c7n/tags.py | 70 +++- tests/common.py | 6 +- .../test_core_network_delete_error/.gitkeep | 0 .../eks.TagResource_1.json | 11 + .../eks.UntagResource_1.json | 11 + .../ec2.CreateTags_1.json | 10 + .../ec2.CreateTags_2.json | 6 + .../ec2.DeleteTags_1.json | 6 + .../ec2.DescribeInstanceImageMetadata_1.json | 132 ++++++ .../ec2.DescribeInstances_1.json | 374 +++++++++++++++++ .../ec2.DescribeInstances_2.json | 362 ++++++++++++++++ .../tagging.TagResources_1.json | 13 + tests/terraform/ec2_instance_tag/main.tf | 36 ++ .../ec2_instance_tag/tf_resources.json | 392 ++++++++++++++++++ tests/test_ebs.py | 20 - tests/test_eks.py | 29 ++ tests/test_networkmanager.py | 2 +- tests/test_tags.py | 182 ++++++++ tests/zpill.py | 20 + 21 files changed, 1658 insertions(+), 92 deletions(-) create mode 100644 tests/data/placebo/test_core_network_delete_error/.gitkeep create mode 100644 tests/data/placebo/test_eks_tag_deleted_cluster/eks.TagResource_1.json create mode 100644 tests/data/placebo/test_eks_tag_deleted_cluster/eks.UntagResource_1.json create mode 100644 tests/data/placebo/test_native_tag_skips_gone_instance/ec2.CreateTags_1.json create mode 100644 tests/data/placebo/test_native_tag_skips_gone_instance/ec2.CreateTags_2.json create mode 100644 tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DeleteTags_1.json create mode 100644 tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstanceImageMetadata_1.json create mode 100644 tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstances_1.json create mode 100644 tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstances_2.json create mode 100644 tests/data/placebo/test_universal_tag_skips_gone_snapshot/tagging.TagResources_1.json create mode 100644 tests/terraform/ec2_instance_tag/main.tf create mode 100644 tests/terraform/ec2_instance_tag/tf_resources.json diff --git a/c7n/resources/ebs.py b/c7n/resources/ebs.py index 0058450d339..087c9508886 100644 --- a/c7n/resources/ebs.py +++ b/c7n/resources/ebs.py @@ -22,7 +22,7 @@ from c7n.resources.kms import ResourceKmsKeyAlias from c7n.resources.securityhub import PostFinding from c7n.query import QueryResourceManager, TypeInfo -from c7n.tags import Tag, coalesce_copy_user_tags +from c7n.tags import coalesce_copy_user_tags from c7n.utils import ( camelResource, chunks, @@ -94,16 +94,6 @@ def get_resources(self, ids, cache=True, augment=True): class ErrorHandler: - @staticmethod - def remove_snapshot(rid, resource_set): - found = None - for r in resource_set: - if r['SnapshotId'] == rid: - found = r - break - if found: - resource_set.remove(found) - @staticmethod def extract_bad_snapshot(e): """Handle various client side errors when describing snapshots""" @@ -132,16 +122,6 @@ def extract_bad_volume(e): log.warning("Volume id malformed %s" % e_vol_id) return e_vol_id - @staticmethod - def remove_volume(rid, resource_set): - found = None - for r in resource_set: - if r['VolumeId'] == rid: - found = r - break - if found: - resource_set.remove(found) - class SnapshotQueryParser(QueryParser): @@ -200,24 +180,6 @@ class VolumeQueryParser(QueryParser): type_name = 'EBS Volume' -@Snapshot.action_registry.register('tag') -class SnapshotTag(Tag): - - permissions = ('ec2:CreateTags',) - - def process_resource_set(self, client, resource_set, tags): - while resource_set: - try: - return super(SnapshotTag, self).process_resource_set( - client, resource_set, tags) - except ClientError as e: - bad_snap = ErrorHandler.extract_bad_snapshot(e) - if bad_snap: - ErrorHandler.remove_snapshot(bad_snap, resource_set) - continue - raise - - @Snapshot.filter_registry.register('age') class SnapshotAge(AgeFilter): """EBS Snapshot Age Filter @@ -739,24 +701,6 @@ def get_resources(self, ids, cache=True, augment=True): return [] -@EBS.action_registry.register('tag') -class VolumeTag(Tag): - - permissions = ('ec2:CreateTags',) - - def process_resource_set(self, client, resource_set, tags): - while resource_set: - try: - return super(VolumeTag, self).process_resource_set( - client, resource_set, tags) - except ClientError as e: - bad_vol = ErrorHandler.extract_bad_volume(e) - if bad_vol: - ErrorHandler.remove_volume(bad_vol, resource_set) - continue - raise - - @EBS.filter_registry.register('snapshots') class EBSSnapshotsFilter(ListItemFilter): """ diff --git a/c7n/resources/eks.py b/c7n/resources/eks.py index 74f0b9d70ad..9c174099c17 100644 --- a/c7n/resources/eks.py +++ b/c7n/resources/eks.py @@ -10,9 +10,11 @@ from c7n.manager import resources from c7n.resources.aws import shape_schema from c7n import tags, query +from c7n.tags import is_resource_gone from c7n.query import QueryResourceManager, TypeInfo, DescribeSource, \ ChildResourceManager, ChildDescribeSource from c7n.utils import local_session, type_schema, get_retry +from botocore.exceptions import ClientError from botocore.waiter import WaiterModel, create_waiter_with_client from .aws import shape_validate from .ecs import ContainerConfigSource @@ -339,7 +341,9 @@ def process_resource_set(self, client, resource_set, tags): client.tag_resource, resourceArn=r['arn'], tags={t['Key']: t['Value'] for t in tags}) - except client.exceptions.ResourceNotFoundException: + except ClientError as e: + if not is_resource_gone(e.response['Error']['Code']): + raise continue @@ -358,7 +362,9 @@ def process_resource_set(self, client, resource_set, tags): self.manager.retry( client.untag_resource, resourceArn=r['arn'], tagKeys=tags) - except client.exceptions.ResourceNotFoundException: + except ClientError as e: + if not is_resource_gone(e.response['Error']['Code']): + raise continue diff --git a/c7n/tags.py b/c7n/tags.py index 2ecb97235d0..9e78f70a327 100644 --- a/c7n/tags.py +++ b/c7n/tags.py @@ -8,6 +8,8 @@ snapshots) and resources that support Amazon's Resource Groups Tagging API """ +import re + from collections import Counter from concurrent.futures import as_completed @@ -17,6 +19,8 @@ import time +from botocore.exceptions import ClientError + from c7n.manager import resources as aws_resources from c7n.actions import BaseAction as Action, AutoTagUser from c7n.exceptions import PolicyValidationError, PolicyExecutionError @@ -28,6 +32,54 @@ DEFAULT_TAG = "maid_status" +# Tag actions inherently race with resource deletion, we enumerate a +# resource set and then mutate it. ec2 fails the whole batch when any +# resource in it is unusable, but names the offending ids in the error +# message, so we can drop those and tag the rest. +EC2_BAD_RESOURCE_ID = re.compile(r'^Invalid.+\.(NotFound|Malformed)$') +QUOTED_RESOURCE_IDS = re.compile(r"['\"]([^'\"]+)['\"]") + +# Other services report a resource as already gone with a service specific +# error code - query protocol services use NotFound(Fault), +# json protocol services use (Resource)NotFoundException. +RESOURCE_GONE_ERROR = re.compile(r'NotFound(Fault|Exception)?$') + + +def is_resource_gone(error_code): + """Does an api error code mean the resource we wanted is already gone?""" + return bool(RESOURCE_GONE_ERROR.search(error_code)) + + +def extract_bad_resource_ids(e): + """Extract the resource ids an ec2 api call rejected from its error. + + ec2 names the ids it couldn't use in the error message, ie + ``The instance IDs 'i-0e8f4b9a' do not exist``. + """ + if not EC2_BAD_RESOURCE_ID.match(e.response['Error']['Code']): + return () + return tuple( + rid.strip(" []") + for quoted in QUOTED_RESOURCE_IDS.findall(e.response['Error']['Message']) + for rid in quoted.split(',')) + + +def retry_skip_bad_ids(retry, method, resource_ids, log, **kw): + """Invoke a batched ec2 tag api call, skipping unusable resource ids. + + A resource deleted between enumeration and tagging fails the entire + batch, so drop the ids ec2 rejected and tag the ones still around. + """ + while resource_ids: + try: + return retry(method, Resources=resource_ids, **kw) + except ClientError as e: + bad_ids = set(extract_bad_resource_ids(e)).intersection(resource_ids) + if not bad_ids: + raise + log.info("skipping missing resources %s", ", ".join(sorted(bad_ids))) + resource_ids = [r for r in resource_ids if r not in bad_ids] + # Sentinel: the resolver returns this to omit a tag from a resource's payload. TAG_VALUE_SKIP = object() @@ -528,9 +580,9 @@ def _resolve_and_group(self, resources, spec_map): def process_resource_set(self, client, resource_set, tags): mid = self.manager.get_model().id - self.manager.retry( - client.create_tags, - Resources=[v[mid] for v in resource_set], + retry_skip_bad_ids( + self.manager.retry, client.create_tags, + [v[mid] for v in resource_set], self.log, Tags=tags, DryRun=self.manager.config.dryrun) @@ -584,9 +636,9 @@ def process(self, resources): self.process_resource_set, self.id_key, resources, tags, self.log) def process_resource_set(self, client, resource_set, tag_keys): - return self.manager.retry( - client.delete_tags, - Resources=[v[self.id_key] for v in resource_set], + return retry_skip_bad_ids( + self.manager.retry, client.delete_tags, + [v[self.id_key] for v in resource_set], self.log, Tags=[{'Key': k} for k in tag_keys], DryRun=self.manager.config.dryrun) @@ -1409,7 +1461,7 @@ def universal_retry(method, ResourceARNList, **kw): error_code = failures[f_arn]['ErrorCode'] if error_code == 'ThrottlingException': throttles.add(f_arn) - elif error_code == 'ResourceNotFoundException': + elif is_resource_gone(error_code): continue else: errors[f_arn] = error_code @@ -1417,6 +1469,10 @@ def universal_retry(method, ResourceARNList, **kw): if errors: raise Exception("Resource Tag Errors %s" % (errors)) + if not throttles: + # everything else was skipped, the api rejects an empty arn list + return response + if idx == max_attempts - 1: raise Exception("Resource Tag Throttled %s" % (", ".join(throttles))) diff --git a/tests/common.py b/tests/common.py index 220a2efd92f..3fa93deaeb0 100644 --- a/tests/common.py +++ b/tests/common.py @@ -10,7 +10,7 @@ from c7n.testing import TestUtils, TextTestIO, functional # NOQA -from .zpill import PillTest, ACCOUNT_ID +from .zpill import PillTest, ACCOUNT_ID # NOQA logging.getLogger("placebo.pill").setLevel(logging.DEBUG) @@ -33,10 +33,6 @@ class BaseTest(TestUtils, PillTest): # custodian_schema = C7N_SCHEMA - @property - def account_id(self): - return ACCOUNT_ID - def _get_policy_config(self, **kw): if 'account_id' not in kw: kw['account_id'] = self.account_id diff --git a/tests/data/placebo/test_core_network_delete_error/.gitkeep b/tests/data/placebo/test_core_network_delete_error/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/data/placebo/test_eks_tag_deleted_cluster/eks.TagResource_1.json b/tests/data/placebo/test_eks_tag_deleted_cluster/eks.TagResource_1.json new file mode 100644 index 00000000000..b067a0dbc0b --- /dev/null +++ b/tests/data/placebo/test_eks_tag_deleted_cluster/eks.TagResource_1.json @@ -0,0 +1,11 @@ +{ + "status_code": 404, + "data": { + "Error": { + "Message": "Resource was not found", + "Code": "NotFoundException" + }, + "ResponseMetadata": {}, + "message": "Resource was not found" + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_eks_tag_deleted_cluster/eks.UntagResource_1.json b/tests/data/placebo/test_eks_tag_deleted_cluster/eks.UntagResource_1.json new file mode 100644 index 00000000000..b067a0dbc0b --- /dev/null +++ b/tests/data/placebo/test_eks_tag_deleted_cluster/eks.UntagResource_1.json @@ -0,0 +1,11 @@ +{ + "status_code": 404, + "data": { + "Error": { + "Message": "Resource was not found", + "Code": "NotFoundException" + }, + "ResponseMetadata": {}, + "message": "Resource was not found" + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.CreateTags_1.json b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.CreateTags_1.json new file mode 100644 index 00000000000..ebc15c522c0 --- /dev/null +++ b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.CreateTags_1.json @@ -0,0 +1,10 @@ +{ + "status_code": 400, + "data": { + "Error": { + "Code": "InvalidInstanceID.NotFound", + "Message": "The instance ID 'i-0023456789abcdef0' does not exist" + }, + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.CreateTags_2.json b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.CreateTags_2.json new file mode 100644 index 00000000000..5b2170a073c --- /dev/null +++ b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.CreateTags_2.json @@ -0,0 +1,6 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DeleteTags_1.json b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DeleteTags_1.json new file mode 100644 index 00000000000..5b2170a073c --- /dev/null +++ b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DeleteTags_1.json @@ -0,0 +1,6 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstanceImageMetadata_1.json b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstanceImageMetadata_1.json new file mode 100644 index 00000000000..77c70a527a7 --- /dev/null +++ b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstanceImageMetadata_1.json @@ -0,0 +1,132 @@ +{ + "status_code": 200, + "data": { + "InstanceImageMetadata": [ + { + "InstanceId": "i-09347f9d6785dbe53", + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 32, + "second": 26, + "microsecond": 0 + }, + "AvailabilityZone": "us-east-1f", + "ZoneId": "use1-az5", + "State": { + "Code": 48, + "Name": "terminated" + }, + "OwnerId": "644160558196", + "ImageMetadata": { + "ImageId": "ami-0757b090c88c85bab", + "Name": "amzn2-ami-hvm-2.0.20260908.0-x86_64-gp2", + "OwnerId": "644160558196", + "State": "available", + "ImageOwnerAlias": "amazon", + "CreationDate": "2026-09-08T17:09:15Z", + "DeprecationTime": "2026-12-07T17:11:00Z", + "IsPublic": true + } + }, + { + "InstanceId": "i-0fd2708167c1cdb5a", + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 32, + "second": 26, + "microsecond": 0 + }, + "AvailabilityZone": "us-east-1f", + "ZoneId": "use1-az5", + "State": { + "Code": 48, + "Name": "terminated" + }, + "OwnerId": "644160558196", + "ImageMetadata": { + "ImageId": "ami-0757b090c88c85bab", + "Name": "amzn2-ami-hvm-2.0.20260908.0-x86_64-gp2", + "OwnerId": "644160558196", + "State": "available", + "ImageOwnerAlias": "amazon", + "CreationDate": "2026-09-08T17:09:15Z", + "DeprecationTime": "2026-12-07T17:11:00Z", + "IsPublic": true + } + }, + { + "InstanceId": "i-0e6f711d31e6675f4", + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "AvailabilityZone": "us-east-1c", + "ZoneId": "use1-az2", + "State": { + "Code": 16, + "Name": "running" + }, + "OwnerId": "644160558196", + "ImageMetadata": { + "ImageId": "ami-0757b090c88c85bab", + "Name": "amzn2-ami-hvm-2.0.20260908.0-x86_64-gp2", + "OwnerId": "644160558196", + "State": "available", + "ImageOwnerAlias": "amazon", + "CreationDate": "2026-09-08T17:09:15Z", + "DeprecationTime": "2026-12-07T17:11:00Z", + "IsPublic": true + } + }, + { + "InstanceId": "i-042a559882c4907dc", + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "AvailabilityZone": "us-east-1c", + "ZoneId": "use1-az2", + "State": { + "Code": 16, + "Name": "running" + }, + "OwnerId": "644160558196", + "ImageMetadata": { + "ImageId": "ami-0757b090c88c85bab", + "Name": "amzn2-ami-hvm-2.0.20260908.0-x86_64-gp2", + "OwnerId": "644160558196", + "State": "available", + "ImageOwnerAlias": "amazon", + "CreationDate": "2026-09-08T17:09:15Z", + "DeprecationTime": "2026-12-07T17:11:00Z", + "IsPublic": true + } + } + ], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstances_1.json b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstances_1.json new file mode 100644 index 00000000000..737c83c3c3d --- /dev/null +++ b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstances_1.json @@ -0,0 +1,374 @@ +{ + "status_code": 200, + "data": { + "Reservations": [ + { + "ReservationId": "r-00cf274ff3ee020d9", + "OwnerId": "644160558196", + "Groups": [], + "Instances": [ + { + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 54, + "microsecond": 0 + }, + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-0e9414ef1d2eb432c", + "EbsCardIndex": 0 + } + } + ], + "ClientToken": "terraform-aHfqApEb7WCuXQ32mgMxEZBCP7", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "AttachmentId": "eni-attach-026ad86d2c3a449f5", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "Ipv6Addresses": [], + "MacAddress": "12:cd:c8:af:f4:41", + "NetworkInterfaceId": "eni-0816616bd448027b2", + "OwnerId": "644160558196", + "PrivateIpAddress": "10.0.1.200", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.0.1.200" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "InterfaceType": "interface", + "Operator": { + "Managed": false + } + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "SourceDestCheck": true, + "Tags": [ + { + "Key": "App", + "Value": "Custodian" + } + ], + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default", + "RebootMigration": "default" + }, + "CurrentInstanceBootMode": "legacy-bios", + "NetworkPerformanceOptions": { + "BandwidthWeighting": "default" + }, + "Operator": { + "Managed": false, + "HiddenByDefault": false + }, + "SecondaryInterfaces": [], + "InstanceId": "i-042a559882c4907dc", + "ImageId": "ami-0757b090c88c85bab", + "State": { + "Code": 16, + "Name": "running" + }, + "PrivateDnsName": "ip-10-0-1-200.ec2.internal", + "PublicDnsName": "", + "StateTransitionReason": "", + "AmiLaunchIndex": 0, + "ProductCodes": [], + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "Placement": { + "AvailabilityZoneId": "use1-az2", + "GroupName": "", + "Tenancy": "default", + "AvailabilityZone": "us-east-1c" + }, + "Monitoring": { + "State": "disabled" + }, + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "PrivateIpAddress": "10.0.1.200" + } + ] + }, + { + "ReservationId": "r-0bc85187826350a15", + "OwnerId": "644160558196", + "Groups": [], + "Instances": [ + { + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 54, + "microsecond": 0 + }, + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-039e696c5d3bd6f13", + "EbsCardIndex": 0 + } + } + ], + "ClientToken": "terraform-f8i4majTXcSBEqpc3kEVYJlrjG", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "AttachmentId": "eni-attach-074366427f4cb6808", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "Ipv6Addresses": [], + "MacAddress": "12:68:a3:b5:b6:99", + "NetworkInterfaceId": "eni-086f9944f87e7b575", + "OwnerId": "644160558196", + "PrivateIpAddress": "10.0.1.73", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.0.1.73" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "InterfaceType": "interface", + "Operator": { + "Managed": false + } + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "SourceDestCheck": true, + "Tags": [ + { + "Key": "App", + "Value": "Custodian" + } + ], + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default", + "RebootMigration": "default" + }, + "CurrentInstanceBootMode": "legacy-bios", + "NetworkPerformanceOptions": { + "BandwidthWeighting": "default" + }, + "Operator": { + "Managed": false, + "HiddenByDefault": false + }, + "SecondaryInterfaces": [], + "InstanceId": "i-0e6f711d31e6675f4", + "ImageId": "ami-0757b090c88c85bab", + "State": { + "Code": 16, + "Name": "running" + }, + "PrivateDnsName": "ip-10-0-1-73.ec2.internal", + "PublicDnsName": "", + "StateTransitionReason": "", + "AmiLaunchIndex": 0, + "ProductCodes": [], + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "Placement": { + "AvailabilityZoneId": "use1-az2", + "GroupName": "", + "Tenancy": "default", + "AvailabilityZone": "us-east-1c" + }, + "Monitoring": { + "State": "disabled" + }, + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "PrivateIpAddress": "10.0.1.73" + } + ] + } + ], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstances_2.json b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstances_2.json new file mode 100644 index 00000000000..773b61c1ae8 --- /dev/null +++ b/tests/data/placebo/test_native_tag_skips_gone_instance/ec2.DescribeInstances_2.json @@ -0,0 +1,362 @@ +{ + "status_code": 200, + "data": { + "Reservations": [ + { + "ReservationId": "r-00cf274ff3ee020d9", + "OwnerId": "644160558196", + "Groups": [], + "Instances": [ + { + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 54, + "microsecond": 0 + }, + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-0e9414ef1d2eb432c", + "EbsCardIndex": 0 + } + } + ], + "ClientToken": "terraform-aHfqApEb7WCuXQ32mgMxEZBCP7", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "AttachmentId": "eni-attach-026ad86d2c3a449f5", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "Ipv6Addresses": [], + "MacAddress": "12:cd:c8:af:f4:41", + "NetworkInterfaceId": "eni-0816616bd448027b2", + "OwnerId": "644160558196", + "PrivateIpAddress": "10.0.1.200", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.0.1.200" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "InterfaceType": "interface", + "Operator": { + "Managed": false + } + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "SourceDestCheck": true, + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default", + "RebootMigration": "default" + }, + "CurrentInstanceBootMode": "legacy-bios", + "NetworkPerformanceOptions": { + "BandwidthWeighting": "default" + }, + "Operator": { + "Managed": false, + "HiddenByDefault": false + }, + "SecondaryInterfaces": [], + "InstanceId": "i-042a559882c4907dc", + "ImageId": "ami-0757b090c88c85bab", + "State": { + "Code": 16, + "Name": "running" + }, + "PrivateDnsName": "ip-10-0-1-200.ec2.internal", + "PublicDnsName": "", + "StateTransitionReason": "", + "AmiLaunchIndex": 0, + "ProductCodes": [], + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "Placement": { + "AvailabilityZoneId": "use1-az2", + "GroupName": "", + "Tenancy": "default", + "AvailabilityZone": "us-east-1c" + }, + "Monitoring": { + "State": "disabled" + }, + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "PrivateIpAddress": "10.0.1.200" + } + ] + }, + { + "ReservationId": "r-0bc85187826350a15", + "OwnerId": "644160558196", + "Groups": [], + "Instances": [ + { + "Architecture": "x86_64", + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 54, + "microsecond": 0 + }, + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-039e696c5d3bd6f13", + "EbsCardIndex": 0 + } + } + ], + "ClientToken": "terraform-f8i4majTXcSBEqpc3kEVYJlrjG", + "EbsOptimized": false, + "EnaSupport": true, + "Hypervisor": "xen", + "NetworkInterfaces": [ + { + "Attachment": { + "AttachTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "AttachmentId": "eni-attach-074366427f4cb6808", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Status": "attached", + "NetworkCardIndex": 0 + }, + "Description": "", + "Groups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "Ipv6Addresses": [], + "MacAddress": "12:68:a3:b5:b6:99", + "NetworkInterfaceId": "eni-086f9944f87e7b575", + "OwnerId": "644160558196", + "PrivateIpAddress": "10.0.1.73", + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.0.1.73" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "InterfaceType": "interface", + "Operator": { + "Managed": false + } + } + ], + "RootDeviceName": "/dev/xvda", + "RootDeviceType": "ebs", + "SecurityGroups": [ + { + "GroupId": "sg-0034c1e69b8e6bcc5", + "GroupName": "default" + } + ], + "SourceDestCheck": true, + "VirtualizationType": "hvm", + "CpuOptions": { + "CoreCount": 1, + "ThreadsPerCore": 1 + }, + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open" + }, + "HibernationOptions": { + "Configured": false + }, + "MetadataOptions": { + "State": "applied", + "HttpTokens": "optional", + "HttpPutResponseHopLimit": 1, + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "InstanceMetadataTags": "disabled" + }, + "EnclaveOptions": { + "Enabled": false + }, + "PlatformDetails": "Linux/UNIX", + "UsageOperation": "RunInstances", + "UsageOperationUpdateTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "PrivateDnsNameOptions": { + "HostnameType": "ip-name", + "EnableResourceNameDnsARecord": false, + "EnableResourceNameDnsAAAARecord": false + }, + "MaintenanceOptions": { + "AutoRecovery": "default", + "RebootMigration": "default" + }, + "CurrentInstanceBootMode": "legacy-bios", + "NetworkPerformanceOptions": { + "BandwidthWeighting": "default" + }, + "Operator": { + "Managed": false, + "HiddenByDefault": false + }, + "SecondaryInterfaces": [], + "InstanceId": "i-0e6f711d31e6675f4", + "ImageId": "ami-0757b090c88c85bab", + "State": { + "Code": 16, + "Name": "running" + }, + "PrivateDnsName": "ip-10-0-1-73.ec2.internal", + "PublicDnsName": "", + "StateTransitionReason": "", + "AmiLaunchIndex": 0, + "ProductCodes": [], + "InstanceType": "t2.micro", + "LaunchTime": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 9, + "hour": 14, + "minute": 39, + "second": 53, + "microsecond": 0 + }, + "Placement": { + "AvailabilityZoneId": "use1-az2", + "GroupName": "", + "Tenancy": "default", + "AvailabilityZone": "us-east-1c" + }, + "Monitoring": { + "State": "disabled" + }, + "SubnetId": "subnet-01fd2d48d6f162150", + "VpcId": "vpc-0977a508b64394d7e", + "PrivateIpAddress": "10.0.1.73" + } + ] + } + ], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_universal_tag_skips_gone_snapshot/tagging.TagResources_1.json b/tests/data/placebo/test_universal_tag_skips_gone_snapshot/tagging.TagResources_1.json new file mode 100644 index 00000000000..b3fea6b6c33 --- /dev/null +++ b/tests/data/placebo/test_universal_tag_skips_gone_snapshot/tagging.TagResources_1.json @@ -0,0 +1,13 @@ +{ + "status_code": 200, + "data": { + "FailedResourcesMap": { + "arn:aws:elasticache:us-east-1:644160558196:snapshot:c7n-deleted": { + "StatusCode": 404, + "ErrorCode": "SnapshotNotFoundFault", + "ErrorMessage": "c7n-deleted is either not present or not available" + } + }, + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/terraform/ec2_instance_tag/main.tf b/tests/terraform/ec2_instance_tag/main.tf new file mode 100644 index 00000000000..61796ec1aac --- /dev/null +++ b/tests/terraform/ec2_instance_tag/main.tf @@ -0,0 +1,36 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +provider "aws" {} + +data "aws_ami" "amazon_linux" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["amzn2-ami-hvm-*-x86_64-gp2"] + } +} + +resource "aws_vpc" "example" { + cidr_block = "10.0.0.0/16" +} + +resource "aws_subnet" "example" { + vpc_id = aws_vpc.example.id + cidr_block = "10.0.1.0/24" +} + +# two instances, so a batch that loses one id still has more than one +# survivor to prove the retry keeps all of them +resource "aws_instance" "first" { + ami = data.aws_ami.amazon_linux.id + instance_type = "t2.micro" + subnet_id = aws_subnet.example.id +} + +resource "aws_instance" "second" { + ami = data.aws_ami.amazon_linux.id + instance_type = "t2.micro" + subnet_id = aws_subnet.example.id +} diff --git a/tests/terraform/ec2_instance_tag/tf_resources.json b/tests/terraform/ec2_instance_tag/tf_resources.json new file mode 100644 index 00000000000..06b1ea31101 --- /dev/null +++ b/tests/terraform/ec2_instance_tag/tf_resources.json @@ -0,0 +1,392 @@ +{ + "pytest-terraform": 1, + "outputs": {}, + "resources": { + "aws_ami": { + "amazon_linux": { + "allow_unsafe_filter": null, + "architecture": "x86_64", + "arn": "arn:aws:ec2:us-east-1::image/ami-0757b090c88c85bab", + "block_device_mappings": [ + { + "device_name": "/dev/xvda", + "ebs": { + "delete_on_termination": "true", + "encrypted": "false", + "iops": "0", + "snapshot_id": "snap-0b2af24c3d555bc22", + "throughput": "0", + "volume_initialization_rate": "0", + "volume_size": "8", + "volume_type": "gp2" + }, + "no_device": "", + "virtual_name": "" + } + ], + "boot_mode": "", + "creation_date": "2026-09-08T17:09:15.000Z", + "deprecation_time": "2026-12-07T17:11:00.000Z", + "description": "Amazon Linux 2 AMI 2.0.20260908.0 x86_64 HVM gp2", + "ena_support": true, + "executable_users": null, + "filter": [ + { + "name": "name", + "values": [ + "amzn2-ami-hvm-*-x86_64-gp2" + ] + } + ], + "hypervisor": "xen", + "id": "ami-0757b090c88c85bab", + "image_id": "ami-0757b090c88c85bab", + "image_location": "amazon/amzn2-ami-hvm-2.0.20260908.0-x86_64-gp2", + "image_owner_alias": "amazon", + "image_type": "machine", + "imds_support": "", + "include_deprecated": false, + "kernel_id": "", + "last_launched_time": "", + "most_recent": true, + "name": "amzn2-ami-hvm-2.0.20260908.0-x86_64-gp2", + "name_regex": null, + "owner_id": "644160558196", + "owners": [ + "amazon" + ], + "platform": "", + "platform_details": "Linux/UNIX", + "product_codes": [], + "public": true, + "ramdisk_id": "", + "region": "us-east-1", + "root_device_name": "/dev/xvda", + "root_device_type": "ebs", + "root_snapshot_id": "snap-0b2af24c3d555bc22", + "sriov_net_support": "simple", + "state": "available", + "state_reason": { + "code": "UNSET", + "message": "UNSET" + }, + "tags": {}, + "timeouts": null, + "tpm_support": "", + "uefi_data": null, + "usage_operation": "RunInstances", + "virtualization_type": "hvm" + } + }, + "aws_instance": { + "first": { + "ami": "ami-0757b090c88c85bab", + "arn": "arn:aws:ec2:us-east-1:644160558196:instance/i-0e6f711d31e6675f4", + "associate_public_ip_address": false, + "availability_zone": "us-east-1c", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_options": [ + { + "amd_sev_snp": "", + "core_count": 1, + "nested_virtualization": "", + "threads_per_core": 1 + } + ], + "credit_specification": [ + { + "cpu_credits": "standard" + } + ], + "disable_api_stop": false, + "disable_api_termination": false, + "ebs_block_device": [], + "ebs_optimized": false, + "enable_primary_ipv6": null, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "force_destroy": false, + "get_password_data": false, + "hibernation": false, + "host_id": "", + "host_resource_group_arn": null, + "iam_instance_profile": "", + "id": "i-0e6f711d31e6675f4", + "instance_initiated_shutdown_behavior": "stop", + "instance_lifecycle": "", + "instance_market_options": [], + "instance_state": "running", + "instance_type": "t2.micro", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "", + "launch_template": [], + "maintenance_options": [ + { + "auto_recovery": "default" + } + ], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_protocol_ipv6": "disabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional", + "instance_metadata_tags": "disabled" + } + ], + "monitoring": false, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_group_id": "", + "placement_partition_number": 0, + "primary_network_interface": [ + { + "delete_on_termination": true, + "network_interface_id": "eni-086f9944f87e7b575" + } + ], + "primary_network_interface_id": "eni-086f9944f87e7b575", + "private_dns": "ip-10-0-1-73.ec2.internal", + "private_dns_name_options": [ + { + "enable_resource_name_dns_a_record": false, + "enable_resource_name_dns_aaaa_record": false, + "hostname_type": "ip-name" + } + ], + "private_ip": "10.0.1.73", + "public_dns": "", + "public_ip": "", + "region": "us-east-1", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": true, + "iops": 100, + "kms_key_id": "arn:aws:kms:us-east-1:644160558196:key/04fa09fb-afe3-46c8-ad3c-410b8047f7b9", + "tags": {}, + "tags_all": {}, + "throughput": 0, + "volume_id": "vol-039e696c5d3bd6f13", + "volume_size": 8, + "volume_type": "gp2" + } + ], + "secondary_network_interface": [], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "spot_instance_request_id": "", + "subnet_id": "subnet-01fd2d48d6f162150", + "tags": null, + "tags_all": {}, + "tenancy": "default", + "timeouts": null, + "user_data": null, + "user_data_base64": null, + "user_data_replace_on_change": false, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-0034c1e69b8e6bcc5" + ] + }, + "second": { + "ami": "ami-0757b090c88c85bab", + "arn": "arn:aws:ec2:us-east-1:644160558196:instance/i-042a559882c4907dc", + "associate_public_ip_address": false, + "availability_zone": "us-east-1c", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_options": [ + { + "amd_sev_snp": "", + "core_count": 1, + "nested_virtualization": "", + "threads_per_core": 1 + } + ], + "credit_specification": [ + { + "cpu_credits": "standard" + } + ], + "disable_api_stop": false, + "disable_api_termination": false, + "ebs_block_device": [], + "ebs_optimized": false, + "enable_primary_ipv6": null, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "force_destroy": false, + "get_password_data": false, + "hibernation": false, + "host_id": "", + "host_resource_group_arn": null, + "iam_instance_profile": "", + "id": "i-042a559882c4907dc", + "instance_initiated_shutdown_behavior": "stop", + "instance_lifecycle": "", + "instance_market_options": [], + "instance_state": "running", + "instance_type": "t2.micro", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "", + "launch_template": [], + "maintenance_options": [ + { + "auto_recovery": "default" + } + ], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_protocol_ipv6": "disabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional", + "instance_metadata_tags": "disabled" + } + ], + "monitoring": false, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_group_id": "", + "placement_partition_number": 0, + "primary_network_interface": [ + { + "delete_on_termination": true, + "network_interface_id": "eni-0816616bd448027b2" + } + ], + "primary_network_interface_id": "eni-0816616bd448027b2", + "private_dns": "ip-10-0-1-200.ec2.internal", + "private_dns_name_options": [ + { + "enable_resource_name_dns_a_record": false, + "enable_resource_name_dns_aaaa_record": false, + "hostname_type": "ip-name" + } + ], + "private_ip": "10.0.1.200", + "public_dns": "", + "public_ip": "", + "region": "us-east-1", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": true, + "iops": 100, + "kms_key_id": "arn:aws:kms:us-east-1:644160558196:key/04fa09fb-afe3-46c8-ad3c-410b8047f7b9", + "tags": {}, + "tags_all": {}, + "throughput": 0, + "volume_id": "vol-0e9414ef1d2eb432c", + "volume_size": 8, + "volume_type": "gp2" + } + ], + "secondary_network_interface": [], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "spot_instance_request_id": "", + "subnet_id": "subnet-01fd2d48d6f162150", + "tags": null, + "tags_all": {}, + "tenancy": "default", + "timeouts": null, + "user_data": null, + "user_data_base64": null, + "user_data_replace_on_change": false, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-0034c1e69b8e6bcc5" + ] + } + }, + "aws_subnet": { + "example": { + "arn": "arn:aws:ec2:us-east-1:644160558196:subnet/subnet-01fd2d48d6f162150", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-1c", + "availability_zone_id": "use1-az2", + "cidr_block": "10.0.1.0/24", + "customer_owned_ipv4_pool": "", + "enable_dns64": false, + "enable_lni_at_device_index": 0, + "enable_resource_name_dns_a_record_on_launch": false, + "enable_resource_name_dns_aaaa_record_on_launch": false, + "id": "subnet-01fd2d48d6f162150", + "ipv4_ipam_pool_id": null, + "ipv4_netmask_length": null, + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "ipv6_ipam_pool_id": null, + "ipv6_native": false, + "ipv6_netmask_length": null, + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "644160558196", + "private_dns_hostname_type_on_launch": "ip-name", + "region": "us-east-1", + "tags": null, + "tags_all": {}, + "timeouts": null, + "vpc_id": "vpc-0977a508b64394d7e" + } + }, + "aws_vpc": { + "example": { + "arn": "arn:aws:ec2:us-east-1:644160558196:vpc/vpc-0977a508b64394d7e", + "assign_generated_ipv6_cidr_block": false, + "cidr_block": "10.0.0.0/16", + "default_network_acl_id": "acl-05bc7b1ba43105f96", + "default_route_table_id": "rtb-02919e3e487a62814", + "default_security_group_id": "sg-0034c1e69b8e6bcc5", + "dhcp_options_id": "dopt-05763bf1939671093", + "enable_dns_hostnames": false, + "enable_dns_support": true, + "enable_network_address_usage_metrics": false, + "id": "vpc-0977a508b64394d7e", + "instance_tenancy": "default", + "ipv4_ipam_pool_id": null, + "ipv4_netmask_length": null, + "ipv6_association_id": "", + "ipv6_cidr_block": "", + "ipv6_cidr_block_network_border_group": "", + "ipv6_ipam_pool_id": "", + "ipv6_netmask_length": 0, + "main_route_table_id": "rtb-02919e3e487a62814", + "owner_id": "644160558196", + "region": "us-east-1", + "tags": null, + "tags_all": {} + } + } + } +} \ No newline at end of file diff --git a/tests/test_ebs.py b/tests/test_ebs.py index 2b74aef54d2..7a2abb7e0ff 100644 --- a/tests/test_ebs.py +++ b/tests/test_ebs.py @@ -177,16 +177,6 @@ def test_tag_error(self): tagger.process_resource_set(client, snaps, [{'Key': 'bar', 'Value': 'foo'}]) client.create_tags.assert_called_once() - def test_remove_snapshot(self): - snaps = [{'SnapshotId': 'a'}, {'SnapshotId': 'b'}, {'SnapshotId': 'c'}] - - t1 = list(snaps) - ErrorHandler.remove_snapshot('c', t1) - self.assertEqual([t['SnapshotId'] for t in t1], ['a', 'b']) - - ErrorHandler.remove_snapshot('d', snaps) - self.assertEqual(len(snaps), 3) - def test_get_bad_snapshot_malformed(self): operation_name = "DescribeSnapshots" error_response = { @@ -235,16 +225,6 @@ def test_get_bad_volume_notfound(self): vol = ErrorHandler.extract_bad_volume(e) self.assertEqual(vol, "vol-notfound") - def test_remove_volume(self): - vols = [{'VolumeId': 'a'}, {'VolumeId': 'b'}, {'VolumeId': 'c'}] - - t1 = list(vols) - ErrorHandler.remove_volume('c', t1) - self.assertEqual([t['VolumeId'] for t in t1], ['a', 'b']) - - ErrorHandler.remove_volume('d', vols) - self.assertEqual(len(vols), 3) - def test_volume_tag_error(self): vols = [{'VolumeId': 'vol-aa'}] error_response = { diff --git a/tests/test_eks.py b/tests/test_eks.py index 92b6d86564d..1b4d83bb5ff 100644 --- a/tests/test_eks.py +++ b/tests/test_eks.py @@ -183,6 +183,35 @@ def test_tag_and_remove_tag(self): name='devx')['cluster']['tags'], {'App': 'Custodian'}) + def test_tag_deleted_cluster(self): + # eks raises NotFoundException (not ResourceNotFoundException) when + # a cluster is gone by the time we tag it. No terraform fixture + # here - the cluster's absence is the whole point, so we tag an + # arn that isn't there rather than spend 20m creating a cluster + # to delete it. + aws_region = 'us-east-1' + factory = self.replay_flight_data( + 'test_eks_tag_deleted_cluster', region=aws_region) + + p = self.load_policy({ + 'name': 'eks-tag-deleted', + 'resource': 'aws.eks', + 'actions': [ + {'type': 'tag', 'tags': {'App': 'Custodian'}}, + {'type': 'remove-tag', 'tags': ['Env']}]}, + session_factory=factory, config={'region': aws_region}) + + gone = { + 'name': 'c7n-deleted', + 'arn': 'arn:aws:eks:%s:%s:cluster/c7n-deleted' % ( + aws_region, self.account_id), + 'tags': {'Env': 'Dev'}} + tag, remove_tag = p.resource_manager.actions + + # neither raises, the cluster is simply skipped + tag.process([gone]) + remove_tag.process([gone]) + def test_kms_filter(self): factory = self.replay_flight_data('test_eks_kms_filter') kms = factory().client('kms') diff --git a/tests/test_networkmanager.py b/tests/test_networkmanager.py index 0c0072eac31..bd0b02b0c9f 100644 --- a/tests/test_networkmanager.py +++ b/tests/test_networkmanager.py @@ -203,7 +203,7 @@ def test_mark_for_op_global_network(self): def test_core_network_delete_error(self): invalid_network_id = 'core-network-7a6b617270696e736b69' - factory = self.record_flight_data("test_core_network_delete_error") + factory = self.replay_flight_data("test_core_network_delete_error") client = factory().client("networkmanager") mock_factory = MagicMock() mock_factory.region = 'us-east-1' diff --git a/tests/test_tags.py b/tests/test_tags.py index 5aa746c3b28..91f4974a9c1 100644 --- a/tests/test_tags.py +++ b/tests/test_tags.py @@ -7,6 +7,8 @@ from freezegun import freeze_time from unittest.mock import MagicMock, call +from botocore.exceptions import ClientError + from c7n import tags as tagmod from c7n.tags import universal_retry, coalesce_copy_user_tags from c7n.exceptions import PolicyExecutionError, PolicyValidationError @@ -743,3 +745,183 @@ def test_universal_static_fast_path(self): {"K": "v"}) tag_resources.assert_called_once_with( ResourceARNList=["arn:x"], Tags={"K": "v"}) + + +# a syntactically valid instance id that doesn't exist, so ec2 answers the +# tag call with its real "resource gone" error rather than a malformed id one +GONE_INSTANCE_ID = 'i-0023456789abcdef0' + + +@terraform('ec2_instance_tag') +def test_native_tag_skips_gone_instance(test, ec2_instance_tag): + """A terminated instance in the batch shouldn't cost the rest their tags. + + ec2 fails the whole create_tags call when any resource in it is gone, + so we drop the ids it named and tag the rest. aws.ec2-instance-ami is + the resource from #10999 - it tags via instance id. + """ + aws_region = 'us-east-1' + session_factory = test.replay_flight_data( + 'test_native_tag_skips_gone_instance', region=aws_region) + + instance_ids = [ + ec2_instance_tag['aws_instance.first.id'], + ec2_instance_tag['aws_instance.second.id'], + ] + + p = test.load_policy({ + 'name': 'ec2-instance-ami-tag', + 'resource': 'aws.ec2-instance-ami', + 'actions': [ + {'type': 'tag', 'tags': {'App': 'Custodian'}}, + {'type': 'remove-tag', 'tags': ['App']}]}, + session_factory=session_factory, config={'region': aws_region}) + + resources = [r for r in p.resource_manager.resources() + if r['InstanceId'] in instance_ids] + test.assertEqual(len(resources), 2) + + # the batch ec2 rejects, plus the ones it should still tag + resource_set = resources + [{'InstanceId': GONE_INSTANCE_ID}] + tag, remove_tag = p.resource_manager.actions + + tag.process(resource_set) + client = session_factory().client('ec2') + tags = { + i['InstanceId']: {t['Key']: t['Value'] for t in i.get('Tags', [])} + for r in client.describe_instances( + InstanceIds=instance_ids)['Reservations'] + for i in r['Instances']} + for instance_id in instance_ids: + test.assertEqual(tags[instance_id].get('App'), 'Custodian') + + # and the same salvage on the way back out + remove_tag.id_key = 'InstanceId' + remove_tag.process(resource_set) + tags = { + i['InstanceId']: {t['Key']: t['Value'] for t in i.get('Tags', [])} + for r in client.describe_instances( + InstanceIds=instance_ids)['Reservations'] + for i in r['Instances']} + for instance_id in instance_ids: + test.assertNotIn('App', tags[instance_id]) + + +def test_universal_tag_skips_gone_snapshot(test): + """The rgta path passes through the service's own resource gone fault. + + elasticache reports a deleted snapshot as SnapshotNotFoundFault, which + universal_retry used to raise on - it only skipped + ResourceNotFoundException. No fixture, a snapshot that isn't there is + the point. + """ + aws_region = 'us-east-1' + session_factory = test.replay_flight_data( + 'test_universal_tag_skips_gone_snapshot', region=aws_region) + + p = test.load_policy({ + 'name': 'cache-snapshot-tag', + 'resource': 'aws.cache-snapshot', + 'actions': [{'type': 'tag', 'tags': {'App': 'Custodian'}}]}, + session_factory=session_factory, + config={'region': aws_region, 'account_id': test.account_id}) + + # doesn't raise, the snapshot is skipped + p.resource_manager.actions[0].process([{'SnapshotName': 'c7n-deleted'}]) + + +class ResourceGoneTest(BaseTest): + """Tag actions race with resource deletion, a resource which is gone by + the time we tag it should be skipped, not abort the whole resource set. + + The recorded tests above cover what aws actually answers with. These + cover our own control flow around it, which needs no api. + """ + + def client_error(self, code, message, op='CreateTags'): + return ClientError( + {'Error': {'Code': code, 'Message': message}}, op) + + def get_tag_action(self, name): + p = self.load_policy({ + 'name': 'tag-instances', + 'resource': 'aws.ec2', + 'actions': [ + {'type': 'tag', 'tags': {'Env': 'Dev'}}, + {'type': 'remove-tag', 'tags': ['Env']}]}) + action = {a.type: a for a in p.resource_manager.actions}[name] + action.id_key = 'InstanceId' + return action + + def test_extract_bad_resource_ids(self): + # message shapes here are the ones in our recorded api data, see + # tests/data/placebo - ec2 words them per resource type + for code, message, expected in ( + ('InvalidNetworkInterfaceID.NotFound', + "The networkInterface ID 'eni-d834cdcf' does not exist", + ('eni-d834cdcf',)), + ('InvalidVolume.NotFound', + "The volume 'vol-notfound' does not exist.", + ('vol-notfound',)), + ('InvalidSnapshotID.Malformed', + 'Invalid id: "snap-malformedsnap"', + ('snap-malformedsnap',)), + # not an ec2 bad id error, nothing to skip + ('UnauthorizedOperation', "You are not authorized", ()), + ('SnapshotNotFoundFault', "'snap-1' not found", ())): + self.assertEqual( + tagmod.extract_bad_resource_ids( + self.client_error(code, message)), + expected) + + def test_native_tag_raises_other_errors(self): + action = self.get_tag_action('tag') + client = MagicMock() + client.create_tags.side_effect = self.client_error( + 'UnauthorizedOperation', "You are not authorized") + self.assertRaises( + ClientError, + action.process_resource_set, + client, [{'InstanceId': 'i-1'}], [{'Key': 'Env', 'Value': 'Dev'}]) + + def test_native_tag_raises_on_unknown_bad_id(self): + # we only skip the resources ec2 actually named, an id we don't + # recognize means we don't understand the error - don't swallow it + action = self.get_tag_action('tag') + client = MagicMock() + client.create_tags.side_effect = self.client_error( + 'InvalidInstanceID.NotFound', + "The instance IDs 'i-9' do not exist") + self.assertRaises( + ClientError, + action.process_resource_set, + client, [{'InstanceId': 'i-1'}], [{'Key': 'Env', 'Value': 'Dev'}]) + + def test_universal_retry_skips_gone_and_retries_throttle(self): + # a throttle alongside a skip still retries the throttled arn + self.patch(time, "sleep", MagicMock()) + method = MagicMock() + method.side_effect = [ + {"FailedResourcesMap": { + "arn:abc": {"ErrorCode": "ThrottlingException"}, + "arn:def": {"ErrorCode": "NotFoundException"}}}, + {"Result": 32}, + ] + self.assertEqual( + universal_retry(method, ["arn:abc", "arn:def"]), {"Result": 32}) + self.assertEqual( + method.call_args_list, + [call(ResourceARNList=["arn:abc", "arn:def"]), + call(ResourceARNList=["arn:abc"])]) + + def test_universal_retry_all_skipped_stops(self): + # nothing left to retry, and the api rejects an empty arn list + sleep = MagicMock() + self.patch(time, "sleep", sleep) + method = MagicMock() + method.side_effect = [{ + "FailedResourcesMap": { + "arn:abc": {"ErrorCode": "SnapshotNotFoundFault"}}}] + universal_retry(method, ["arn:abc"]) + method.assert_called_once() + sleep.assert_not_called() diff --git a/tests/zpill.py b/tests/zpill.py index 8b526643b1d..267dd230f57 100644 --- a/tests/zpill.py +++ b/tests/zpill.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import base64 import fnmatch +import functools from io import BytesIO import json import os @@ -17,6 +18,7 @@ from placebo import pill from c7n.testing import CustodianTestCore +from c7n.utils import get_account_id_from_sts # Custodian Test Account. This is used only for testing. @@ -267,6 +269,11 @@ def save_response(self, service, operation, response_data, http_response=200): super(RedPill, self).save_response(service, operation, response_data, http_response) +@functools.lru_cache(maxsize=None) +def get_recording_account_id(): + return get_account_id_from_sts(boto3.Session()) + + class PillTest(CustodianTestCore): archive_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "placebo_data.zip") @@ -276,6 +283,19 @@ class PillTest(CustodianTestCore): recording = False + @property + def account_id(self): + """The account id to construct arns with. + + Recorded api data is sanitized to ACCOUNT_ID, but while recording + (or running functionally) we're talking to a real account and need + its real id, which is what the aws provider's initialization would + have resolved for us. + """ + if not self.recording: + return ACCOUNT_ID + return get_recording_account_id() + def cleanUp(self): self.pill = None From 16f20d030354fe74fcffa82c4010dc3da4ba4a6f Mon Sep 17 00:00:00 2001 From: AJ Kerrigan Date: Mon, 14 Sep 2026 20:58:55 -0400 Subject: [PATCH 2/7] aws - cloudwatch-dashboard - use native cloudwatch apis for tagging (#11090) --------- Co-authored-by: Claude Opus 5 (1M context) --- c7n/resources/cw.py | 94 +++++++++++++++- .../monitoring.ListDashboards_1.json | 23 ++++ .../monitoring.ListDashboards_2.json | 23 ++++ .../monitoring.ListTagsForResource_1.json | 7 ++ .../monitoring.ListTagsForResource_2.json | 16 +++ .../monitoring.ListTagsForResource_3.json | 16 +++ .../monitoring.ListTagsForResource_4.json | 12 +++ .../monitoring.TagResource_1.json | 6 ++ .../monitoring.UntagResource_1.json | 6 ++ .../cloudwatch_dashboard_tag/main.tf | 26 +++++ .../tf_resources.json | 27 +++++ tests/test_cw_dashboard.py | 101 ++++++++++++++++++ 12 files changed, 353 insertions(+), 4 deletions(-) create mode 100644 tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListDashboards_1.json create mode 100644 tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListDashboards_2.json create mode 100644 tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_1.json create mode 100644 tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_2.json create mode 100644 tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_3.json create mode 100644 tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_4.json create mode 100644 tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.TagResource_1.json create mode 100644 tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.UntagResource_1.json create mode 100644 tests/terraform/cloudwatch_dashboard_tag/main.tf create mode 100644 tests/terraform/cloudwatch_dashboard_tag/tf_resources.json create mode 100644 tests/test_cw_dashboard.py diff --git a/c7n/resources/cw.py b/c7n/resources/cw.py index 4bb205aa88f..ac99af30ef2 100644 --- a/c7n/resources/cw.py +++ b/c7n/resources/cw.py @@ -6,7 +6,9 @@ from concurrent.futures import as_completed from datetime import datetime, timedelta -from c7n.actions import BaseAction +from botocore.exceptions import ClientError + +from c7n.actions import AutoTagUser, BaseAction from c7n.exceptions import PolicyValidationError from c7n.filters import Filter, MetricsFilter from c7n.filters.core import parse_date, ValueFilter @@ -17,7 +19,8 @@ QueryResourceManager, TypeInfo, DescribeSource, ConfigSource, DescribeWithResourceTags) from c7n.resolver import ValuesFrom -from c7n.tags import universal_augment +from c7n.tags import ( + RemoveTag, Tag, TagActionFilter, TagDelayedAction, universal_augment) from c7n.utils import type_schema, local_session, chunks, get_retry, jmespath_search @@ -688,6 +691,32 @@ def process(self, resources): logGroupName=r['logGroupName'], **params) +class DescribeDashboard(DescribeSource): + + def augment(self, resources): + # Dashboards aren't supported by the resource groups tagging api, + # fetch their tags from cloudwatch directly. + resources = super().augment(resources) + client = local_session(self.manager.session_factory).client('cloudwatch') + results = [] + for r, arn in zip(resources, self.manager.get_arns(resources)): + try: + tags = self.manager.retry( + client.list_tags_for_resource, ResourceARN=arn).get('Tags', []) + except ClientError as e: + if e.response['Error']['Code'] not in ( + 'ResourceNotFound', 'ResourceNotFoundException'): + raise + # the dashboard was deleted between enumeration and augment + self.manager.log.warning( + "Resource not found: list_tags_for_resource using %s" % { + 'ResourceARN': arn}) + continue + r['Tags'] = tags + results.append(r) + return results + + @resources.register("cloudwatch-dashboard") class CloudWatchDashboard(QueryResourceManager): class resource_type(TypeInfo): @@ -698,14 +727,71 @@ class resource_type(TypeInfo): id = "DashboardName" name = "DashboardName" cfn_type = "AWS::CloudWatch::Dashboard" - universal_taggable = object() global_resource = True + permissions_augment = ("cloudwatch:ListTagsForResource",) source_mapping = { - "describe": DescribeWithResourceTags, + "describe": DescribeDashboard, } +@CloudWatchDashboard.action_registry.register('tag') +class DashboardTag(Tag): + """Add tags to a cloudwatch dashboard + + :example: + + .. code-block:: yaml + + policies: + - name: cloudwatch-dashboard-tag-owner + resource: aws.cloudwatch-dashboard + filters: + - "tag:Owner": absent + actions: + - type: tag + key: Owner + value: platform-team + """ + + permissions = ("cloudwatch:TagResource",) + + def process_resource_set(self, client, resource_set, tags): + for arn in self.manager.get_arns(resource_set): + self.manager.retry(client.tag_resource, ResourceARN=arn, Tags=tags) + + +@CloudWatchDashboard.action_registry.register('remove-tag') +class DashboardRemoveTag(RemoveTag): + """Remove tags from a cloudwatch dashboard + + :example: + + .. code-block:: yaml + + policies: + - name: cloudwatch-dashboard-remove-owner + resource: aws.cloudwatch-dashboard + filters: + - "tag:Owner": present + actions: + - type: remove-tag + tags: [Owner] + """ + + permissions = ("cloudwatch:UntagResource",) + + def process_resource_set(self, client, resource_set, tag_keys): + for arn in self.manager.get_arns(resource_set): + self.manager.retry( + client.untag_resource, ResourceARN=arn, TagKeys=tag_keys) + + +CloudWatchDashboard.filter_registry.register('marked-for-op', TagActionFilter) +CloudWatchDashboard.action_registry.register('mark-for-op', TagDelayedAction) +CloudWatchDashboard.action_registry.register('auto-tag-user', AutoTagUser) + + @resources.register("destination") class Destination(QueryResourceManager): class resource_type(TypeInfo): diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListDashboards_1.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListDashboards_1.json new file mode 100644 index 00000000000..2b5e0228c20 --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListDashboards_1.json @@ -0,0 +1,23 @@ +{ + "status_code": 200, + "data": { + "DashboardEntries": [ + { + "DashboardName": "c7n-test-e3dd19005e27e4f1", + "DashboardArn": "arn:aws:cloudwatch::644160558196:dashboard/c7n-test-e3dd19005e27e4f1", + "LastModified": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 8, + "hour": 22, + "minute": 59, + "second": 16, + "microsecond": 0 + }, + "Size": 158 + } + ], + "ResponseMetadata": {} + } +} diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListDashboards_2.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListDashboards_2.json new file mode 100644 index 00000000000..9280faa7708 --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListDashboards_2.json @@ -0,0 +1,23 @@ +{ + "status_code": 200, + "data": { + "DashboardEntries": [ + { + "DashboardName": "c7n-test-e3dd19005e27e4f1", + "DashboardArn": "arn:aws:cloudwatch::644160558196:dashboard/c7n-test-e3dd19005e27e4f1", + "LastModified": { + "__class__": "datetime", + "year": 2026, + "month": 9, + "day": 8, + "hour": 22, + "minute": 59, + "second": 17, + "microsecond": 0 + }, + "Size": 346 + } + ], + "ResponseMetadata": {} + } +} diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_1.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_1.json new file mode 100644 index 00000000000..e41b16b3e98 --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_1.json @@ -0,0 +1,7 @@ +{ + "status_code": 200, + "data": { + "Tags": [], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_2.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_2.json new file mode 100644 index 00000000000..2ce4832506c --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_2.json @@ -0,0 +1,16 @@ +{ + "status_code": 200, + "data": { + "Tags": [ + { + "Key": "App", + "Value": "Custodian" + }, + { + "Key": "Env", + "Value": "Dev" + } + ], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_3.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_3.json new file mode 100644 index 00000000000..2ce4832506c --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_3.json @@ -0,0 +1,16 @@ +{ + "status_code": 200, + "data": { + "Tags": [ + { + "Key": "App", + "Value": "Custodian" + }, + { + "Key": "Env", + "Value": "Dev" + } + ], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_4.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_4.json new file mode 100644 index 00000000000..954290d0fc1 --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.ListTagsForResource_4.json @@ -0,0 +1,12 @@ +{ + "status_code": 200, + "data": { + "Tags": [ + { + "Key": "App", + "Value": "Custodian" + } + ], + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.TagResource_1.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.TagResource_1.json new file mode 100644 index 00000000000..5b2170a073c --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.TagResource_1.json @@ -0,0 +1,6 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.UntagResource_1.json b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.UntagResource_1.json new file mode 100644 index 00000000000..5b2170a073c --- /dev/null +++ b/tests/data/placebo/test_cloudwatch_dashboard_tag/monitoring.UntagResource_1.json @@ -0,0 +1,6 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": {} + } +} \ No newline at end of file diff --git a/tests/terraform/cloudwatch_dashboard_tag/main.tf b/tests/terraform/cloudwatch_dashboard_tag/main.tf new file mode 100644 index 00000000000..6d48460fe8b --- /dev/null +++ b/tests/terraform/cloudwatch_dashboard_tag/main.tf @@ -0,0 +1,26 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +provider "aws" {} + +resource "random_id" "id" { + byte_length = 8 +} + +resource "aws_cloudwatch_dashboard" "test" { + dashboard_name = "c7n-test-${random_id.id.hex}" + + dashboard_body = jsonencode({ + widgets = [ + { + type = "text" + x = 0 + y = 0 + width = 6 + height = 6 + properties = { + markdown = "cloud custodian test dashboard" + } + } + ] + }) +} diff --git a/tests/terraform/cloudwatch_dashboard_tag/tf_resources.json b/tests/terraform/cloudwatch_dashboard_tag/tf_resources.json new file mode 100644 index 00000000000..61561c8fbc4 --- /dev/null +++ b/tests/terraform/cloudwatch_dashboard_tag/tf_resources.json @@ -0,0 +1,27 @@ +{ + "pytest-terraform": 1, + "outputs": {}, + "resources": { + "aws_cloudwatch_dashboard": { + "test": { + "dashboard_arn": "arn:aws:cloudwatch::644160558196:dashboard/c7n-test-e3dd19005e27e4f1", + "dashboard_body": "{\"widgets\":[{\"height\":6,\"properties\":{\"markdown\":\"cloud custodian test dashboard\"},\"type\":\"text\",\"width\":6,\"x\":0,\"y\":0}]}", + "dashboard_name": "c7n-test-e3dd19005e27e4f1", + "id": "c7n-test-e3dd19005e27e4f1", + "region": "us-east-2" + } + }, + "random_id": { + "id": { + "b64_std": "490ZAF4n5PE=", + "b64_url": "490ZAF4n5PE", + "byte_length": 8, + "dec": "16419307305833063665", + "hex": "e3dd19005e27e4f1", + "id": "490ZAF4n5PE", + "keepers": null, + "prefix": null + } + } + } +} \ No newline at end of file diff --git a/tests/test_cw_dashboard.py b/tests/test_cw_dashboard.py new file mode 100644 index 00000000000..03052c7d6f9 --- /dev/null +++ b/tests/test_cw_dashboard.py @@ -0,0 +1,101 @@ +# Copyright The Cloud Custodian Authors. +# SPDX-License-Identifier: Apache-2.0 +from unittest import mock + +from botocore.exceptions import ClientError +from pytest_terraform import terraform + +from c7n.resources import cw + +from .common import BaseTest + + +@terraform('cloudwatch_dashboard_tag') +def test_cloudwatch_dashboard_tag(test, cloudwatch_dashboard_tag): + # Dashboards are tagged through the native cloudwatch apis, the + # resource groups tagging api rejects their region-less arns. + aws_region = 'us-east-1' + session_factory = test.replay_flight_data( + 'test_cloudwatch_dashboard_tag', region=aws_region) + + dashboard_name = cloudwatch_dashboard_tag[ + 'aws_cloudwatch_dashboard.test.dashboard_name'] + + p = test.load_policy({ + 'name': 'dashboard-tag', + 'resource': 'aws.cloudwatch-dashboard', + 'filters': [{'DashboardName': dashboard_name}], + 'actions': [ + {'type': 'tag', 'tags': {'App': 'Custodian', 'Env': 'Dev'}}]}, + session_factory=session_factory, config={'region': aws_region}) + resources = p.run() + test.assertEqual(len(resources), 1) + + client = session_factory().client('cloudwatch') + arn = resources[0]['DashboardArn'] + test.assertEqual( + {t['Key']: t['Value'] + for t in client.list_tags_for_resource(ResourceARN=arn)['Tags']}, + {'App': 'Custodian', 'Env': 'Dev'}) + + # filtering on the tag we just set exercises the read path - these + # tags aren't visible through the resource groups tagging api. + p = test.load_policy({ + 'name': 'dashboard-remove-tag', + 'resource': 'aws.cloudwatch-dashboard', + 'filters': [ + {'DashboardName': dashboard_name}, + {'tag:App': 'Custodian'}], + 'actions': [{'type': 'remove-tag', 'tags': ['Env']}]}, + session_factory=session_factory, config={'region': aws_region}) + resources = p.run() + test.assertEqual(len(resources), 1) + + test.assertEqual( + client.list_tags_for_resource(ResourceARN=arn)['Tags'], + [{'Key': 'App', 'Value': 'Custodian'}]) + + +def _client_error(code): + return ClientError({'Error': {'Code': code, 'Message': code}}, 'Operation') + + +class CloudWatchDashboardTest(BaseTest): + + def get_source(self): + p = self.load_policy({ + 'name': 'dashboard-augment', + 'resource': 'aws.cloudwatch-dashboard'}) + return p.resource_manager.source + + def test_augment_skips_deleted_dashboard(self): + source = self.get_source() + client = mock.MagicMock() + client.list_tags_for_resource.side_effect = [ + _client_error('ResourceNotFoundException'), + {'Tags': [{'Key': 'App', 'Value': 'Custodian'}]}, + ] + dashboards = [ + {'DashboardName': 'gone', + 'DashboardArn': 'arn:aws:cloudwatch::644160558196:dashboard/gone'}, + {'DashboardName': 'here', + 'DashboardArn': 'arn:aws:cloudwatch::644160558196:dashboard/here'}, + ] + with mock.patch.object(cw, 'local_session') as ls: + ls.return_value.client.return_value = client + resources = source.augment(dashboards) + self.assertEqual( + [r['DashboardName'] for r in resources], ['here']) + self.assertEqual( + resources[0]['Tags'], [{'Key': 'App', 'Value': 'Custodian'}]) + + def test_augment_error(self): + source = self.get_source() + client = mock.MagicMock() + client.list_tags_for_resource.side_effect = _client_error('AccessDeniedException') + with mock.patch.object(cw, 'local_session') as ls: + ls.return_value.client.return_value = client + with self.assertRaises(ClientError): + source.augment([ + {'DashboardName': 'here', + 'DashboardArn': 'arn:aws:cloudwatch::644160558196:dashboard/here'}]) From bf3a18095ddae8b8ebe60689d989ca21477756b2 Mon Sep 17 00:00:00 2001 From: Usman Tahir Qureshi <123512712+UsmanQT@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:02:18 -0400 Subject: [PATCH 3/7] aws - kms, route53 - exclude AWS-managed resources from actions (#11050) - kms-key: AWS managed keys (KeyManager == 'AWS', e.g. the keys behind alias/aws/s3, alias/aws/ebs) reject tag mutations, key policy changes, rotation changes, and ScheduleKeyDeletion - AWS owns their full lifecycle. - resolver-rule: Route 53's global auto-defined rules (e.g. rslvr-autodefined-rr-internet-resolver) have no owning account and reject tag mutations the same way. Fixes #11027 Co-authored-by: Usman Qureshi Co-authored-by: Claude Sonnet 5 --- c7n/resources/kms.py | 63 +++++++++++++++++++++++++++++++--- c7n/resources/route53.py | 40 ++++++++++++++++++++++ tests/test_kms.py | 74 +++++++++++++++++++++++++++++++++++++++- tests/test_route53.py | 35 +++++++++++++++++++ 4 files changed, 206 insertions(+), 6 deletions(-) diff --git a/c7n/resources/kms.py b/c7n/resources/kms.py index 50113a5780f..91bedd1305e 100644 --- a/c7n/resources/kms.py +++ b/c7n/resources/kms.py @@ -14,6 +14,7 @@ from c7n.query import ( ConfigSource, DescribeSource, QueryResourceManager, RetryPageIterator, TypeInfo) from c7n.utils import local_session, type_schema, select_keys +from c7n import tags as tagmod from c7n.tags import universal_augment from .securityhub import PostFinding @@ -136,6 +137,46 @@ def alias_map(self): return alias_map +class SkipAwsManagedKey: + """Mixin for kms-key actions that cannot operate on AWS managed keys + (KeyManager == 'AWS', e.g. the keys backing alias/aws/s3, + alias/aws/ebs, etc). AWS owns the lifecycle, policy, and rotation of + these keys; ScheduleKeyDeletion, key policy changes, rotation + changes, and tag mutations are all rejected by the API for them. + Filters them out before delegating to the real action and emits a + warning so operators know why they were skipped.""" + + def process(self, resources): + excluded = [r for r in resources if r.get('KeyManager') == 'AWS'] + if excluded: + self.manager.log.warning( + "Skipping %d AWS managed key(s) which cannot be modified " + "by any customer account", + len(excluded)) + resources = [r for r in resources if r.get('KeyManager') != 'AWS'] + if not resources: + return + return super().process(resources) + + +@Key.action_registry.register('mark') +@Key.action_registry.register('tag') +class KeyTag(SkipAwsManagedKey, tagmod.UniversalTag): + pass + + +@Key.action_registry.register('unmark') +@Key.action_registry.register('untag') +@Key.action_registry.register('remove-tag') +class KeyRemoveTag(SkipAwsManagedKey, tagmod.UniversalUntag): + pass + + +@Key.action_registry.register('mark-for-op') +class KeyMarkForOp(SkipAwsManagedKey, tagmod.UniversalTagDelayedAction): + pass + + @Key.filter_registry.register('key-rotation-status') class KeyRotationStatus(ValueFilter): """Filters KMS keys by the rotation status @@ -313,7 +354,6 @@ def get_matching_aliases(self, resources, event=None): return matched -@Key.action_registry.register('remove-statements') @KeyAlias.action_registry.register('remove-statements') class RemovePolicyStatement(RemovePolicyBase): """Action to remove policy statements from KMS @@ -380,8 +420,12 @@ def process_resource(self, client, resource, key_id): 'Statements': found} -@Key.action_registry.register('set-rotation') -class KmsKeyRotation(BaseAction): +@Key.action_registry.register('remove-statements') +class KeyRemovePolicyStatement(SkipAwsManagedKey, RemovePolicyStatement): + pass + + +class _KmsKeyRotationBase(BaseAction): """Toggle KMS key rotation :example: @@ -411,6 +455,11 @@ def process(self, keys): client.disable_key_rotation(KeyId=k['KeyId']) +@Key.action_registry.register('set-rotation') +class KmsKeyRotation(SkipAwsManagedKey, _KmsKeyRotationBase): + pass + + @KeyAlias.action_registry.register('post-finding') @Key.action_registry.register('post-finding') class KmsPostFinding(PostFinding): @@ -607,8 +656,7 @@ def get_item_values(self, resource): return [result] -@Key.action_registry.register("schedule-deletion") -class KmsKeyScheduleDeletion(BaseAction): +class _KmsKeyScheduleDeletionBase(BaseAction): """Schedule KMS key deletion If the number of days is not specified, the default value of 30 days is used. @@ -641,3 +689,8 @@ def process(self, keys): client.schedule_key_deletion( KeyId=k["KeyId"], PendingWindowInDays=self.data.get("days", 30) ) + + +@Key.action_registry.register("schedule-deletion") +class KmsKeyScheduleDeletion(SkipAwsManagedKey, _KmsKeyScheduleDeletionBase): + pass diff --git a/c7n/resources/route53.py b/c7n/resources/route53.py index 40eb07d929a..1f37f7b366d 100644 --- a/c7n/resources/route53.py +++ b/c7n/resources/route53.py @@ -616,6 +616,46 @@ class resource_type(TypeInfo): } +class SkipAutoDefinedResolverRule: + """Mixin for ResolverRule actions that cannot operate on Route 53's + global auto-defined rules (e.g. rslvr-autodefined-rr-internet-resolver). + These have no owning account and cannot be tagged or modified by any + customer. Filters them out before delegating to the real action and + emits a warning so operators know why they were skipped.""" + + def process(self, resources): + excluded = [r for r in resources + if r.get('Id', '').startswith('rslvr-autodefined-rr-')] + if excluded: + self.manager.log.warning( + "Skipping %d AWS auto-defined resolver rule(s) which " + "have no owning account and cannot be modified", + len(excluded)) + resources = [r for r in resources + if not r.get('Id', '').startswith('rslvr-autodefined-rr-')] + if not resources: + return + return super().process(resources) + + +@ResolverRule.action_registry.register('mark') +@ResolverRule.action_registry.register('tag') +class ResolverRuleTag(SkipAutoDefinedResolverRule, tags.UniversalTag): + pass + + +@ResolverRule.action_registry.register('unmark') +@ResolverRule.action_registry.register('untag') +@ResolverRule.action_registry.register('remove-tag') +class ResolverRuleRemoveTag(SkipAutoDefinedResolverRule, tags.UniversalUntag): + pass + + +@ResolverRule.action_registry.register('mark-for-op') +class ResolverRuleMarkForOp(SkipAutoDefinedResolverRule, tags.UniversalTagDelayedAction): + pass + + @resources.register('resolver-logs') class ResolverQueryLogConfig(QueryResourceManager): diff --git a/tests/test_kms.py b/tests/test_kms.py index f033e0660c1..0077eab7cb4 100644 --- a/tests/test_kms.py +++ b/tests/test_kms.py @@ -3,7 +3,7 @@ import json import time from datetime import datetime, timedelta -from unittest.mock import patch +from unittest.mock import patch, MagicMock import boto3 import moto @@ -1002,3 +1002,75 @@ def test_schedule_deletion_default_days(self): deletion_date = key_meta["DeletionDate"].astimezone(tzutc()) assert deletion_date > thirty_days_away assert deletion_date < (thirty_days_away + timedelta(days=1)) + + +class KMSSkipAwsManagedKeyTest(BaseTest): + # https://github.com/cloud-custodian/cloud-custodian/issues/11027 + # AWS managed keys (KeyManager == 'AWS') can't have their policy, + # rotation, or tags modified by any customer account - actions + # should skip them rather than let the API call fail. + + aws_key = { + 'KeyId': 'aws-key-1', + 'Arn': 'arn:aws:kms:us-east-1:111111111111:key/aws-key-1', + 'KeyManager': 'AWS', + } + customer_key = { + 'KeyId': 'cust-key-1', + 'Arn': 'arn:aws:kms:us-east-1:111111111111:key/cust-key-1', + 'KeyManager': 'CUSTOMER', + } + + def _policy(self, action, client_name='kms'): + mock_factory = MagicMock() + mock_factory.region = 'us-east-1' + client = mock_factory().client(client_name) + policy = self.load_policy( + {'name': 'k', 'resource': 'kms-key', 'actions': [action]}, + session_factory=mock_factory) + return policy, client + + def test_schedule_deletion_skips_aws_managed(self): + policy, client = self._policy({'type': 'schedule-deletion', 'days': 7}) + policy.resource_manager.actions[0].process([self.aws_key, self.customer_key]) + client.schedule_key_deletion.assert_called_once_with( + KeyId='cust-key-1', PendingWindowInDays=7) + + def test_schedule_deletion_all_aws_managed_noop(self): + policy, client = self._policy({'type': 'schedule-deletion'}) + policy.resource_manager.actions[0].process([self.aws_key]) + client.schedule_key_deletion.assert_not_called() + + def test_set_rotation_skips_aws_managed(self): + policy, client = self._policy({'type': 'set-rotation', 'state': True}) + policy.resource_manager.actions[0].process([self.aws_key, self.customer_key]) + client.enable_key_rotation.assert_called_once_with(KeyId='cust-key-1') + + def test_remove_statements_skips_aws_managed(self): + policy, client = self._policy({'type': 'remove-statements', 'statement_ids': 'matched'}) + policy.resource_manager.actions[0].process([self.aws_key]) + client.get_key_policy.assert_not_called() + + def _tag_policy(self, action): + mock_factory = MagicMock() + mock_factory.region = 'us-east-1' + tag_resources = mock_factory().client('resourcegroupstaggingapi').tag_resources + tag_resources.return_value = {} + policy = self.load_policy( + {'name': 'k', 'resource': 'kms-key', 'actions': [action]}, + session_factory=mock_factory) + return policy, tag_resources + + def test_tag_skips_aws_managed(self): + policy, tag_resources = self._tag_policy( + {'type': 'tag', 'tags': {'Owner': 'platform'}}) + policy.resource_manager.actions[0].process([self.aws_key, self.customer_key]) + tag_resources.assert_called_once_with( + ResourceARNList=['arn:aws:kms:us-east-1:111111111111:key/cust-key-1'], + Tags={'Owner': 'platform'}) + + def test_tag_all_aws_managed_noop(self): + policy, tag_resources = self._tag_policy( + {'type': 'tag', 'tags': {'Owner': 'platform'}}) + policy.resource_manager.actions[0].process([self.aws_key]) + tag_resources.assert_not_called() diff --git a/tests/test_route53.py b/tests/test_route53.py index 66c65acbeda..2771aa0e120 100644 --- a/tests/test_route53.py +++ b/tests/test_route53.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import time import logging +from unittest.mock import MagicMock import pytest from pytest_terraform import terraform @@ -698,3 +699,37 @@ def test_resolver_rule_query(self): resources = p.run() self.assertEqual(len(resources), 1) self.assertEqual(resources[0]["RuleType"], "FORWARD") + + # https://github.com/cloud-custodian/cloud-custodian/issues/11027 + # AWS's global auto-defined resolver rules (e.g. + # rslvr-autodefined-rr-internet-resolver) have no owning account and + # can't be tagged/modified by any customer - actions should skip them. + + auto_defined_rule = {'Id': 'rslvr-autodefined-rr-internet-resolver'} + customer_rule = {'Id': 'rslvr-rr-real1'} + + def _tag_policy(self, action): + mock_factory = MagicMock() + mock_factory.region = 'us-east-1' + tag_resources = mock_factory().client('resourcegroupstaggingapi').tag_resources + tag_resources.return_value = {} + policy = self.load_policy( + {'name': 'r', 'resource': 'resolver-rule', 'actions': [action]}, + session_factory=mock_factory) + return policy, tag_resources + + def test_tag_skips_auto_defined_rule(self): + policy, tag_resources = self._tag_policy( + {'type': 'tag', 'tags': {'Owner': 'platform'}}) + policy.resource_manager.actions[0].process( + [self.auto_defined_rule, self.customer_rule]) + tag_resources.assert_called_once() + arns = tag_resources.call_args.kwargs['ResourceARNList'] + self.assertEqual(len(arns), 1) + self.assertIn('rslvr-rr-real1', arns[0]) + + def test_tag_all_auto_defined_noop(self): + policy, tag_resources = self._tag_policy( + {'type': 'tag', 'tags': {'Owner': 'platform'}}) + policy.resource_manager.actions[0].process([self.auto_defined_rule]) + tag_resources.assert_not_called() From 459879a591b5cb80229ef0abec306f5852955fcf Mon Sep 17 00:00:00 2001 From: AJ Kerrigan Date: Fri, 4 Sep 2026 10:11:22 -0400 Subject: [PATCH 4/7] tags - standardize dynamic tagging support (#10968) * Centralize dynamic tag lookup logic where possible * Extend the "default-value with no key" pattern from AWS to GCP & Azure * Fix aws.asg-specific tag logic, and extract shared behavior between ASG & generic tag logic into a mixin --- c7n/lookup.py | 79 ++++++-- c7n/resources/asg.py | 92 ++++++--- c7n/tags.py | 161 ++++++++-------- tests/test_asg.py | 174 +++++++++++++++++- tests/test_lookup.py | 126 ++++++++++++- tests/test_tags.py | 56 +++++- tools/c7n_azure/c7n_azure/actions/tagging.py | 51 +++-- .../c7n_azure/tests_azure/test_actions_tag.py | 172 +++++++++++++---- tools/c7n_gcp/c7n_gcp/actions/labels.py | 21 ++- tools/c7n_gcp/tests/test_gcp_labels.py | 72 ++++++++ 10 files changed, 832 insertions(+), 172 deletions(-) diff --git a/c7n/lookup.py b/c7n/lookup.py index 123d5659e4b..46cf6d0aa21 100644 --- a/c7n/lookup.py +++ b/c7n/lookup.py @@ -6,28 +6,52 @@ import copy +class Skip: + """Sentinel returned by Lookup.resolve_value to omit a value entirely.""" + + __slots__ = () + + def __repr__(self): + return 'Lookup.SKIP' + + class Lookup: RESOURCE_SOURCE = 'resource' + SKIP = Skip() + + # A lookup needs a key, a fallback, or both. With a key it resolves against + # the resource and falls back to 'default-value' when the path yields + # nothing. With no key it is just the fallback -- which callers that pass a + # name to resolve_value treat as a conditional default, writing it only + # when that name is not already present on the resource. schema = { 'type': 'object', - 'oneOf': [ - { - 'properties': { - 'type': {'type': 'string', 'enum': [RESOURCE_SOURCE]}, - 'key': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['type', 'key'] - } + 'properties': { + 'type': {'type': 'string', 'enum': [RESOURCE_SOURCE]}, + 'key': {'type': 'string'}, + 'default-value': {}, + }, + 'additionalProperties': False, + 'required': ['type'], + 'anyOf': [ + {'required': ['key']}, + {'required': ['default-value']}, ] } @staticmethod - def lookup_type(schema): + def lookup_type(schema, default_schema=None): + """Schema for a value that may be given directly or looked up. + + ``default_schema`` types the lookup's fallback, and defaults to + ``schema``. Pass it when the fallback is narrower than the value -- + tag values, for instance, accept unquoted YAML scalars for backwards + compatibility, but a fallback is always written as a string. + """ lookup_schema = copy.deepcopy(Lookup.schema) - for lookup in lookup_schema['oneOf']: - lookup['properties']['default-value'] = schema + lookup_schema['properties']['default-value'] = ( + schema if default_schema is None else default_schema) return { 'oneOf': [ @@ -36,6 +60,30 @@ def lookup_type(schema): ] } + @staticmethod + def resolve_value(spec, resource, name=None, current_names=()): + """Resolve a value spec against a resource. + + Handles both lookup forms and plain scalars: + + - a scalar is returned unchanged + - ``{type: resource, key: }`` resolves against the resource, + falling back to ``default-value`` when the path yields nothing + - ``{type: resource, default-value: }`` (no key) is a conditional + default: it resolves to ``x`` only when ``name`` is absent from + ``current_names``, and to ``Lookup.SKIP`` otherwise. + """ + if not Lookup.is_lookup(spec): + return spec + if 'key' not in spec and name in current_names: + return Lookup.SKIP + return Lookup.extract(spec, resource) + + @staticmethod + def has_lookups(spec_map): + """True if any value in the mapping is a lookup rather than a scalar.""" + return any(Lookup.is_lookup(v) for v in spec_map.values()) + @staticmethod def extract(source, data=None): if Lookup.is_lookup(source): @@ -54,10 +102,11 @@ def get_value(source, data=None): @staticmethod def get_value_from_resource(source, resource): - value = jmespath_search(source['key'], resource) + if 'key' in source: + value = jmespath_search(source['key'], resource) + if value is not None: + return value - if value is not None: - return value if 'default-value' not in source: raise Exception('Lookup for key, {}, returned None'.format(source['key'])) else: diff --git a/c7n/resources/asg.py b/c7n/resources/asg.py index ca18a28e1b4..c6fc0bfcf61 100644 --- a/c7n/resources/asg.py +++ b/c7n/resources/asg.py @@ -20,9 +20,11 @@ from c7n.manager import resources from c7n import query from c7n.resources.securityhub import PostFinding -from c7n.tags import TagActionFilter, DEFAULT_TAG, TagCountFilter, TagTrim, TagDelayedAction +from c7n.tags import ( + TagActionFilter, DEFAULT_TAG, TagCountFilter, TagTrim, TagDelayedAction, + TagValueResolver, has_dynamic_tag_values, tag_value_schema) from c7n.utils import ( - FormatDate, local_session, type_schema, chunks, get_retry, select_keys) + local_session, type_schema, chunks, get_retry, select_keys) from .ec2 import deserialize_user_data @@ -1104,7 +1106,7 @@ def process_resource_set(self, client, asgs, tags): @ASG.action_registry.register('tag') @ASG.action_registry.register('mark') -class Tag(Action): +class Tag(TagValueResolver, Action): """Action to add a tag to an ASG The *propagate* parameter can be used to specify that the tag being added @@ -1125,13 +1127,32 @@ class Tag(Action): key: OwnerName value: OwnerName propagate: true + + Tag values may also be looked up per-asg from the asg's attributes: + + .. code-block:: yaml + + policies: + - name: asg-tag-owner + resource: asg + actions: + - type: tag + propagate: true + tags: + Owner: + type: resource + key: "Tags[?Key=='Team'].Value | [0]" # from the Team tag + default-value: unknown + CostCenter: + type: resource + default-value: unassigned # set only if CostCenter absent """ schema = type_schema( 'tag', key={'type': 'string'}, - value={'type': 'string'}, - tags={'type': 'object'}, + value=tag_value_schema(), + tags={'type': 'object', 'additionalProperties': tag_value_schema()}, # Backwards compatibility tag={'type': 'string'}, msg={'type': 'string'}, @@ -1141,34 +1162,57 @@ class Tag(Action): permissions = ('autoscaling:CreateOrUpdateTags',) batch_size = 1 - def get_tag_set(self): - tags = [] + def get_tag_spec_map(self): + spec_map = {} + tags = self.data.get('tags', {}) key = self.data.get('key', self.data.get('tag', DEFAULT_TAG)) - value = self.data.get( - 'value', self.data.get( - 'msg', 'AutoScaleGroup does not meet policy guidelines')) + value = self.data.get('value', self.data.get('msg')) + # The mark default only applies to the single-tag form. A policy that + # spells out a tags mapping gets that mapping and nothing else -- + # otherwise every asg picks up a marker tag it never asked for, and a + # conditional default can never resolve to an empty payload. + if value is None and not tags: + value = 'AutoScaleGroup does not meet policy guidelines' if key and value: - tags.append({'Key': key, 'Value': value}) + spec_map[key] = value + + spec_map.update(tags) - for k, v in self.data.get('tags', {}).items(): - tags.append({'Key': k, 'Value': v}) + return spec_map - return tags + def resolve_tag_sets(self, asgs, spec_map): + """Yield (asg_set, tags) pairs to submit. + + Static values are shared by every asg, so they go out in batches. + Dynamic values are resolved per asg and then regrouped, so asgs that + land on the same payload still batch together, and an asg whose tags + all resolve to skip drops out entirely. + """ + if not has_dynamic_tag_values(spec_map): + tags = [{'Key': k, 'Value': v} for k, v in spec_map.items()] + self.interpolate_values(tags) + for asg_set in chunks(asgs, self.batch_size): + yield asg_set, tags + return + + for asg_set, resolved in self.resolve_and_group(asgs, spec_map): + tags = [{'Key': k, 'Value': v} for k, v in resolved.items()] + for chunk in chunks(asg_set, self.batch_size): + yield chunk, tags def process(self, asgs): - tags = self.get_tag_set() + spec_map = self.get_tag_spec_map() error = None - self.interpolate_values(tags) - client = self.get_client() with self.executor_factory(max_workers=2) as w: futures = {} - for asg_set in chunks(asgs, self.batch_size): + for asg_set, tags in self.resolve_tag_sets(asgs, spec_map): futures[w.submit( - self.process_resource_set, client, asg_set, tags)] = asg_set + self.process_resource_set, client, asg_set, tags)] = ( + asg_set, tags) for f in as_completed(futures): - asg_set = futures[f] + asg_set, tags = futures[f] if f.exception(): self.log.exception( "Exception tagging tag:%s error:%s asg:%s" % ( @@ -1194,14 +1238,6 @@ def process_resource_set(self, client, asgs, tags): a.setdefault('Tags', []).append(atags) self.manager.retry(client.create_or_update_tags, Tags=tag_params) - def interpolate_values(self, tags): - params = { - 'account_id': self.manager.config.account_id, - 'now': FormatDate.utcnow(), - 'region': self.manager.config.region} - for t in tags: - t['Value'] = t['Value'].format(**params) - def get_client(self): return local_session(self.manager.session_factory).client('autoscaling') diff --git a/c7n/tags.py b/c7n/tags.py index 9e78f70a327..753fb594327 100644 --- a/c7n/tags.py +++ b/c7n/tags.py @@ -82,7 +82,7 @@ def retry_skip_bad_ids(retry, method, resource_ids, log, **kw): # Sentinel: the resolver returns this to omit a tag from a resource's payload. -TAG_VALUE_SKIP = object() +TAG_VALUE_SKIP = Lookup.SKIP def tag_value_schema(): @@ -90,35 +90,18 @@ def tag_value_schema(): Accepts a scalar, a resource lookup by key (with optional fallback), or a conditional default with no key (write only when the tag is absent). + + Tag values are strings, but unquoted YAML scalars have always been + accepted here and coerced on the way out, so the scalar form stays wide. + A fallback has no such history and is held to a string. """ - return { - 'oneOf': [ - {'type': ['string', 'number', 'boolean']}, - { - 'type': 'object', - 'additionalProperties': False, - 'required': ['type', 'key'], - 'properties': { - 'type': {'enum': [Lookup.RESOURCE_SOURCE]}, - 'key': {'type': 'string'}, - 'default-value': {'type': 'string'}, - }, - }, - { - 'type': 'object', - 'additionalProperties': False, - 'required': ['type', 'default-value'], - 'properties': { - 'type': {'enum': [Lookup.RESOURCE_SOURCE]}, - 'default-value': {'type': 'string'}, - }, - }, - ] - } + return Lookup.lookup_type( + {'type': ['string', 'number', 'boolean']}, + default_schema={'type': 'string'}) def has_dynamic_tag_values(spec_map): - return any(Lookup.is_lookup(v) for v in spec_map.values()) + return Lookup.has_lookups(spec_map) def resource_tag_keys(resource): @@ -137,14 +120,76 @@ def resolve_tag_value(spec, tag_name, resource, current_tag_keys): Returns the resolved value, or TAG_VALUE_SKIP to omit the tag. """ - if not Lookup.is_lookup(spec): - return spec - if 'key' in spec: - return Lookup.extract(spec, resource) - # Conditional default (no key): write only if the tag is absent. - if tag_name in current_tag_keys: - return TAG_VALUE_SKIP - return spec['default-value'] + return Lookup.resolve_value(spec, resource, tag_name, current_tag_keys) + + +class TagValueResolver: + """Turn a mapping of tag value specs into concrete tags for a resource. + + A mixin rather than part of Tag itself because the asg tag action isn't a + Tag subclass, but resolves the same per-resource lookups and writes the + same {account_id}/{now}/{region} placeholders. + """ + + def get_interpolation_params(self): + """Placeholder values shared by every tag in a run. + + None of these depend on the resource being tagged, so a caller + resolving values per resource builds them once and passes them back + in rather than re-deriving a fresh {now} for each one. + """ + return { + 'account_id': self.manager.config.account_id, + 'now': utils.FormatDate.utcnow(), + 'region': self.manager.config.region} + + def resolve_and_group(self, resources, spec_map): + """Resolve tag values per resource and group by identical payload. + + Returns a list of (resource_set, resolved_dict). Resources whose + payload is empty (all tags conditionally skipped) are dropped. + """ + groups = {} + # {now} and friends don't vary per resource, so fix them once for the + # whole run -- otherwise a large resource set drifts across seconds and + # splits into a group per timestamp. + params = self.get_interpolation_params() + for r in resources: + resolved = self.resolve_resource_tags(spec_map, r, params) + if not resolved: + continue + sig = tuple(sorted(resolved.items())) + groups.setdefault(sig, ([], resolved))[0].append(r) + return list(groups.values()) + + def resolve_resource_tags(self, spec_map, resource, params): + """Resolve a spec mapping against one resource. + + Returns {name: value} of the tags to write - a conditional default the + resource already carries is left out, so the mapping comes back empty + when there's nothing to do for this resource. + """ + current = resource_tag_keys(resource) + resolved = {} + for name, spec in spec_map.items(): + value = resolve_tag_value(spec, name, resource, current) + if value is TAG_VALUE_SKIP: + continue + resolved[name] = self.interpolate_single_value(value, params) + return resolved + + def interpolate_single_value(self, value, params=None): + """Interpolate placeholders in a single tag value.""" + if params is None: + params = self.get_interpolation_params() + return str(value).format(**params) + + def interpolate_values(self, tags, params=None): + """Interpolate in a list of tags - 'old' ec2 format""" + if params is None: + params = self.get_interpolation_params() + for t in tags: + t['Value'] = self.interpolate_single_value(t['Value'], params) def register_ec2_tags(filters, actions): @@ -476,7 +521,7 @@ def __call__(self, i): return op(tag_count, count) -class Tag(Action): +class Tag(TagValueResolver, Action): """Tag an ec2 resource. Tag values may be looked up per-resource from resource attributes: @@ -551,33 +596,12 @@ def process(self, resources): self.process_resource_set, self.id_key, resources, tags, self.log) return - for resource_set, resolved in self._resolve_and_group(resources, spec_map): + for resource_set, resolved in self.resolve_and_group(resources, spec_map): tags = [{'Key': k, 'Value': v} for k, v in resolved.items()] _common_tag_processer( self.executor_factory, batch_size, self.concurrency, client, self.process_resource_set, self.id_key, resource_set, tags, self.log) - def _resolve_and_group(self, resources, spec_map): - """Resolve tag values per resource and group by identical payload. - - Returns a list of (resource_set, resolved_dict). Resources whose - payload is empty (all tags conditionally skipped) are dropped. - """ - groups = {} - for r in resources: - current = resource_tag_keys(r) - resolved = {} - for name, spec in spec_map.items(): - value = resolve_tag_value(spec, name, r, current) - if value is TAG_VALUE_SKIP: - continue - resolved[name] = self.interpolate_single_value(value) - if not resolved: - continue - sig = tuple(sorted(resolved.items())) - groups.setdefault(sig, ([], resolved))[0].append(r) - return list(groups.values()) - def process_resource_set(self, client, resource_set, tags): mid = self.manager.get_model().id retry_skip_bad_ids( @@ -586,21 +610,6 @@ def process_resource_set(self, client, resource_set, tags): Tags=tags, DryRun=self.manager.config.dryrun) - def interpolate_single_value(self, tag): - """Interpolate in a single tag value. - """ - params = { - 'account_id': self.manager.config.account_id, - 'now': utils.FormatDate.utcnow(), - 'region': self.manager.config.region} - return str(tag).format(**params) - - def interpolate_values(self, tags): - """Interpolate in a list of tags - 'old' ec2 format - """ - for t in tags: - t['Value'] = self.interpolate_single_value(t['Value']) - def get_client(self): return utils.local_session(self.manager.session_factory).client( self.manager.resource_type.service) @@ -1038,7 +1047,7 @@ def process(self, resources): self.process_resource_set, self.id_key, resources, tags, self.log) return - for resource_set, resolved in self._resolve_and_group(resources, spec_map): + for resource_set, resolved in self.resolve_and_group(resources, spec_map): _common_tag_processer( self.executor_factory, batch_size, self.concurrency, client, self.process_resource_set, self.id_key, resource_set, resolved, @@ -1051,11 +1060,13 @@ def process_resource_set(self, client, resource_set, tags): return universal_retry( client.tag_resources, ResourceARNList=arns, Tags=tags) - def interpolate_values(self, tags): + def interpolate_values(self, tags, params=None): """Interpolate in a list of tags - 'new' resourcegroupstaggingapi format """ + if params is None: + params = self.get_interpolation_params() for key in list(tags.keys()): - tags[key] = self.interpolate_single_value(tags[key]) + tags[key] = self.interpolate_single_value(tags[key], params) def get_client(self): # For global resources, manage tags from us-east-1 diff --git a/tests/test_asg.py b/tests/test_asg.py index a03374051a2..76eb35a5e36 100644 --- a/tests/test_asg.py +++ b/tests/test_asg.py @@ -11,9 +11,9 @@ from .common import BaseTest from c7n.exceptions import PolicyValidationError -from c7n.resources.asg import LaunchInfo +from c7n.resources.asg import LaunchInfo, Tag as AsgTag from c7n.resources.aws import shape_validate -from c7n.utils import jmespath_search +from c7n.utils import FormatDate, jmespath_search class LaunchConfigTest(BaseTest): @@ -1178,3 +1178,173 @@ def test_asg_scaling_policy_enabled(self): resources = p.run() self.assertEqual(len(resources), 1) self.assertTrue(resources[0].get('Enabled')) + + +class AsgDynamicTagTest(BaseTest): + """Dynamic tag values on the asg-specific tag action. + + The asg tag action is not a c7n.tags.Tag subclass, so it needs its own + wiring for the resource lookup form of a tag value. + """ + + def _run(self, asgs, action): + mock_factory = mock.MagicMock() + mock_factory.region = 'us-east-1' + create_tags = mock_factory().client('autoscaling').create_or_update_tags + create_tags.return_value = {} + policy = self.load_policy( + {"name": "d", "resource": "asg", "actions": [action]}, + session_factory=mock_factory) + policy.resource_manager.actions[0].process(asgs) + return create_tags + + def test_lookup_key_hit(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-1", + "Tags": [{"Key": "Team", "Value": "platform"}]}], + {"type": "tag", "tags": {"Owner": { + "type": "resource", "key": "Tags[?Key=='Team'].Value | [0]"}}}) + create_tags.assert_called_once_with( + Tags=[{"Key": "Owner", "Value": "platform", + "PropagateAtLaunch": False, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-1"}]) + + def test_lookup_key_miss_uses_default(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-1"}], + {"type": "tag", "tags": {"Owner": { + "type": "resource", "key": "Nope", "default-value": "unknown"}}}) + create_tags.assert_called_once_with( + Tags=[{"Key": "Owner", "Value": "unknown", + "PropagateAtLaunch": False, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-1"}]) + + def test_conditional_default_skips_asg_with_tag_present(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-has", + "Tags": [{"Key": "Owner", "Value": "keep-me"}]}], + {"type": "tag", "tags": {"Owner": { + "type": "resource", "default-value": "unassigned"}}}) + create_tags.assert_not_called() + + def test_conditional_default_writes_per_asg(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-has", + "Tags": [{"Key": "Owner", "Value": "keep-me"}]}, + {"AutoScalingGroupName": "asg-missing"}], + {"type": "tag", "tags": {"Owner": { + "type": "resource", "default-value": "unassigned"}}}) + create_tags.assert_called_once_with( + Tags=[{"Key": "Owner", "Value": "unassigned", + "PropagateAtLaunch": False, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-missing"}]) + + def test_lookup_in_single_value_form(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-1", + "Tags": [{"Key": "Team", "Value": "platform"}]}], + {"type": "tag", "key": "Owner", "propagate": True, + "value": {"type": "resource", "key": "Tags[?Key=='Team'].Value | [0]"}}) + create_tags.assert_called_once_with( + Tags=[{"Key": "Owner", "Value": "platform", + "PropagateAtLaunch": True, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-1"}]) + + def test_placeholder_interpolated_in_lookup_default(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-1"}], + {"type": "tag", "tags": {"Stamp": { + "type": "resource", "key": "Nope", + "default-value": "made-in-{region}"}}}) + create_tags.assert_called_once_with( + Tags=[{"Key": "Stamp", "Value": "made-in-us-east-1", + "PropagateAtLaunch": False, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-1"}]) + + def test_dynamic_values_batch_by_payload(self): + # batch_size is 1 today, so raise it to show the dynamic path batches + # asgs that resolve to the same payload rather than going one at a time + asgs = [{"AutoScalingGroupName": "asg-1", + "Tags": [{"Key": "Env", "Value": "prod"}]}, + {"AutoScalingGroupName": "asg-2", + "Tags": [{"Key": "Env", "Value": "dev"}]}, + {"AutoScalingGroupName": "asg-3", + "Tags": [{"Key": "Env", "Value": "prod"}]}] + with mock.patch.object(AsgTag, "batch_size", 2): + create_tags = self._run( + asgs, + {"type": "tag", "tags": {"Tier": { + "type": "resource", "key": "Tags[?Key=='Env'].Value | [0]"}}}) + self.assertEqual(create_tags.call_count, 2) + by_value = {c.kwargs["Tags"][0]["Value"]: {t["ResourceId"] for t in c.kwargs["Tags"]} + for c in create_tags.call_args_list} + self.assertEqual(by_value["prod"], {"asg-1", "asg-3"}) + self.assertEqual(by_value["dev"], {"asg-2"}) + + def test_now_resolved_once_for_the_whole_run(self): + # a clock that moves on every read - the run should only read it once, + # so both asgs get stamped alike even though batch_size sends them + # in separate calls + ticks = [FormatDate(datetime(2022, 6, 27, 12, 34, 56)), + FormatDate(datetime(2022, 6, 27, 12, 34, 57))] + with mock.patch.object(FormatDate, "utcnow", side_effect=ticks) as utcnow: + create_tags = self._run( + [{"AutoScalingGroupName": "asg-1"}, + {"AutoScalingGroupName": "asg-2"}], + {"type": "tag", "tags": {"Stamp": { + "type": "resource", "key": "Nope", + "default-value": "at-{now}"}}}) + self.assertEqual(utcnow.call_count, 1) + stamped = [c.kwargs["Tags"][0]["Value"] for c in create_tags.call_args_list] + self.assertEqual(stamped, ["at-2022-06-27 12:34:56"] * 2) + + def test_malformed_tag_value_rejected(self): + with self.assertRaises(PolicyValidationError): + self.load_policy( + {"name": "d", "resource": "asg", + "actions": [{"type": "tag", "tags": {"Owner": {"foo": "bar"}}}]}, + validate=True) + + def test_tags_mapping_does_not_add_default_marker(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-1"}], + {"type": "tag", "tags": {"Owner": "platform"}}) + create_tags.assert_called_once_with( + Tags=[{"Key": "Owner", "Value": "platform", + "PropagateAtLaunch": False, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-1"}]) + + def test_bare_mark_still_writes_default_marker(self): + create_tags = self._run( + [{"AutoScalingGroupName": "asg-1"}], {"type": "mark"}) + create_tags.assert_called_once_with( + Tags=[{"Key": "maid_status", + "Value": "AutoScaleGroup does not meet policy guidelines", + "PropagateAtLaunch": False, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-1"}]) + + def test_auto_tag_user_does_not_add_default_marker(self): + mock_factory = mock.MagicMock() + mock_factory.region = 'us-east-1' + create_tags = mock_factory().client('autoscaling').create_or_update_tags + create_tags.return_value = {} + policy = self.load_policy( + {"name": "d", "resource": "asg", + "mode": {"type": "cloudtrail", "events": ["CreateAutoScalingGroup"]}, + "actions": [{"type": "auto-tag-user", "tag": "CreatorName"}]}, + session_factory=mock_factory) + action = policy.resource_manager.actions[0] + action.set_resource_tags( + {"CreatorName": "alice"}, [{"AutoScalingGroupName": "asg-1"}]) + create_tags.assert_called_once_with( + Tags=[{"Key": "CreatorName", "Value": "alice", + "PropagateAtLaunch": False, + "ResourceType": "auto-scaling-group", + "ResourceId": "asg-1"}]) diff --git a/tests/test_lookup.py b/tests/test_lookup.py index 6033633b989..c5649d09d15 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -2,6 +2,8 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 +import jsonschema + from .common import BaseTest from c7n.lookup import Lookup @@ -18,12 +20,12 @@ def test_lookup_type(self): self.assertEqual(number_schema, lookup_default_number['oneOf'][1]) self.assertEqual(number_schema, - lookup_default_number['oneOf'][0]['oneOf'][0] + lookup_default_number['oneOf'][0] ['properties']['default-value']) self.assertEqual(string_schema, lookup_default_string['oneOf'][1]) self.assertEqual(string_schema, - lookup_default_string['oneOf'][0]['oneOf'][0] + lookup_default_string['oneOf'][0] ['properties']['default-value']) def test_extract_no_lookup(self): @@ -76,6 +78,15 @@ def test_get_value_from_resource_value_not_exists(self): value = Lookup.get_value_from_resource(source, resource) self.assertEqual(value, 'value_2') + def test_extract_lookup_without_key_returns_default(self): + source = {'type': Lookup.RESOURCE_SOURCE, 'default-value': 'value_2'} + self.assertEqual(Lookup.extract(source, {'name': 'x'}), 'value_2') + + def test_get_value_from_resource_without_key_returns_default(self): + source = {'type': Lookup.RESOURCE_SOURCE, 'default-value': 'value_2'} + self.assertEqual( + Lookup.get_value_from_resource(source, {'name': 'x'}), 'value_2') + def test_get_value_from_resource_value_not_exists_exception(self): resource = { 'field_level_1': { @@ -89,3 +100,114 @@ def test_get_value_from_resource_value_not_exists_exception(self): with self.assertRaises(Exception): Lookup.get_value_from_resource(source, resource) + + +class LookupSchemaTest(BaseTest): + """A lookup needs a key, a fallback, or both.""" + + string_schema = {'type': 'string'} + + def validates(self, doc, schema=None): + try: + jsonschema.validate(doc, schema or Lookup.lookup_type(self.string_schema)) + return True + except jsonschema.ValidationError: + return False + + def test_key_only_accepted(self): + self.assertTrue(self.validates({'type': 'resource', 'key': 'name'})) + + def test_key_with_default_accepted(self): + self.assertTrue( + self.validates( + {'type': 'resource', 'key': 'name', 'default-value': 'x'})) + + def test_default_only_accepted(self): + self.assertTrue(self.validates({'type': 'resource', 'default-value': 'x'})) + + def test_scalar_accepted(self): + self.assertTrue(self.validates('plain')) + + def test_neither_key_nor_default_rejected(self): + self.assertFalse(self.validates({'type': 'resource'})) + + def test_unknown_property_rejected(self): + self.assertFalse(self.validates({'type': 'resource', 'ky': 'name'})) + + def test_non_lookup_dict_rejected(self): + self.assertFalse(self.validates({'foo': 'bar'})) + + def test_default_value_is_typed_by_caller(self): + self.assertFalse( + self.validates({'type': 'resource', 'default-value': 5})) + self.assertTrue( + self.validates({'type': 'resource', 'default-value': 5}, + Lookup.lookup_type({'type': 'number'}))) + + def test_default_value_may_be_typed_separately_from_the_value(self): + schema = Lookup.lookup_type( + {'type': ['string', 'number']}, default_schema={'type': 'string'}) + # the bare scalar keeps the wider type + self.assertTrue(self.validates(5, schema)) + self.assertTrue(self.validates('plain', schema)) + # the fallback is narrowed + self.assertTrue( + self.validates( + {'type': 'resource', 'key': 'k', 'default-value': 'd'}, schema)) + self.assertFalse( + self.validates( + {'type': 'resource', 'key': 'k', 'default-value': 5}, schema)) + + def test_default_schema_defaults_to_the_value_schema(self): + one_arg = Lookup.lookup_type({'type': 'number'}) + explicit = Lookup.lookup_type( + {'type': 'number'}, default_schema={'type': 'number'}) + self.assertEqual(one_arg, explicit) + + +class LookupResolveValueTest(BaseTest): + + def test_static_value_passthrough(self): + self.assertEqual(Lookup.resolve_value('plain', {}), 'plain') + + def test_key_hit(self): + resource = {'State': {'Name': 'running'}} + spec = {'type': 'resource', 'key': 'State.Name'} + self.assertEqual(Lookup.resolve_value(spec, resource), 'running') + + def test_key_miss_uses_default(self): + spec = {'type': 'resource', 'key': 'nope', 'default-value': 'dv'} + self.assertEqual(Lookup.resolve_value(spec, {}), 'dv') + + def test_key_miss_without_default_raises(self): + spec = {'type': 'resource', 'key': 'nope'} + with self.assertRaises(Exception) as cm: + Lookup.resolve_value(spec, {}) + self.assertIn('nope', str(cm.exception)) + + def test_conditional_default_writes_when_name_absent(self): + spec = {'type': 'resource', 'default-value': 'dv'} + self.assertEqual( + Lookup.resolve_value(spec, {}, 'Owner', set()), 'dv') + + def test_conditional_default_skips_when_name_present(self): + spec = {'type': 'resource', 'default-value': 'dv'} + self.assertIs( + Lookup.resolve_value(spec, {}, 'Owner', {'Owner'}), Lookup.SKIP) + + def test_skip_sentinel_is_distinct_from_none_and_empty_string(self): + self.assertIsNot(Lookup.SKIP, None) + self.assertNotEqual(Lookup.SKIP, '') + + +class LookupHasLookupsTest(BaseTest): + + def test_all_static(self): + self.assertFalse(Lookup.has_lookups({'A': 'x', 'B': 'y'})) + + def test_empty(self): + self.assertFalse(Lookup.has_lookups({})) + + def test_one_dynamic(self): + self.assertTrue( + Lookup.has_lookups({'A': 'x', 'B': {'type': 'resource', 'key': 'k'}})) diff --git a/tests/test_tags.py b/tests/test_tags.py index 91f4974a9c1..29cbc536ab3 100644 --- a/tests/test_tags.py +++ b/tests/test_tags.py @@ -4,15 +4,17 @@ module to test some universal tagging infrastructure not directly exposed. """ import time +import jsonschema +from datetime import datetime from freezegun import freeze_time -from unittest.mock import MagicMock, call +from unittest.mock import MagicMock, call, patch from botocore.exceptions import ClientError from c7n import tags as tagmod from c7n.tags import universal_retry, coalesce_copy_user_tags from c7n.exceptions import PolicyExecutionError, PolicyValidationError -from c7n.utils import yaml_load +from c7n.utils import FormatDate, yaml_load from .common import BaseTest @@ -608,6 +610,40 @@ def test_copy_related_tag_validate_aws_prefix_fake_resource(self): self.assertRaises(PolicyValidationError, self.load_policy, policy) +class TagValueSchemaTest(BaseTest): + """Tag values are strings, but unquoted YAML scalars have always been + accepted and coerced. The fallback is a newer field and stays strict. + """ + + def validates(self, value): + try: + jsonschema.validate( + {'X': value}, + {'type': 'object', + 'additionalProperties': tagmod.tag_value_schema()}) + return True + except jsonschema.ValidationError: + return False + + def test_scalar_accepts_number_and_boolean(self): + self.assertTrue(self.validates('plain')) + self.assertTrue(self.validates(12345)) + self.assertTrue(self.validates(True)) + + def test_default_value_must_be_a_string(self): + self.assertTrue( + self.validates({'type': 'resource', 'key': 'K', 'default-value': 'd'})) + self.assertFalse( + self.validates({'type': 'resource', 'key': 'K', 'default-value': 5})) + self.assertFalse( + self.validates({'type': 'resource', 'default-value': True})) + + def test_non_scalar_values_rejected(self): + self.assertFalse(self.validates(['a'])) + self.assertFalse(self.validates(None)) + self.assertFalse(self.validates({'foo': 'bar'})) + + class ResolveTagValueTest(BaseTest): def test_static_passthrough(self): self.assertEqual( @@ -717,6 +753,22 @@ def test_placeholder_interpolated_in_default_value(self): Tags=[{"Key": "Stamp", "Value": "created-2022-06-27 12:34:56-in-us-east-1"}], DryRun=False) + def test_now_resolved_once_for_the_whole_run(self): + # a clock that moves on every read - the run should only read it once, + # so every resource lands in one group with one timestamp + ticks = [FormatDate(datetime(2022, 6, 27, 12, 34, 56)), + FormatDate(datetime(2022, 6, 27, 12, 34, 57))] + with patch.object(FormatDate, "utcnow", side_effect=ticks) as utcnow: + create_tags = self._run( + [{"InstanceId": "i-1"}, {"InstanceId": "i-2"}], + {"Stamp": {"type": "resource", "key": "Nope", + "default-value": "at-{now}"}}) + self.assertEqual(utcnow.call_count, 1) + create_tags.assert_called_once_with( + Resources=["i-1", "i-2"], + Tags=[{"Key": "Stamp", "Value": "at-2022-06-27 12:34:56"}], + DryRun=False) + class DynamicUniversalTagTest(BaseTest): def _run(self, resources, tags, resource='rds'): diff --git a/tools/c7n_azure/c7n_azure/actions/tagging.py b/tools/c7n_azure/c7n_azure/actions/tagging.py index 05370fde165..32397896760 100644 --- a/tools/c7n_azure/c7n_azure/actions/tagging.py +++ b/tools/c7n_azure/c7n_azure/actions/tagging.py @@ -40,18 +40,33 @@ class Tag(AzureBaseAction): tag: Environment value: Test - - name: azure-tag-from-resource-attr + :example: + + Tag values may also be resolved per resource. A value of the form + ``{type: resource, key: }`` is looked up against the resource, + falling back to ``default-value`` when the path yields nothing. Omitting + ``key`` gives a conditional default: the tag is written only when it is + not already present, leaving existing values untouched. + + .. code-block:: yaml + + policies: + - name: azure-vm-set-default-tags resource: azure.vm - description: | - Tag all existing virtual machines with values taken from the resource + filters: + - or: + - "tag:Env": empty + - "tag:Owner": empty actions: - type: tag tags: - Environment: Test - ResourceName: - type: resource - key: name - default-value: name_not_found + Env: + type: resource + key: tags.Environment # seed from an existing tag if present + default-value: unknown # otherwise fall back + Owner: + type: resource + default-value: platform@example.com # only set if Owner absent """ schema = utils.type_schema( @@ -61,7 +76,9 @@ class Tag(AzureBaseAction): 'tag': Lookup.lookup_type({'type': 'string'}), 'tags': { 'type': 'object', - 'additionalProperties': Lookup.lookup_type({'type': 'string'}) + 'additionalProperties': Lookup.lookup_type( + {'type': ['string', 'number', 'boolean']}, + default_schema={'type': 'string'}), } } ) @@ -88,10 +105,18 @@ def _process_resource(self, resource): def _get_tags(self, resource): tags = self.data.get('tags') - if tags: - return {k: Lookup.extract(v, resource) for k, v in tags.items()} - return {Lookup.extract( - self.data.get('tag'), resource): Lookup.extract(self.data.get('value'), resource)} + if not tags: + return {Lookup.extract(self.data.get('tag'), resource): + Lookup.extract(self.data.get('value'), resource)} + + current = set(resource.get('tags') or ()) + resolved = {} + for name, spec in tags.items(): + value = Lookup.resolve_value(spec, resource, name, current) + if value is Lookup.SKIP: + continue + resolved[name] = value + return resolved class RemoveTag(AzureBaseAction): diff --git a/tools/c7n_azure/tests_azure/test_actions_tag.py b/tools/c7n_azure/tests_azure/test_actions_tag.py index 8ddbd51de49..d38893ed42d 100644 --- a/tools/c7n_azure/tests_azure/test_actions_tag.py +++ b/tools/c7n_azure/tests_azure/test_actions_tag.py @@ -195,58 +195,162 @@ def test_add_or_update_tags(self, update_resource_tags): self.assertEqual(tags, expected_tags) + +class ActionsTagMappingLookupTest(BaseTest): + """Resource lookups used as values inside the ``tags`` mapping.""" + + existing_tags = {'pre-existing-1': 'unmodified', 'pre-existing-2': 'unmodified'} + + def _get_action(self, data): + # build through a real policy so the action is registry-registered and + # the mapping is run through schema validation on the way in + policy = self.load_policy( + tools.get_policy([dict(data, type='tag')]), validate=True) + return policy.resource_manager.actions[0] + @patch('c7n_azure.tags.TagHelper.update_resource_tags') - def test_add_or_update_tags_from_resource(self, update_resource_tags): - """Verifies values in the tags dictionary are resolved from the resource - rather than passed through as the raw lookup directive - """ + def test_mapping_lookup_by_key_resolves(self, update_resource_tags): + action = self._get_action({ + 'tags': {'from_name': {'type': 'resource', 'key': 'name'}}}) + resource = tools.get_resource(self.existing_tags) - action = self._get_action( - { - 'tags': { - 'tag1': 'value1', - 'tag2': { - 'type': 'resource', - 'key': 'name' - } - } - }) + action.process([resource]) + + expected_tags = self.existing_tags.copy() + expected_tags.update({'from_name': resource['name']}) + self.assertEqual(tools.get_tags_parameter(update_resource_tags), expected_tags) + @patch('c7n_azure.tags.TagHelper.update_resource_tags') + def test_mapping_lookup_miss_uses_default_value(self, update_resource_tags): + action = self._get_action({ + 'tags': {'env': {'type': 'resource', + 'key': 'doesnotexist', + 'default-value': 'production'}}}) resource = tools.get_resource(self.existing_tags) action.process([resource]) - tags = tools.get_tags_parameter(update_resource_tags) + expected_tags = self.existing_tags.copy() + expected_tags.update({'env': 'production'}) + self.assertEqual(tools.get_tags_parameter(update_resource_tags), expected_tags) + + @patch('c7n_azure.tags.TagHelper.update_resource_tags') + def test_mapping_lookup_preserves_existing_tag_value(self, update_resource_tags): + """key pointing at an existing tag keeps that tag's current value. + + A second, changing tag forces the write so the preserved value is + visible in the payload. + """ + action = self._get_action({ + 'tags': {'pre-existing-1': {'type': 'resource', + 'key': 'tags."pre-existing-1"', + 'default-value': 'fallback'}, + 'new': 'added'}}) + resource = tools.get_resource(self.existing_tags) + + action.process([resource]) expected_tags = self.existing_tags.copy() - expected_tags.update({'tag1': 'value1', 'tag2': resource['name']}) + expected_tags.update({'new': 'added'}) + self.assertEqual( + tools.get_tags_parameter(update_resource_tags), expected_tags) - self.assertEqual(tags, expected_tags) + @patch('c7n_azure.tags.TagHelper.update_resource_tags') + def test_no_write_when_lookup_resolves_to_current_value(self, update_resource_tags): + """Resolving a tag to the value it already has is a no-op.""" + action = self._get_action({ + 'tags': {'pre-existing-1': {'type': 'resource', + 'key': 'tags."pre-existing-1"', + 'default-value': 'fallback'}}}) + resource = tools.get_resource(self.existing_tags) + + action.process([resource]) + + update_resource_tags.assert_not_called() @patch('c7n_azure.tags.TagHelper.update_resource_tags') - def test_add_or_update_tags_from_resource_default(self, update_resource_tags): - """Verifies values in the tags dictionary fall back to default-value - when the looked up key does not exist on the resource - """ + def test_mapping_mixes_static_and_lookup_values(self, update_resource_tags): + action = self._get_action({ + 'tags': {'static': 'plain', + 'dynamic': {'type': 'resource', 'key': 'name'}}}) + resource = tools.get_resource(self.existing_tags) - action = self._get_action( - { - 'tags': { - 'tag1': { - 'type': 'resource', - 'key': 'doesnotexist', - 'default-value': 'default_value' - } - } - }) + action.process([resource]) + + expected_tags = self.existing_tags.copy() + expected_tags.update({'static': 'plain', 'dynamic': resource['name']}) + self.assertEqual(tools.get_tags_parameter(update_resource_tags), expected_tags) + @patch('c7n_azure.tags.TagHelper.update_resource_tags') + def test_conditional_default_writes_when_tag_absent(self, update_resource_tags): + action = self._get_action({ + 'tags': {'owner': {'type': 'resource', 'default-value': 'platform'}}}) resource = tools.get_resource(self.existing_tags) action.process([resource]) - tags = tools.get_tags_parameter(update_resource_tags) + expected_tags = self.existing_tags.copy() + expected_tags.update({'owner': 'platform'}) + self.assertEqual(tools.get_tags_parameter(update_resource_tags), expected_tags) + + @patch('c7n_azure.tags.TagHelper.update_resource_tags') + def test_conditional_default_skipped_when_tag_present(self, update_resource_tags): + action = self._get_action({ + 'tags': {'pre-existing-1': {'type': 'resource', + 'default-value': 'should-not-be-written'}}}) + resource = tools.get_resource(self.existing_tags) + + action.process([resource]) + + # nothing left to write, so no update call is made at all + update_resource_tags.assert_not_called() + + @patch('c7n_azure.tags.TagHelper.update_resource_tags') + def test_conditional_default_partial_skip(self, update_resource_tags): + action = self._get_action({ + 'tags': {'pre-existing-1': {'type': 'resource', 'default-value': 'skipped'}, + 'owner': {'type': 'resource', 'default-value': 'platform'}}}) + resource = tools.get_resource(self.existing_tags) + + action.process([resource]) expected_tags = self.existing_tags.copy() - expected_tags.update({'tag1': 'default_value'}) + expected_tags.update({'owner': 'platform'}) + self.assertEqual(tools.get_tags_parameter(update_resource_tags), expected_tags) - self.assertEqual(tags, expected_tags) + @patch('c7n_azure.tags.TagHelper.update_resource_tags') + def test_lookup_resolved_per_resource(self, update_resource_tags): + """Two resources with different values each get their own tag value.""" + action = self._get_action({ + 'tags': {'from_name': {'type': 'resource', 'key': 'name'}}}) + first = tools.get_resource({}) + second = tools.get_resource({}) + second['name'] = 'other-vm' + + action.process([first, second]) + + written = [c[0][2]['from_name'] for c in update_resource_tags.call_args_list] + self.assertEqual(written, ['cctestvm', 'other-vm']) + + def test_schema_accepts_lookup_forms_in_mapping(self): + self.assertTrue(self.load_policy( + tools.get_policy([ + {'type': 'tag', + 'tags': { + 'by_key': {'type': 'resource', 'key': 'name'}, + 'by_key_default': {'type': 'resource', + 'key': 'name', + 'default-value': 'dv'}, + 'conditional': {'type': 'resource', 'default-value': 'dv'}, + 'static': 'plain'}} + ]), validate=True)) + + def test_schema_rejects_malformed_mapping_values(self): + for bad in ({'type': 'resource'}, + {'type': 'resource', 'ky': 'name'}, + {'foo': 'bar'}, + ['a']): + with self.assertRaises(Exception): + self.load_policy( + tools.get_policy([{'type': 'tag', 'tags': {'X': bad}}]), + validate=True) diff --git a/tools/c7n_gcp/c7n_gcp/actions/labels.py b/tools/c7n_gcp/c7n_gcp/actions/labels.py index 28797476661..07b4477eef2 100644 --- a/tools/c7n_gcp/c7n_gcp/actions/labels.py +++ b/tools/c7n_gcp/c7n_gcp/actions/labels.py @@ -115,6 +115,18 @@ class SetLabelsAction(BaseLabelAction): key: name default-value: name_not_found + - name: gcp-add-default-label + resource: gcp.instance + description: | + Omitting key gives a conditional default - the label is set only + when it is absent, leaving an existing value untouched + actions: + - type: set-labels + labels: + owner: + type: resource + default-value: platform + - name: gcp-remove-label resource: gcp.instance description: | @@ -135,7 +147,14 @@ def validate(self): raise FilterValidationError("Must specify one of labels or remove") def get_labels_to_add(self, resource): - return {k: Lookup.extract(v, resource) for k, v in self.data.get('labels', {}).items()} + current = set(self._get_current_labels(resource)) + labels = {} + for name, spec in self.data.get('labels', {}).items(): + value = Lookup.resolve_value(spec, resource, name, current) + if value is Lookup.SKIP: + continue + labels[name] = value + return labels def get_labels_to_delete(self, resource): return self.data.get('remove') diff --git a/tools/c7n_gcp/tests/test_gcp_labels.py b/tools/c7n_gcp/tests/test_gcp_labels.py index 6c019515bf8..cc4c813898a 100644 --- a/tools/c7n_gcp/tests/test_gcp_labels.py +++ b/tools/c7n_gcp/tests/test_gcp_labels.py @@ -47,6 +47,78 @@ def test_schema_validate(self): ])) +class SetLabelsLookupTest(BaseTest): + """Resource lookups used as values inside the ``labels`` mapping.""" + + def _get_action(self, labels): + policy = self.load_policy( + get_policy([{'type': 'set-labels', 'labels': labels}])) + return policy.resource_manager.actions[0] + + def test_lookup_by_key_resolves(self): + action = self._get_action({'env': {'type': 'resource', 'key': 'name'}}) + self.assertEqual( + action.get_labels_to_add({'name': 'instance-1', 'labels': {}}), + {'env': 'instance-1'}) + + def test_lookup_miss_uses_default_value(self): + action = self._get_action( + {'env': {'type': 'resource', + 'key': 'doesnotexist', + 'default-value': 'production'}}) + self.assertEqual( + action.get_labels_to_add({'name': 'instance-1', 'labels': {}}), + {'env': 'production'}) + + def test_static_value_passthrough(self): + action = self._get_action({'env': 'test'}) + self.assertEqual( + action.get_labels_to_add({'name': 'instance-1', 'labels': {}}), + {'env': 'test'}) + + def test_conditional_default_writes_when_label_absent(self): + action = self._get_action( + {'owner': {'type': 'resource', 'default-value': 'platform'}}) + self.assertEqual( + action.get_labels_to_add({'name': 'instance-1', 'labels': {}}), + {'owner': 'platform'}) + + def test_conditional_default_skipped_when_label_present(self): + action = self._get_action( + {'owner': {'type': 'resource', 'default-value': 'platform'}}) + self.assertEqual( + action.get_labels_to_add( + {'name': 'instance-1', 'labels': {'owner': 'keep-me'}}), + {}) + + def test_conditional_default_leaves_existing_label_intact(self): + """The skipped label must survive the merge with current labels.""" + action = self._get_action( + {'owner': {'type': 'resource', 'default-value': 'platform'}, + 'env': 'test'}) + resource = {'name': 'instance-1', 'labels': {'owner': 'keep-me'}} + merged = action._merge_labels( + action._get_current_labels(resource), + action.get_labels_to_add(resource), + action.get_labels_to_delete(resource)) + self.assertEqual(merged, {'owner': 'keep-me', 'env': 'test'}) + + def test_schema_accepts_conditional_default(self): + self.assertTrue(self.load_policy(get_policy([ + {'type': 'set-labels', + 'labels': {'owner': {'type': 'resource', 'default-value': 'x'}}} + ]))) + + def test_schema_rejects_malformed_label_values(self): + for bad in ({'type': 'resource'}, + {'type': 'resource', 'ky': 'name'}, + {'foo': 'bar'}): + with self.assertRaises(Exception): + self.load_policy( + get_policy([{'type': 'set-labels', 'labels': {'x': bad}}]), + validate=True) + + class LabelDelayedActionTest(BaseTest): def test_schema_validate(self): From 1b5ad409c27f7c283f396f991bf11e637c350286 Mon Sep 17 00:00:00 2001 From: James Troup Date: Thu, 27 Aug 2026 17:03:04 +0000 Subject: [PATCH 5/7] gcp - cloud-run - drop non-schema fields from label mutations (#11052) --- tools/c7n_gcp/c7n_gcp/resources/cloudrun.py | 59 ++++- ...loud-custodian-locations---services_1.json | 57 +++++ ...ocations-us-central1-services-hello_1.json | 52 +++++ tools/c7n_gcp/tests/test_cloudrun.py | 209 ++++++++++++++++++ 4 files changed, 373 insertions(+), 4 deletions(-) create mode 100644 tools/c7n_gcp/tests/data/flights/gcp-cloud-run-service-mark-for-op/get-v1-projects-cloud-custodian-locations---services_1.json create mode 100644 tools/c7n_gcp/tests/data/flights/gcp-cloud-run-service-mark-for-op/put-v1-projects-cloud-custodian-locations-us-central1-services-hello_1.json diff --git a/tools/c7n_gcp/c7n_gcp/resources/cloudrun.py b/tools/c7n_gcp/c7n_gcp/resources/cloudrun.py index 54f479b60d0..7941f7b746e 100644 --- a/tools/c7n_gcp/c7n_gcp/resources/cloudrun.py +++ b/tools/c7n_gcp/c7n_gcp/resources/cloudrun.py @@ -1,13 +1,64 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 -import copy - from c7n_gcp.provider import resources from c7n_gcp.query import QueryResourceManager, TypeInfo from c7n_gcp.filters import IamPolicyFilter from c7n_gcp.filters.iampolicy import IamPolicyValueFilter +from c7n.exceptions import PolicyExecutionError from c7n.utils import local_session, jmespath_search +# The only fields the Knative schema defines at the root of a Service or a Job. +KNATIVE_FIELDS = ('apiVersion', 'kind', 'metadata', 'spec', 'status') + +# Prefixes of the annotations c7n keeps on a resource. Only the colon form is +# written to a nested object; the dotted `c7n.metrics` is set at the root alone. +# Nesting matches on the colon form only, because kubernetes annotation keys - +# user controlled, and carried in `metadata.annotations` - may contain a dot but +# never a colon, and this is a whole object replace: a key dropped from the body +# is a key deleted from the service. +C7N_ANNOTATION = 'c7n:' +C7N_ROOT_ANNOTATIONS = (C7N_ANNOTATION, 'c7n.') + + +def _without_annotations(value): + "Copy a value, dropping any c7n annotation held anywhere beneath it." + if isinstance(value, dict): + return {k: _without_annotations(v) for k, v in value.items() + if not k.startswith(C7N_ANNOTATION)} + if isinstance(value, list): + return [_without_annotations(v) for v in value] + # everything else in a decoded api response is an immutable scalar + return value + + +def knative_body(resource): + """Copy a resource into a replace request body, dropping non schema fields. + + Cloud Run replaces the whole object, so the body is the resource itself + rather than a fragment, and the api rejects the request with a 400 on any + field it can't place. A resource carries two kinds it can't: the top level + `labels` key that augment() flattens out of the metadata, and the + annotations the policy's filters leave behind - at the root, and nested + within `spec` when a list-item filter matched on something there. + + A root field that is neither raises rather than being dropped. Under a + whole object replace a field left out of the body is deleted from the + service, so a field this doesn't recognize is one it cannot safely send + or safely omit. + """ + unrecognized = { + k for k in resource + if k not in KNATIVE_FIELDS and k != 'labels' + and not k.startswith(C7N_ROOT_ANNOTATIONS) + } + if unrecognized: + raise PolicyExecutionError( + '%s %s has fields absent from the cloud run schema: %s' % ( + resource.get('kind', 'resource'), + resource.get('metadata', {}).get('name'), + ', '.join(sorted(unrecognized)))) + return {k: _without_annotations(v) for k, v in resource.items() if k in KNATIVE_FIELDS} + @resources.register("cloud-run-service") class CloudRunService(QueryResourceManager): @@ -35,7 +86,7 @@ def get_label_params(resource, all_labels): location = metadata['labels']['cloud.googleapis.com/location'] namespace = metadata['namespace'] svc_name = metadata['name'] - body = copy.deepcopy(resource) + body = knative_body(resource) body['metadata']['labels'] = all_labels return { 'name': 'projects/{}/locations/{}/services/{}'.format( @@ -98,7 +149,7 @@ def get_label_params(resource, all_labels): metadata = resource['metadata'] namespace = metadata['namespace'] job_name = metadata['name'] - body = copy.deepcopy(resource) + body = knative_body(resource) body['metadata']['labels'] = all_labels return { 'name': 'namespaces/{}/jobs/{}'.format(namespace, job_name), diff --git a/tools/c7n_gcp/tests/data/flights/gcp-cloud-run-service-mark-for-op/get-v1-projects-cloud-custodian-locations---services_1.json b/tools/c7n_gcp/tests/data/flights/gcp-cloud-run-service-mark-for-op/get-v1-projects-cloud-custodian-locations---services_1.json new file mode 100644 index 00000000000..998083a29fb --- /dev/null +++ b/tools/c7n_gcp/tests/data/flights/gcp-cloud-run-service-mark-for-op/get-v1-projects-cloud-custodian-locations---services_1.json @@ -0,0 +1,57 @@ +{ + "headers": { + "content-type": "application/json; charset=UTF-8", + "status": "200" + }, + "body": { + "apiVersion": "serving.knative.dev/v1", + "kind": "ServiceList", + "items": [ + { + "apiVersion": "serving.knative.dev/v1", + "kind": "Service", + "metadata": { + "name": "hello", + "namespace": "cloud-custodian", + "selfLink": "/apis/serving.knative.dev/v1/namespaces/cloud-custodian/services/hello", + "uid": "46b28549-6c5c-4245-bc8d-9cec06d262b6", + "resourceVersion": "AAX4MAndusc", + "generation": 1, + "labels": { + "cloud.googleapis.com/location": "us-central1" + }, + "annotations": { + "run.googleapis.com/ingress": "all" + }, + "creationTimestamp": "2023-03-31T10:53:45.882066Z" + }, + "spec": { + "template": { + "metadata": { + "name": "hello-00001-tiv", + "annotations": {} + }, + "spec": { + "containerConcurrency": 80, + "timeoutSeconds": 300, + "containers": [ + { + "image": "us-docker.pkg.dev/cloudrun/container/hello", + "ports": [{"name": "http1", "containerPort": 8080}] + } + ] + } + }, + "traffic": [{"percent": 100, "latestRevision": true}] + }, + "status": { + "observedGeneration": 1, + "conditions": [ + {"type": "Ready", "status": "True"} + ], + "url": "https://hello-g3kdvpt64a-uc.a.run.app" + } + } + ] + } +} diff --git a/tools/c7n_gcp/tests/data/flights/gcp-cloud-run-service-mark-for-op/put-v1-projects-cloud-custodian-locations-us-central1-services-hello_1.json b/tools/c7n_gcp/tests/data/flights/gcp-cloud-run-service-mark-for-op/put-v1-projects-cloud-custodian-locations-us-central1-services-hello_1.json new file mode 100644 index 00000000000..b64a30335e2 --- /dev/null +++ b/tools/c7n_gcp/tests/data/flights/gcp-cloud-run-service-mark-for-op/put-v1-projects-cloud-custodian-locations-us-central1-services-hello_1.json @@ -0,0 +1,52 @@ +{ + "headers": { + "content-type": "application/json; charset=UTF-8", + "status": "200" + }, + "body": { + "apiVersion": "serving.knative.dev/v1", + "kind": "Service", + "metadata": { + "name": "hello", + "namespace": "cloud-custodian", + "selfLink": "/apis/serving.knative.dev/v1/namespaces/cloud-custodian/services/hello", + "uid": "46b28549-6c5c-4245-bc8d-9cec06d262b6", + "resourceVersion": "AAX4MBndusc", + "generation": 2, + "labels": { + "cloud.googleapis.com/location": "us-central1", + "custodian_status": "resource_policy-notify-2026_08_21__0_0" + }, + "annotations": { + "run.googleapis.com/ingress": "all" + }, + "creationTimestamp": "2023-03-31T10:53:45.882066Z" + }, + "spec": { + "template": { + "metadata": { + "name": "hello-00001-tiv", + "annotations": {} + }, + "spec": { + "containerConcurrency": 80, + "timeoutSeconds": 300, + "containers": [ + { + "image": "us-docker.pkg.dev/cloudrun/container/hello", + "ports": [{"name": "http1", "containerPort": 8080}] + } + ] + } + }, + "traffic": [{"percent": 100, "latestRevision": true}] + }, + "status": { + "observedGeneration": 2, + "conditions": [ + {"type": "Ready", "status": "True"} + ], + "url": "https://hello-g3kdvpt64a-uc.a.run.app" + } + } +} diff --git a/tools/c7n_gcp/tests/test_cloudrun.py b/tools/c7n_gcp/tests/test_cloudrun.py index 7483c0c7a95..f4f7061a1ef 100644 --- a/tools/c7n_gcp/tests/test_cloudrun.py +++ b/tools/c7n_gcp/tests/test_cloudrun.py @@ -1,9 +1,153 @@ # Copyright The Cloud Custodian Authors. # SPDX-License-Identifier: Apache-2.0 +import pytest + +from freezegun import freeze_time + from gcp_common import BaseTest +from c7n.exceptions import PolicyExecutionError from c7n.utils import yaml_load +from c7n_gcp.actions.core import MethodAction +from c7n_gcp.resources.cloudrun import CloudRunJob, CloudRunService, knative_body + +# The root fields the api accepts, independent of the module under test. +SCHEMA_FIELDS = {'apiVersion', 'kind', 'metadata', 'spec', 'status'} +LOCATION = {'cloud.googleapis.com/location': 'us-central1'} + + +def listed_resource(api_version, kind, name): + "A resource as augment() and a value filter leave it once listed." + return { + 'apiVersion': api_version, + 'kind': kind, + 'metadata': { + 'name': name, + 'namespace': 'cloud-custodian', + 'labels': dict(LOCATION), + }, + 'spec': {'template': {}}, + 'status': {'observedGeneration': 1}, + 'labels': dict(LOCATION), + 'c7n:MatchedFilters': ['metadata.name'], + } + + +def capture_api_params(test): + """Record the params of every api call the policy's actions make.""" + captured = [] + invoke_api = MethodAction.invoke_api + + def record(action, client, op_name, params): + captured.append((op_name, params)) + return invoke_api(action, client, op_name, params) + + test.patch(MethodAction, 'invoke_api', record) + return captured + + +def assert_replace_body(body, labels): + "A replace body holds the schema fields and the merged labels, nothing else." + assert set(body) == SCHEMA_FIELDS + assert 'labels' not in body + assert 'c7n:MatchedFilters' not in body + assert body['metadata']['labels'] == labels + + +def assert_label_params(model, resource, expected_name): + "get_label_params addresses the resource and leaves it fit to read again." + all_labels = dict(LOCATION, environment='test') + params = model.get_label_params(resource, all_labels) + + assert params['name'] == expected_name + assert_replace_body(params['body'], all_labels) + # the action reads its current label state from the resource, so both the + # synthetic key and the metadata it came from have to survive the call + assert resource['labels'] == LOCATION + assert resource['metadata']['labels'] == LOCATION + + +class KnativeBodyTest(BaseTest): + + def test_keeps_only_the_schema_fields(self): + body = knative_body({ + 'apiVersion': 'serving.knative.dev/v1', + 'kind': 'Service', + 'metadata': {'name': 'hello'}, + 'spec': {}, + 'status': {}, + 'labels': {'env': 'test'}, + 'c7n:MatchedFilters': ['metadata.name'], + }) + assert set(body) == SCHEMA_FIELDS + + def test_drops_annotations_nested_in_spec(self): + # a list-item filter annotates the objects it matched in place + body = knative_body({ + 'metadata': {'name': 'hello'}, + 'spec': { + 'template': { + 'spec': { + 'containers': [{ + 'image': 'us-docker.pkg.dev/cloudrun/container/hello', + 'c7n:MatchedFilters': ['image'], + }], + }, + }, + }, + 'c7n.metrics': {'run.googleapis.com/request_count': []}, + }) + assert body['spec'] == { + 'template': { + 'spec': { + 'containers': [{ + 'image': 'us-docker.pkg.dev/cloudrun/container/hello'}], + }, + }, + } + + def test_keeps_user_annotations(self): + # a kubernetes annotation key may contain a dot, so only the colon form + # is recognized below the root - a replace deletes what the body omits + annotations = { + 'run.googleapis.com/ingress': 'all', + 'c7n.example.com/owner': 'platform-team', + } + body = knative_body({ + 'metadata': {'name': 'hello', 'annotations': dict(annotations)}, + 'spec': {'template': {'metadata': {'annotations': dict(annotations)}}}, + }) + assert body['metadata']['annotations'] == annotations + assert body['spec']['template']['metadata']['annotations'] == annotations + + def test_copies_rather_than_mutates(self): + resource = { + 'metadata': {'name': 'hello'}, + 'spec': {'containers': [{'c7n:MatchedFilters': ['image']}]}, + } + knative_body(resource)['metadata']['name'] = 'changed' + assert resource['metadata']['name'] == 'hello' + assert resource['spec']['containers'][0] == {'c7n:MatchedFilters': ['image']} + + def test_raises_on_fields_outside_the_schema(self): + with pytest.raises(PolicyExecutionError) as raised: + knative_body({ + 'kind': 'Service', + 'metadata': {'name': 'hello'}, + 'somethingNew': 'unrecognized', + }) + assert 'somethingNew' in str(raised.value) + assert 'hello' in str(raised.value) + + def test_accepts_the_synthetic_fields(self): + knative_body({ + 'metadata': {'name': 'hello'}, + 'labels': {'env': 'test'}, + 'c7n:MatchedFilters': ['metadata.name'], + 'c7n.metrics': {'run.googleapis.com/request_count': []}, + }) + class RunServiceTest(BaseTest): def test_query(self): @@ -28,11 +172,18 @@ def test_query_empty(self): resources = p.run() assert resources == [] + def test_label_params(self): + assert_label_params( + CloudRunService.resource_type, + listed_resource('serving.knative.dev/v1', 'Service', 'hello'), + 'projects/cloud-custodian/locations/us-central1/services/hello') + def test_set_labels(self): project_id = 'cloud-custodian' factory = self.replay_flight_data( "gcp-cloud-run-service-set-labels", project_id=project_id ) + captured = capture_api_params(self) p = self.load_policy( { 'name': 'cloud-run-svc-set-labels', @@ -53,6 +204,49 @@ def test_set_labels(self): assert len(resources) == 1 assert resources[0]['metadata']['name'] == 'hello' + assert len(captured) == 1 + op_name, params = captured.pop() + assert op_name == 'replaceService' + assert_replace_body(params['body'], { + 'cloud.googleapis.com/location': 'us-central1', + 'environment': 'test', + }) + + @freeze_time("2026-08-19") + def test_mark_for_op(self): + project_id = 'cloud-custodian' + factory = self.replay_flight_data( + "gcp-cloud-run-service-mark-for-op", project_id=project_id + ) + captured = capture_api_params(self) + p = self.load_policy( + { + 'name': 'cloud-run-svc-mark-for-op', + 'resource': 'gcp.cloud-run-service', + 'filters': [ + {'type': 'value', + 'key': 'metadata.name', + 'value': 'hello'} + ], + 'actions': [ + {'type': 'mark-for-op', + 'op': 'notify', + 'days': 2} + ] + }, + session_factory=factory, + ) + resources = p.run() + assert len(resources) == 1 + + assert len(captured) == 1 + op_name, params = captured.pop() + assert op_name == 'replaceService' + assert_replace_body(params['body'], { + 'cloud.googleapis.com/location': 'us-central1', + 'custodian_status': 'resource_policy-notify-2026_08_21__0_0', + }) + def test_filter(self): factory = self.replay_flight_data("gcp-cloud-run-service") @@ -103,11 +297,18 @@ def test_query(self): assert len(resources) == 1 assert resources[0]["metadata"]["name"] == "job" + def test_label_params(self): + assert_label_params( + CloudRunJob.resource_type, + listed_resource('run.googleapis.com/v1', 'Job', 'job'), + 'namespaces/cloud-custodian/jobs/job') + def test_set_labels(self): project_id = 'cloud-custodian' factory = self.replay_flight_data( "gcp-cloud-run-job-set-labels", project_id=project_id ) + captured = capture_api_params(self) p = self.load_policy( { 'name': 'cloud-run-job-set-labels', @@ -128,6 +329,14 @@ def test_set_labels(self): assert len(resources) == 1 assert resources[0]['metadata']['name'] == 'job' + assert len(captured) == 1 + op_name, params = captured.pop() + assert op_name == 'replaceJob' + assert_replace_body(params['body'], { + 'cloud.googleapis.com/location': 'us-central1', + 'environment': 'test', + }) + class RevisionServiceTest(BaseTest): def test_query(self): From 50d330ad1d25bd24b3fc2f3824ec6af3e3ef5da1 Mon Sep 17 00:00:00 2001 From: James Troup Date: Wed, 19 Aug 2026 22:52:37 +0000 Subject: [PATCH 6/7] chore: aws(s3) - per bucket tolerance in the tag action (#19) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### what `modify_bucket_tags()` tolerates the same transport failures `BucketAssembly.assemble()` already does, on both the tag re-fetch and the write. A bucket whose endpoint doesn't answer is logged and skipped; the rest of the batch is tagged as normal. `ConnectionClosedError` joins that set in both places. A degraded endpoint can accept a connection and then drop it rather than refusing or hanging, and that class was in neither handler - so it would still abort the tag action, and still abort collection for the whole account, which is what cd79871f1 set out to stop. `ProxyConnectionError`, `SSLError` and `ResponseStreamingError` are deliberately left to raise: the first two say something is wrong with the path to s3 rather than with one bucket, and the third can't plausibly happen on a response this small. ### why One bucket whose endpoint hangs silently costs healthy buckets their tags, on every run. `modify_bucket_tags()` caught only `ClientError`, so a connect timeout raised out of it and abandoned the rest of its batch. `_common_tag_processer` chunks at `Tag.batch_size` - 25 for `tag`, 100 for `remove-tag` - and walks each chunk serially, so the loss is every bucket after the hung one in its own chunk. How many that is depends on where the hung bucket sorts in `list_buckets` order, so it is silent, it varies, and it repeats every run. Measured against an account matching the reported one, 76 buckets with one timing out on every call: | hung bucket's position | before | after | | ---------------------- | ------ | ----- | | last | run errors, 75 of 75 tagged | succeeds, 75 of 75 | | eleventh | run errors, 61 of 75 tagged | succeeds, 75 of 75 | The run also reported error either way, so an account that was in fact being tagged still looked like a failure to anything gating on run status. This is what remains after cd79871f1, which stopped the same bucket from aborting resource collection outright. `list_buckets` is global, so every run in every region enumerates the whole account - one bucket whose endpoint hangs is on the path of every s3 policy, not only ones targeting its region. The bucket that couldn't be read is skipped rather than written to: tags are set atomically for the whole set, so writing one we couldn't re-read would drop whatever tags it has. ### testing New tests cover a hung bucket on the re-fetch and on the write leaving the rest of the batch tagged; a connection closed mid response surviving both collection and the tag action; and an end to end case - collection then tag action, with one bucket timing out on every call - asserting that a hung bucket at the head of a chunk costs none of the 29 buckets behind it. That last one fails against the previous behaviour with the timeout propagating out of the action, which is the reported symptom. ### docs No docs needed. --------------------------- 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- c7n/resources/s3.py | 17 ++++-- tests/test_s3.py | 129 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 142 insertions(+), 4 deletions(-) diff --git a/c7n/resources/s3.py b/c7n/resources/s3.py index 6d34f12848c..517bb1e5a60 100644 --- a/c7n/resources/s3.py +++ b/c7n/resources/s3.py @@ -38,7 +38,8 @@ from botocore.client import Config from botocore.exceptions import ( - ClientError, ConnectTimeoutError, EndpointConnectionError, ReadTimeoutError) + ClientError, ConnectionClosedError, ConnectTimeoutError, EndpointConnectionError, + ReadTimeoutError) from collections import defaultdict from concurrent.futures import as_completed @@ -619,7 +620,8 @@ def assemble(self, bucket): "Bucket ssl error %s: %s %s", bucket['Name'], bucket.get('Location', 'unknown'), e) continue - except (ConnectTimeoutError, ReadTimeoutError, EndpointConnectionError) as e: + except (ConnectTimeoutError, ReadTimeoutError, EndpointConnectionError, + ConnectionClosedError) as e: # Endpoint is unreachable or hung - could be a degraded/ # unreachable region, or a bucket deleted between # list_buckets and here. @@ -705,6 +707,14 @@ def modify_bucket_tags(session_factory, buckets, add_tags=(), remove_tags=()): if e.response['Error']['Code'] != 'NoSuchTagSet': raise bucket['Tags'] = [] + except (ConnectTimeoutError, ReadTimeoutError, EndpointConnectionError, + ConnectionClosedError) as e: + # Endpoint is unreachable or hung - skip this bucket rather than + # abandoning the rest of the batch. + log.warning( + "Unable to get new set of bucket tags needed to modify tags, " + "skipping tag action for bucket: %s error: %s", bucket["Name"], e) + continue new_tags = {t['Key']: t['Value'] for t in add_tags} for t in bucket.get('Tags', ()): @@ -715,7 +725,8 @@ def modify_bucket_tags(session_factory, buckets, add_tags=(), remove_tags=()): try: client.put_bucket_tagging( Bucket=bucket['Name'], Tagging={'TagSet': tag_set}) - except ClientError as e: + except (ClientError, ConnectTimeoutError, ReadTimeoutError, + EndpointConnectionError, ConnectionClosedError) as e: log.exception( 'Exception tagging bucket %s: %s', bucket['Name'], e) continue diff --git a/tests/test_s3.py b/tests/test_s3.py index b54b668c79b..eef077c2e1e 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -14,7 +14,9 @@ from unittest import TestCase from contextlib import suppress -from botocore.exceptions import ClientError, ConnectTimeoutError, EndpointConnectionError +from botocore.exceptions import ( + ClientError, ConnectionClosedError, ConnectTimeoutError, EndpointConnectionError, + ReadTimeoutError) from dateutil.tz import tzutc import pytest from pytest_terraform import terraform @@ -36,6 +38,12 @@ ) +# every transport failure botocore raises that we mean to survive +TRANSPORT_ERRORS = pytest.mark.parametrize('error_class', [ + ConnectTimeoutError, ReadTimeoutError, EndpointConnectionError, + ConnectionClosedError]) + + def test_s3_assembly_validate(test): for tcase in ('all', 'none', 'detect', ['Website', 'Replication']): @@ -166,6 +174,125 @@ def test_s3_assembly_endpoint_connection_error(test): assert 'Location' not in bucket +def test_s3_assembly_connection_closed(test): + # a connection closed mid response is as unreachable as a connect + # timeout - a degraded endpoint can accept and then drop + policy = test.load_policy({'name': 's3-attrs', 'resource': 's3'}) + assembly = s3.BucketAssembly(policy.resource_manager) + assembly.initialize() + + client = mock.MagicMock() + client.meta.region_name = assembly.default_region + client.get_bucket_location.side_effect = ConnectionClosedError(endpoint_url='x') + assembly.region_clients[assembly.default_region] = client + + bucket = assembly.assemble({'Name': 'hung-bucket'}) + assert bucket['Name'] == 'hung-bucket' + assert 'Location' not in bucket + + +@TRANSPORT_ERRORS +def test_s3_modify_tags_continues_past_unreachable_bucket(test, error_class): + # one hung bucket shouldn't cost us the rest of the batch + client = mock.MagicMock() + client.get_bucket_tagging.side_effect = [ + error_class(endpoint_url='x'), + {'TagSet': [{'Key': 'Existing', 'Value': 'keep'}]}] + test.patch(s3, 'bucket_client', lambda session, b, kms=False: client) + + s3.modify_bucket_tags( + lambda: mock.MagicMock(), + [{'Name': 'hung'}, {'Name': 'healthy'}], + add_tags=[{'Key': 'Owner', 'Value': 'devs'}]) + + # the healthy bucket is still tagged, and keeps the tags it had + client.put_bucket_tagging.assert_called_once_with( + Bucket='healthy', + Tagging={'TagSet': [{'Key': 'Owner', 'Value': 'devs'}, + {'Key': 'Existing', 'Value': 'keep'}]}) + + +@TRANSPORT_ERRORS +def test_s3_modify_tags_continues_past_failed_write(test, error_class): + # a bucket that answers the read and then hangs on the write is skipped + # too, and the rest of the batch still gets tagged + client = mock.MagicMock() + client.get_bucket_tagging.return_value = {'TagSet': []} + client.put_bucket_tagging.side_effect = [ + error_class(endpoint_url='x'), None] + test.patch(s3, 'bucket_client', lambda session, b, kms=False: client) + + s3.modify_bucket_tags( + lambda: mock.MagicMock(), + [{'Name': 'hung'}, {'Name': 'healthy'}], + add_tags=[{'Key': 'Owner', 'Value': 'devs'}]) + + assert client.put_bucket_tagging.call_count == 2 + + +def test_s3_hung_bucket_doesnt_cost_the_rest_of_its_batch(test): + """one bucket whose endpoint hangs, end to end, over a batch boundary. + + the tag action chunks at 25 and walks the buckets within a chunk + serially, so a hung bucket that raises takes every bucket after it in + its own chunk down with it, and fails the run. + """ + hung, healthy = 'hung-bucket', ['healthy-%02d' % i for i in range(29)] + buckets = [{'Name': hung}] + [{'Name': n} for n in healthy] + + def make_client(region): + client = mock.MagicMock() + client.meta.region_name = region + if region == 'elsewhere-1': + for method_name, _, _, _, _ in s3.S3_AUGMENT_TABLE: + getattr(client, method_name).side_effect = ConnectTimeoutError( + endpoint_url='https://s3.elsewhere-1.amazonaws.com') + return client + client.get_bucket_location.side_effect = lambda Bucket: { + 'LocationConstraint': 'elsewhere-1' if Bucket == hung else None} + client.get_bucket_tagging.return_value = {'TagSet': []} + return client + + clients = {} + + def session(*args, **kw): + s = mock.MagicMock() + + def client(svc, region_name=None, **kw): + region = region_name or 'us-east-1' + if region not in clients: + clients[region] = make_client(region) + return clients[region] + + s.client.side_effect = client + return s + + policy = test.load_policy({ + 'name': 's3-tag', 'resource': 's3', + 'actions': [{'type': 'tag', 'key': 'Owner', 'value': 'devs'}]}) + test.patch(s3, 'local_session', session) + + resources = s3.DescribeS3(policy.resource_manager).augment(buckets) + assert len(resources) == len(buckets) + + # the endpoint is just as unreachable for the action as for collection + written = {} + + def action_client(sess, b, kms=False): + if b['Name'] not in written: + written[b['Name']] = make_client( + 'elsewhere-1' if b['Name'] == hung else 'us-east-1') + return written[b['Name']] + + test.patch(s3, 'bucket_client', action_client) + policy.resource_manager.actions[0].process(resources) + + tagged = {n for n, c in written.items() if c.put_bucket_tagging.called} + assert hung not in tagged + assert tagged == set(healthy), "the hung bucket cost %d of its batch" % ( + len(set(healthy) - tagged)) + + def test_s3_express(test): session_factory = test.replay_flight_data('test_s3_express') p = test.load_policy( From 430a75e743ac2f5df079c60bedc51698ce8762e2 Mon Sep 17 00:00:00 2001 From: James Troup Date: Sat, 26 Sep 2026 15:06:02 +0100 Subject: [PATCH 7/7] chore: cherry pick tagging fixes and bump post version --- c7n/version.py | 2 +- pyproject.toml | 2 +- tools/c7n_awscc/pyproject.toml | 2 +- tools/c7n_azure/pyproject.toml | 2 +- tools/c7n_gcp/pyproject.toml | 2 +- tools/c7n_kube/pyproject.toml | 2 +- tools/c7n_left/pyproject.toml | 2 +- tools/c7n_mailer/pyproject.toml | 2 +- tools/c7n_oci/pyproject.toml | 2 +- tools/c7n_openstack/pyproject.toml | 2 +- tools/c7n_org/pyproject.toml | 2 +- tools/c7n_policystream/pyproject.toml | 2 +- tools/c7n_sphinxext/pyproject.toml | 2 +- tools/c7n_tencentcloud/pyproject.toml | 2 +- uv.lock | 26 +++++++++++++------------- 15 files changed, 27 insertions(+), 27 deletions(-) diff --git a/c7n/version.py b/c7n/version.py index e04cbc94a77..32ff6e9a53c 100644 --- a/c7n/version.py +++ b/c7n/version.py @@ -1,2 +1,2 @@ # Generated via tools/dev/devpkg.py -version = "0.9.52a5.post1+a83dda5c9" +version = "0.9.52a5.post2+a83dda5c9" diff --git a/pyproject.toml b/pyproject.toml index 43b6d397fed..3d5e20cc1b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ "urllib3>2", ] name = "c7n" -version = "0.9.52a5.post1+a83dda5c9" +version = "0.9.52a5.post2+a83dda5c9" description = "Cloud Custodian - Policy Rules Engine" readme = "README.md" classifiers = [ diff --git a/tools/c7n_awscc/pyproject.toml b/tools/c7n_awscc/pyproject.toml index 65f3ae02a7a..551d6777c8d 100644 --- a/tools/c7n_awscc/pyproject.toml +++ b/tools/c7n_awscc/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "jsonpatch<2.0,>=1.32", ] name = "c7n_awscc" -version = "0.1.37a5.post1+a83dda5c9" +version = "0.1.37a5.post2+a83dda5c9" readme = "readme.md" description = "Cloud Custodian - AWS Cloud Control Provider" classifiers = [ diff --git a/tools/c7n_azure/pyproject.toml b/tools/c7n_azure/pyproject.toml index b088403cf44..fe422ebe95d 100644 --- a/tools/c7n_azure/pyproject.toml +++ b/tools/c7n_azure/pyproject.toml @@ -87,7 +87,7 @@ dependencies = [ "requests<3.0.0,>=2.22.0", ] name = "c7n_azure" -version = "0.7.51a5.post1+a83dda5c9" +version = "0.7.51a5.post2+a83dda5c9" description = "Cloud Custodian - Azure Support" readme = "readme.md" classifiers = [ diff --git a/tools/c7n_gcp/pyproject.toml b/tools/c7n_gcp/pyproject.toml index e9c2c6d2cbe..26bc32a6e1a 100644 --- a/tools/c7n_gcp/pyproject.toml +++ b/tools/c7n_gcp/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "retrying<2.0.0,>=1.3.3", ] name = "c7n_gcp" -version = "0.4.51a5.post1+a83dda5c9" +version = "0.4.51a5.post2+a83dda5c9" description = "Cloud Custodian - Google Cloud Provider" readme = "readme.md" classifiers = [ diff --git a/tools/c7n_kube/pyproject.toml b/tools/c7n_kube/pyproject.toml index 744c9beb1b0..39d3db3786b 100644 --- a/tools/c7n_kube/pyproject.toml +++ b/tools/c7n_kube/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "kubernetes>=10.0.1", ] name = "c7n_kube" -version = "0.2.51a5.post1+a83dda5c9" +version = "0.2.51a5.post2+a83dda5c9" description = "Cloud Custodian - Kubernetes Provider" readme = "readme.md" classifiers = [ diff --git a/tools/c7n_left/pyproject.toml b/tools/c7n_left/pyproject.toml index f5a2e88ddff..8f1dfe3c8f6 100644 --- a/tools/c7n_left/pyproject.toml +++ b/tools/c7n_left/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "tfparse<1.0,>=0.6", ] name = "c7n_left" -version = "0.3.38a5.post1+a83dda5c9" +version = "0.3.38a5.post2+a83dda5c9" description = "Custodian policies for IAAC definitions" readme = "README.md" classifiers = [ diff --git a/tools/c7n_mailer/pyproject.toml b/tools/c7n_mailer/pyproject.toml index 4f63b5407b7..1a9b4f2fd20 100644 --- a/tools/c7n_mailer/pyproject.toml +++ b/tools/c7n_mailer/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "types-six<2.0.0,>=1.16.10", ] name = "c7n_mailer" -version = "0.6.51a5.post1+a83dda5c9" +version = "0.6.51a5.post2+a83dda5c9" description = "Cloud Custodian - Reference Mailer" readme = "README.md" classifiers = [ diff --git a/tools/c7n_oci/pyproject.toml b/tools/c7n_oci/pyproject.toml index fb7f8e8ecd8..6e5731cbce7 100644 --- a/tools/c7n_oci/pyproject.toml +++ b/tools/c7n_oci/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "pyopenssl>=24" ] name = "c7n_oci" -version = "0.1.26a5.post1+a83dda5c9" +version = "0.1.26a5.post2+a83dda5c9" description = "Cloud Custodian - OCI Support" readme = "readme.md" classifiers = [ diff --git a/tools/c7n_openstack/pyproject.toml b/tools/c7n_openstack/pyproject.toml index ac1e7c6f42e..d5ba37ece40 100644 --- a/tools/c7n_openstack/pyproject.toml +++ b/tools/c7n_openstack/pyproject.toml @@ -9,7 +9,7 @@ dependencies = [ "openstacksdk==0.52.0", ] name = "c7n_openstack" -version = "0.1.42a5.post1+a83dda5c9" +version = "0.1.42a5.post2+a83dda5c9" description = "Cloud Custodian - OpenStack Provider" readme = "readme.md" classifiers = [ diff --git a/tools/c7n_org/pyproject.toml b/tools/c7n_org/pyproject.toml index 4b22cb92c9e..72d594dc30c 100644 --- a/tools/c7n_org/pyproject.toml +++ b/tools/c7n_org/pyproject.toml @@ -9,7 +9,7 @@ dependencies = [ "click>=8.0", ] name = "c7n_org" -version = "0.6.51a5.post1+a83dda5c9" +version = "0.6.51a5.post2+a83dda5c9" description = "Cloud Custodian - Parallel Execution" readme = "README.md" classifiers = [ diff --git a/tools/c7n_policystream/pyproject.toml b/tools/c7n_policystream/pyproject.toml index c155712dcbe..e3d170f2924 100644 --- a/tools/c7n_policystream/pyproject.toml +++ b/tools/c7n_policystream/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "requests<3.0.0,>=2.22.0", ] name = "c7n_policystream" -version = "0.4.51a5.post1+a83dda5c9" +version = "0.4.51a5.post2+a83dda5c9" description = "Cloud Custodian - Git Commits as Logical Policy Changes" readme = "README.md" classifiers = [ diff --git a/tools/c7n_sphinxext/pyproject.toml b/tools/c7n_sphinxext/pyproject.toml index 19e304aad68..4fbbdef03c6 100644 --- a/tools/c7n_sphinxext/pyproject.toml +++ b/tools/c7n_sphinxext/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "sphinx-rtd-theme>=1.0.0", ] name = "c7n_sphinxext" -version = "1.1.51a5.post1+a83dda5c9" +version = "1.1.51a5.post2+a83dda5c9" description = "Cloud Custodian - Sphinx Extensions" classifiers = [ "License :: OSI Approved :: Apache Software License", diff --git a/tools/c7n_tencentcloud/pyproject.toml b/tools/c7n_tencentcloud/pyproject.toml index 152bbdf15f1..0916532ed4e 100644 --- a/tools/c7n_tencentcloud/pyproject.toml +++ b/tools/c7n_tencentcloud/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "tencentcloud-sdk-python<4.0.0,>=3.0.783", ] name = "c7n_tencentcloud" -version = "0.1.34a5.post1+a83dda5c9" +version = "0.1.34a5.post2+a83dda5c9" description = "Cloud Custodian - Tencent Cloud Provider" readme = "readme.md" diff --git a/uv.lock b/uv.lock index e095edb72f4..275ec2a2199 100644 --- a/uv.lock +++ b/uv.lock @@ -1488,7 +1488,7 @@ wheels = [ [[package]] name = "c7n" -version = "0.9.52a5.post1+a83dda5c9" +version = "0.9.52a5.post2+a83dda5c9" source = { editable = "." } dependencies = [ { name = "argcomplete" }, @@ -1577,7 +1577,7 @@ lint = [ [[package]] name = "c7n-awscc" -version = "0.1.37a5.post1+a83dda5c9" +version = "0.1.37a5.post2+a83dda5c9" source = { editable = "tools/c7n_awscc" } dependencies = [ { name = "boto3" }, @@ -1602,7 +1602,7 @@ dev = [{ name = "pytest" }] [[package]] name = "c7n-azure" -version = "0.7.51a5.post1+a83dda5c9" +version = "0.7.51a5.post2+a83dda5c9" source = { editable = "tools/c7n_azure" } dependencies = [ { name = "adal" }, @@ -1785,7 +1785,7 @@ dev = [ [[package]] name = "c7n-gcp" -version = "0.4.51a5.post1+a83dda5c9" +version = "0.4.51a5.post2+a83dda5c9" source = { editable = "tools/c7n_gcp" } dependencies = [ { name = "c7n" }, @@ -1826,7 +1826,7 @@ dev = [ [[package]] name = "c7n-kube" -version = "0.2.51a5.post1+a83dda5c9" +version = "0.2.51a5.post2+a83dda5c9" source = { editable = "tools/c7n_kube" } dependencies = [ { name = "c7n" }, @@ -1855,7 +1855,7 @@ dev = [ [[package]] name = "c7n-left" -version = "0.3.38a5.post1+a83dda5c9" +version = "0.3.38a5.post2+a83dda5c9" source = { editable = "tools/c7n_left" } dependencies = [ { name = "c7n" }, @@ -1888,7 +1888,7 @@ dev = [ [[package]] name = "c7n-mailer" -version = "0.6.51a5.post1+a83dda5c9" +version = "0.6.51a5.post2+a83dda5c9" source = { editable = "tools/c7n_mailer" } dependencies = [ { name = "boto3" }, @@ -1954,7 +1954,7 @@ dev = [ [[package]] name = "c7n-oci" -version = "0.1.26a5.post1+a83dda5c9" +version = "0.1.26a5.post2+a83dda5c9" source = { editable = "tools/c7n_oci" } dependencies = [ { name = "c7n" }, @@ -1979,7 +1979,7 @@ dev = [{ name = "pytest" }] [[package]] name = "c7n-openstack" -version = "0.1.42a5.post1+a83dda5c9" +version = "0.1.42a5.post2+a83dda5c9" source = { editable = "tools/c7n_openstack" } dependencies = [ { name = "c7n" }, @@ -2006,7 +2006,7 @@ dev = [ [[package]] name = "c7n-org" -version = "0.6.51a5.post1+a83dda5c9" +version = "0.6.51a5.post2+a83dda5c9" source = { editable = "tools/c7n_org" } dependencies = [ { name = "c7n" }, @@ -2029,7 +2029,7 @@ dev = [{ name = "pytest" }] [[package]] name = "c7n-policystream" -version = "0.4.51a5.post1+a83dda5c9" +version = "0.4.51a5.post2+a83dda5c9" source = { editable = "tools/c7n_policystream" } dependencies = [ { name = "boto3" }, @@ -2065,7 +2065,7 @@ dev = [ [[package]] name = "c7n-sphinxext" -version = "1.1.51a5.post1+a83dda5c9" +version = "1.1.51a5.post2+a83dda5c9" source = { editable = "tools/c7n_sphinxext" } dependencies = [ { name = "c7n" }, @@ -2092,7 +2092,7 @@ requires-dist = [ [[package]] name = "c7n-tencentcloud" -version = "0.1.34a5.post1+a83dda5c9" +version = "0.1.34a5.post2+a83dda5c9" source = { editable = "tools/c7n_tencentcloud" } dependencies = [ { name = "c7n" },