diff --git a/pkg/s3.go b/pkg/s3.go index 3e6ba54b..3a2daa82 100644 --- a/pkg/s3.go +++ b/pkg/s3.go @@ -69,6 +69,9 @@ var ( // VPC Flow Logs // source: https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-s3.html#flow-logs-s3-path // format: bucket-and-optional-prefix/AWSLogs/account_id/vpcflowlogs/region/year/month/day/aws_account_id_vpcflowlogs_region_flow_log_id_YYYYMMDDTHHmmZ_hash.log.gz + // VPC Flow Logs Hourly Format + // source: https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-s3-path.html#flow-logs-s3-path + // format: bucket-and-optional-prefix/AWSLogs/account_id/vpcflowlogs/region/year/month/day/hour/aws_account_id_vpcflowlogs_region_flow_log_id_YYYYMMDDTHHmmZ_hash.log.gz // example: 123456789012_vpcflowlogs_us-east-1_fl-1234abcd_20180620T1620Z_fe123456.log.gz // CloudTrail // source: https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-log-file-examples.html#cloudtrail-log-filename-format @@ -92,7 +95,7 @@ var ( // source: https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerLogs.html // format: aws-account-id/region/bucket-name/year/month/day/timestamp-hash // example: 123456789012/us-west-2/amzn-s3-demo-source-bucket/2023/03/01/2023-03-01-21-32-16-E568B2907131C0C0 - defaultFilenameRegex = regexp.MustCompile(`AWSLogs\/(?P\d+)\/(?P[a-zA-Z0-9_\-]+)\/(?P[\w-]+)\/(?P\d+)\/(?P\d+)\/(?P\d+)\/(?:health_check_log_)?\d+\_(?:elasticloadbalancing|vpcflowlogs)_(?:\w+-\w+-(?:\w+-)?\d)_(?:(?Papp|net)\.*?)?(?P[a-zA-Z0-9\-]+)`) + defaultFilenameRegex = regexp.MustCompile(`AWSLogs\/(?P\d+)\/(?P[a-zA-Z0-9_\-]+)\/(?P[\w-]+)\/(?P\d+)\/(?P\d+)\/(?P\d+)\/(?:(?P\d+)\/)?(?:health_check_log_)?\d+\_(?:elasticloadbalancing|vpcflowlogs)_(?:\w+-\w+-(?:\w+-)?\d)_(?:(?Papp|net)\.*?)?(?P[a-zA-Z0-9\-]+)`) defaultTimestampRegex = regexp.MustCompile(`(?P\d+-\d+-\d+T\d+:\d+:\d+(?:\.\d+Z)?)`) cloudtrailFilenameRegex = regexp.MustCompile(`AWSLogs\/(?Po-[a-z0-9]{10,32})?\/?(?P\d+)\/(?P[a-zA-Z0-9_\-]+)\/(?P[\w-]+)\/(?P\d+)\/(?P\d+)\/(?P\d+)\/\d+\_(?:CloudTrail|CloudTrail-Digest)_(?:\w+-\w+-(?:\w+-)?\d)_(?:(?:app|nlb|net)\.*?)?.+_(?P[a-zA-Z0-9\-]+)`) cloudfrontFilenameRegex = regexp.MustCompile(`(?P.*)\/(?P[A-Z0-9]+)\.(?P\d+)-(?P\d+)-(?P\d+)-(.+)`) diff --git a/pkg/s3_test.go b/pkg/s3_test.go index ddcf80bb..401ad5ac 100644 --- a/pkg/s3_test.go +++ b/pkg/s3_test.go @@ -258,6 +258,40 @@ func Test_getLabels(t *testing.T) { }, wantErr: false, }, + { + name: "s3_hourly_flow_logs", + args: args{ + record: events.S3EventRecord{ + AWSRegion: "us-east-1", + S3: events.S3Entity{ + Bucket: events.S3Bucket{ + Name: "vpc_logs_test", + OwnerIdentity: events.S3UserIdentity{ + PrincipalID: "test", + }, + }, + Object: events.S3Object{ + Key: "my-bucket/AWSLogs/123456789012/vpcflowlogs/us-east-1/2026/08/06/20/123456789012_vpcflowlogs_us-east-1_fl-1234abcd_20260806T2020Z_fe123456.log.gz", + }, + }, + }, + }, + want: map[string]string{ + "account_id": "123456789012", + "bucket": "vpc_logs_test", + "bucket_owner": "test", + "bucket_region": "us-east-1", + "hour": "20", + "day": "06", + "key": "my-bucket/AWSLogs/123456789012/vpcflowlogs/us-east-1/2026/08/06/20/123456789012_vpcflowlogs_us-east-1_fl-1234abcd_20260806T2020Z_fe123456.log.gz", + "month": "08", + "region": "us-east-1", + "src": "fl-1234abcd", + "type": FlowLogType, + "year": "2026", + }, + wantErr: false, + }, { name: "cloudtrail_digest_logs", args: args{