From f6fa8518bb18acb1c5bde78ae74d555ac07268ea Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:24:10 -0800 Subject: [PATCH 01/25] codebuild terraform module --- .../aws/modules/codebuild/v001/codebuild.tf | 125 ++++++++++++++++++ .../aws/modules/codebuild/v001/outputs.tf | 7 + .../aws/modules/codebuild/v001/variables.tf | 11 ++ 3 files changed, 143 insertions(+) create mode 100644 infra/terraform/aws/modules/codebuild/v001/codebuild.tf create mode 100644 infra/terraform/aws/modules/codebuild/v001/outputs.tf create mode 100644 infra/terraform/aws/modules/codebuild/v001/variables.tf diff --git a/infra/terraform/aws/modules/codebuild/v001/codebuild.tf b/infra/terraform/aws/modules/codebuild/v001/codebuild.tf new file mode 100644 index 000000000..00075e5d3 --- /dev/null +++ b/infra/terraform/aws/modules/codebuild/v001/codebuild.tf @@ -0,0 +1,125 @@ +resource "aws_codebuild_project" "default" { + name = var.project_name + service_role = aws_iam_role.codebuild.arn + + artifacts { + type = "CODEPIPELINE" + } + + environment { + compute_type = var.compute_type + image = "aws/codebuild/standard:7.0" + type = "LINUX_CONTAINER" + privileged_mode = true + + environment_variable { + name = "SERVICE_NAME" + value = var.service_name + } + environment_variable { + name = "ENV_ID" + value = var.env_id + } + environment_variable { + name = "ENV" + value = var.env + } + environment_variable { + name = "AWS_ACCOUNT_ID" + value = var.aws_account_id + } + environment_variable { + name = "REGION" + value = var.aws_region + } + environment_variable { + name = "RUN_TESTS" + value = "true" + } + environment_variable { + name = "PUSH_IMAGE" + value = "true" + } + environment_variable { + name = "DEPLOY" + value = "false" + } + } + + source { + type = "CODEPIPELINE" + buildspec = var.buildspec + } +} + +resource "aws_iam_role" "codebuild" { + name = var.project_name + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "codebuild.amazonaws.com" + } + } + ] + }) +} + +resource "aws_iam_role_policy" "codebuild" { + name = var.project_name + role = aws_iam_role.codebuild.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = "*" + }, + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion" + ] + Resource = "${var.artifacts_bucket_arn}/*" + }, + { + Effect = "Allow" + Action = [ + "ecr:GetAuthorizationToken" + ] + Resource = "*" + }, + { + Effect = "Allow" + Action = [ + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:PutImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload" + ] + Resource = var.ecr_repository_arn + }, + { + Effect = "Allow" + Action = [ + "lambda:UpdateFunctionCode" + ] + Resource = var.lambda_function_arn + } + ] + }) +} diff --git a/infra/terraform/aws/modules/codebuild/v001/outputs.tf b/infra/terraform/aws/modules/codebuild/v001/outputs.tf new file mode 100644 index 000000000..3efe80117 --- /dev/null +++ b/infra/terraform/aws/modules/codebuild/v001/outputs.tf @@ -0,0 +1,7 @@ +output "project_name" { + value = aws_codebuild_project.default.name +} + +output "project_arn" { + value = aws_codebuild_project.default.arn +} diff --git a/infra/terraform/aws/modules/codebuild/v001/variables.tf b/infra/terraform/aws/modules/codebuild/v001/variables.tf new file mode 100644 index 000000000..e10579a69 --- /dev/null +++ b/infra/terraform/aws/modules/codebuild/v001/variables.tf @@ -0,0 +1,11 @@ +variable "project_name" {} +variable "service_name" {} +variable "env" {} +variable "env_id" {} +variable "aws_region" {} +variable "aws_account_id" {} +variable "compute_type" {} +variable "buildspec" {} +variable "artifacts_bucket_arn" {} +variable "ecr_repository_arn" {} +variable "lambda_function_arn" {} From eb92fa13132b47fa1d57244b679d612e28769cb9 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:24:34 -0800 Subject: [PATCH 02/25] codepipeline terraform module --- .../modules/codepipeline/v001/codepipeline.tf | 158 ++++++++++++++++++ .../modules/codepipeline/v001/variables.tf | 7 + 2 files changed, 165 insertions(+) create mode 100644 infra/terraform/aws/modules/codepipeline/v001/codepipeline.tf create mode 100644 infra/terraform/aws/modules/codepipeline/v001/variables.tf diff --git a/infra/terraform/aws/modules/codepipeline/v001/codepipeline.tf b/infra/terraform/aws/modules/codepipeline/v001/codepipeline.tf new file mode 100644 index 000000000..e1906e72b --- /dev/null +++ b/infra/terraform/aws/modules/codepipeline/v001/codepipeline.tf @@ -0,0 +1,158 @@ +locals { + ID_ENV = "${var.env_id}-${var.env}" + PROJECT_CONF_FILE_NAME = "project.yaml" + PROJECT_CONF = yamldecode(file("../../../../../${local.PROJECT_CONF_FILE_NAME}")) + CODEPIPELINE_ENV_VAR = local.PROJECT_CONF.infra.terraform.aws.modules.codepipeline.env_var.set + SERVICES_ZIP = local.PROJECT_CONF.scripts.env_var.set.SERVICES_ZIP.default + BUILD_OBJECT_KEY_PATH = local.CODEPIPELINE_ENV_VAR.BUILD_OBJECT_KEY_PATH.default + BUILD_SOURCE_LOCATION = "${local.BUILD_OBJECT_KEY_PATH}/${local.SERVICES_ZIP}" +} + +resource "aws_codepipeline" "build" { + name = "mxfactorial-build-${local.ID_ENV}" + role_arn = aws_iam_role.codepipeline.arn + pipeline_type = "V2" + + artifact_store { + location = var.artifacts_bucket_name + type = "S3" + } + + stage { + name = "Source" + action { + name = "S3Source" + category = "Source" + owner = "AWS" + provider = "S3" + version = "1" + output_artifacts = ["source_output"] + configuration = { + S3Bucket = var.artifacts_bucket_name + S3ObjectKey = local.BUILD_SOURCE_LOCATION + PollForSourceChanges = false + } + } + } + + stage { + name = "Build" + + dynamic "action" { + for_each = var.codebuild_project_names + content { + name = action.key + category = "Build" + owner = "AWS" + provider = "CodeBuild" + input_artifacts = ["source_output"] + version = "1" + run_order = 1 + configuration = { + ProjectName = action.value + } + } + } + } +} + +resource "aws_cloudwatch_event_rule" "s3_trigger" { + name = "mxfactorial-build-trigger-${local.ID_ENV}" + + event_pattern = jsonencode({ + source = ["aws.s3"] + detail-type = ["Object Created"] + detail = { + bucket = { name = [var.artifacts_bucket_name] } + object = { key = [{ prefix = local.BUILD_SOURCE_LOCATION }] } + } + }) +} + +resource "aws_cloudwatch_event_target" "codepipeline" { + rule = aws_cloudwatch_event_rule.s3_trigger.name + arn = aws_codepipeline.build.arn + role_arn = aws_iam_role.eventbridge_pipeline.arn +} + +resource "aws_iam_role" "codepipeline" { + name = "codepipeline-build-${local.ID_ENV}" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "codepipeline.amazonaws.com" + } + } + ] + }) +} + +resource "aws_iam_role_policy" "codepipeline" { + name = "codepipeline-build-${local.ID_ENV}" + role = aws_iam_role.codepipeline.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:GetBucketVersioning", + "s3:PutObject" + ] + Resource = [ + var.artifacts_bucket_arn, + "${var.artifacts_bucket_arn}/*" + ] + }, + { + Effect = "Allow" + Action = [ + "codebuild:BatchGetBuilds", + "codebuild:StartBuild" + ] + Resource = "*" + } + ] + }) +} + +resource "aws_iam_role" "eventbridge_pipeline" { + name = "eventbridge-pipeline-${local.ID_ENV}" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "events.amazonaws.com" + } + } + ] + }) +} + +resource "aws_iam_role_policy" "eventbridge_pipeline" { + name = "eventbridge-pipeline-${local.ID_ENV}" + role = aws_iam_role.eventbridge_pipeline.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = "codepipeline:StartPipelineExecution" + Resource = aws_codepipeline.build.arn + } + ] + }) +} diff --git a/infra/terraform/aws/modules/codepipeline/v001/variables.tf b/infra/terraform/aws/modules/codepipeline/v001/variables.tf new file mode 100644 index 000000000..a0831601e --- /dev/null +++ b/infra/terraform/aws/modules/codepipeline/v001/variables.tf @@ -0,0 +1,7 @@ +variable "env" {} +variable "env_id" {} +variable "artifacts_bucket_name" {} +variable "artifacts_bucket_arn" {} +variable "codebuild_project_names" { + type = map(string) +} From 8c623cf7159b282e0fb56c043af225e0a35d19c5 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:28:11 -0800 Subject: [PATCH 03/25] add codebuild project for each ecr repo --- .../modules/ecr/v001/buildspecs/build.yaml | 32 +++++++++++++++++++ .../aws/modules/ecr/v001/codebuild.tf | 19 +++++++++++ .../terraform/aws/modules/ecr/v001/outputs.tf | 11 +++++++ .../aws/modules/ecr/v001/variables.tf | 8 +++-- 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 infra/terraform/aws/modules/ecr/v001/buildspecs/build.yaml create mode 100644 infra/terraform/aws/modules/ecr/v001/codebuild.tf create mode 100644 infra/terraform/aws/modules/ecr/v001/outputs.tf diff --git a/infra/terraform/aws/modules/ecr/v001/buildspecs/build.yaml b/infra/terraform/aws/modules/ecr/v001/buildspecs/build.yaml new file mode 100644 index 000000000..921613e0b --- /dev/null +++ b/infra/terraform/aws/modules/ecr/v001/buildspecs/build.yaml @@ -0,0 +1,32 @@ +version: 0.2 + +phases: + pre_build: + commands: + - aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com + - export SHORT_SHA=$(echo ${CODEBUILD_RESOLVED_SOURCE_VERSION:-$CODEBUILD_BUILD_ID} | sed 's/.*://' | cut -c1-7) + - export ECR_URI=$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$ENV_ID/$ENV/$SERVICE_NAME + build: + commands: + - echo "building $SERVICE_NAME (RUN_TESTS=$RUN_TESTS, PUSH_IMAGE=$PUSH_IMAGE)..." + - docker build --build-arg RUN_TESTS=$RUN_TESTS -t $SERVICE_NAME:$SHORT_SHA -f docker/$SERVICE_NAME.Dockerfile . + - docker tag $SERVICE_NAME:$SHORT_SHA $ECR_URI:$SHORT_SHA + - docker tag $SERVICE_NAME:$SHORT_SHA $ECR_URI:latest + post_build: + commands: + - | + if [ "$PUSH_IMAGE" = "true" ]; then + echo "pushing $SERVICE_NAME..." + docker push $ECR_URI:$SHORT_SHA + docker push $ECR_URI:latest + else + echo "skipping push (PUSH_IMAGE=$PUSH_IMAGE)" + fi + - | + if [ "$DEPLOY" = "true" ]; then + echo "deploying $SERVICE_NAME to lambda..." + aws lambda update-function-code \ + --function-name $SERVICE_NAME-$ENV_ID-$ENV \ + --image-uri $ECR_URI:latest \ + --region $REGION + fi diff --git a/infra/terraform/aws/modules/ecr/v001/codebuild.tf b/infra/terraform/aws/modules/ecr/v001/codebuild.tf new file mode 100644 index 000000000..7dd30cc70 --- /dev/null +++ b/infra/terraform/aws/modules/ecr/v001/codebuild.tf @@ -0,0 +1,19 @@ +locals { + ID_ENV = "${var.env_id}-${var.env}" +} + +module "codebuild" { + source = "../../codebuild/v001" + + project_name = "mxfactorial-${var.service_name}-${local.ID_ENV}" + service_name = var.service_name + env = var.env + env_id = var.env_id + aws_region = var.aws_region + aws_account_id = var.aws_account_id + compute_type = var.codebuild_compute_type + buildspec = file("${path.module}/buildspecs/build.yaml") + artifacts_bucket_arn = var.artifacts_bucket_arn + ecr_repository_arn = aws_ecr_repository.default.arn + lambda_function_arn = "arn:aws:lambda:${var.aws_region}:${var.aws_account_id}:function:${var.service_name}-${local.ID_ENV}" +} diff --git a/infra/terraform/aws/modules/ecr/v001/outputs.tf b/infra/terraform/aws/modules/ecr/v001/outputs.tf new file mode 100644 index 000000000..ae672401f --- /dev/null +++ b/infra/terraform/aws/modules/ecr/v001/outputs.tf @@ -0,0 +1,11 @@ +output "ecr_repository_url" { + value = aws_ecr_repository.default.repository_url +} + +output "codebuild_project_name" { + value = module.codebuild.project_name +} + +output "codebuild_project_arn" { + value = module.codebuild.project_arn +} diff --git a/infra/terraform/aws/modules/ecr/v001/variables.tf b/infra/terraform/aws/modules/ecr/v001/variables.tf index b841b14be..210fd1e81 100644 --- a/infra/terraform/aws/modules/ecr/v001/variables.tf +++ b/infra/terraform/aws/modules/ecr/v001/variables.tf @@ -1,8 +1,12 @@ variable "env" {} variable "env_id" {} variable "force_delete" { - type = bool + type = bool default = false } variable "service_name" {} -variable "max_image_storage_count" {} \ No newline at end of file +variable "max_image_storage_count" {} +variable "artifacts_bucket_arn" {} +variable "aws_region" {} +variable "aws_account_id" {} +variable "codebuild_compute_type" {} \ No newline at end of file From e0442b876b66a9b9c66c5223c334fe982144e30f Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:33:39 -0800 Subject: [PATCH 04/25] add build projects and pipeline in storage module --- .../modules/project-storage/v001/README.md | 131 ++++++++++++++++++ .../project-storage/v001/codepipeline.tf | 8 ++ .../aws/modules/project-storage/v001/ecr.tf | 7 + .../modules/project-storage/v001/locals.tf | 13 +- .../modules/project-storage/v001/outputs.tf | 11 ++ .../aws/modules/project-storage/v001/s3.tf | 12 ++ .../modules/project-storage/v001/variables.tf | 7 +- 7 files changed, 181 insertions(+), 8 deletions(-) create mode 100644 infra/terraform/aws/modules/project-storage/v001/README.md create mode 100644 infra/terraform/aws/modules/project-storage/v001/codepipeline.tf create mode 100644 infra/terraform/aws/modules/project-storage/v001/outputs.tf diff --git a/infra/terraform/aws/modules/project-storage/v001/README.md b/infra/terraform/aws/modules/project-storage/v001/README.md new file mode 100644 index 000000000..3b58b67b2 --- /dev/null +++ b/infra/terraform/aws/modules/project-storage/v001/README.md @@ -0,0 +1,131 @@ +

+ systemaccounting +

+ +### project-storage + +resources provisioned in init-dev for dev environment + +#### ENV_ID + +assigned in `.env` at project root + +#### naming conventions + +- `infra/terraform/aws/modules/project-storage/v001/s3.tf` +- `infra/terraform/aws/modules/project-storage/v001/codepipeline.tf` +- `infra/terraform/aws/modules/project-storage/v001/integ.tf` +- `infra/terraform/aws/modules/ecr/v001/ecr.tf` +- `infra/terraform/aws/modules/ecr/v001/codebuild.tf` +- `infra/terraform/aws/modules/codebuild/v001/codebuild.tf` +- `infra/terraform/aws/modules/codepipeline/v001/codepipeline.tf` + +#### manual deletion + +when `terraform destroy` fails, delete in order: + +1. codepipeline +1. eventbridge rule + target +1. codebuild projects (integ + 13 per-service) +1. ecr repos +1. s3 buckets (artifacts, tfstate) +1. iam roles + policies + +#### state list + +``` +module.project_storage_dev.aws_s3_bucket.artifacts +module.project_storage_dev.aws_s3_bucket.tfstate +module.project_storage_dev.aws_s3_bucket_notification.artifacts_eventbridge +module.project_storage_dev.aws_s3_bucket_versioning.artifacts + +module.project_storage_dev.aws_codebuild_project.integ +module.project_storage_dev.aws_iam_role.integ +module.project_storage_dev.aws_iam_role_policy.integ + +module.project_storage_dev.module.codepipeline.aws_codepipeline.build +module.project_storage_dev.module.codepipeline.aws_cloudwatch_event_rule.s3_trigger +module.project_storage_dev.module.codepipeline.aws_cloudwatch_event_target.codepipeline +module.project_storage_dev.module.codepipeline.aws_iam_role.codepipeline +module.project_storage_dev.module.codepipeline.aws_iam_role.eventbridge_pipeline +module.project_storage_dev.module.codepipeline.aws_iam_role_policy.codepipeline +module.project_storage_dev.module.codepipeline.aws_iam_role_policy.eventbridge_pipeline + +module.project_storage_dev.module.ecr_repos["auto-confirm"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["auto-confirm"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["auto-confirm"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["auto-confirm"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["auto-confirm"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["balance-by-account"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["balance-by-account"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["balance-by-account"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["balance-by-account"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["balance-by-account"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["client"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["client"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["client"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["client"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["client"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["go-migrate"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["go-migrate"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["go-migrate"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["go-migrate"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["go-migrate"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["graphql"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["graphql"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["graphql"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["graphql"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["graphql"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["request-approve"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["request-approve"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["request-approve"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["request-approve"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["request-approve"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["request-by-id"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["request-by-id"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["request-by-id"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["request-by-id"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["request-by-id"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["request-create"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["request-create"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["request-create"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["request-create"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["request-create"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["requests-by-account"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["requests-by-account"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["requests-by-account"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["requests-by-account"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["requests-by-account"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["rule"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["rule"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["rule"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["rule"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["rule"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["transaction-by-id"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["transaction-by-id"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["transaction-by-id"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["transaction-by-id"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["transaction-by-id"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["transactions-by-account"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["transactions-by-account"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["transactions-by-account"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["transactions-by-account"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["transactions-by-account"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["warm-cache"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["warm-cache"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["warm-cache"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["warm-cache"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["warm-cache"].module.codebuild.aws_iam_role_policy.codebuild +``` diff --git a/infra/terraform/aws/modules/project-storage/v001/codepipeline.tf b/infra/terraform/aws/modules/project-storage/v001/codepipeline.tf new file mode 100644 index 000000000..a7c7232c1 --- /dev/null +++ b/infra/terraform/aws/modules/project-storage/v001/codepipeline.tf @@ -0,0 +1,8 @@ +module "codepipeline" { + source = "../../codepipeline/v001" + env = var.env + env_id = var.env_id + artifacts_bucket_name = aws_s3_bucket.artifacts.bucket + artifacts_bucket_arn = aws_s3_bucket.artifacts.arn + codebuild_project_names = { for k, v in module.ecr_repos : k => v.codebuild_project_name } +} diff --git a/infra/terraform/aws/modules/project-storage/v001/ecr.tf b/infra/terraform/aws/modules/project-storage/v001/ecr.tf index 5b2967e0b..978918691 100644 --- a/infra/terraform/aws/modules/project-storage/v001/ecr.tf +++ b/infra/terraform/aws/modules/project-storage/v001/ecr.tf @@ -1,3 +1,6 @@ +data "aws_caller_identity" "current" {} +data "aws_region" "current" {} + module "ecr_repos" { for_each = local.ECR_REPOS source = "../../ecr/v001" @@ -6,4 +9,8 @@ module "ecr_repos" { env_id = var.env_id service_name = each.value force_delete = var.force_destroy_storage + artifacts_bucket_arn = aws_s3_bucket.artifacts.arn + aws_region = data.aws_region.current.id + aws_account_id = data.aws_caller_identity.current.account_id + codebuild_compute_type = var.codebuild_compute_type } diff --git a/infra/terraform/aws/modules/project-storage/v001/locals.tf b/infra/terraform/aws/modules/project-storage/v001/locals.tf index 331d4efc2..2c7beeb3f 100644 --- a/infra/terraform/aws/modules/project-storage/v001/locals.tf +++ b/infra/terraform/aws/modules/project-storage/v001/locals.tf @@ -1,9 +1,12 @@ locals { - ID_ENV = "${var.env_id}-${var.env}" - PROJECT_CONF_FILE_NAME = "project.yaml" - PROJECT_CONF = yamldecode(file("../../../../../${local.PROJECT_CONF_FILE_NAME}")) - STORAGE_ENV_VAR = local.PROJECT_CONF.infra.terraform.aws.modules.project-storage.env_var.set - ID_ENV_PREFIX = "${var.env_id}/${var.env}" + ID_ENV = "${var.env_id}-${var.env}" + PROJECT_CONF_FILE_NAME = "project.yaml" + PROJECT_CONF = yamldecode(file("../../../../../${local.PROJECT_CONF_FILE_NAME}")) + STORAGE_ENV_VAR = local.PROJECT_CONF.infra.terraform.aws.modules.project-storage.env_var.set + ID_ENV_PREFIX = "${var.env_id}/${var.env}" + SERVICES_ZIP = local.PROJECT_CONF.scripts.env_var.set.SERVICES_ZIP.default + INTEG_TEST_OBJECT_KEY_PATH = local.STORAGE_ENV_VAR.INTEG_TEST_OBJECT_KEY_PATH.default + INTEG_SOURCE_LOCATION = "${local.INTEG_TEST_OBJECT_KEY_PATH}/${local.SERVICES_ZIP}" // add a terraform_data precondition to fail // if a service is not found in project.yaml diff --git a/infra/terraform/aws/modules/project-storage/v001/outputs.tf b/infra/terraform/aws/modules/project-storage/v001/outputs.tf new file mode 100644 index 000000000..9bed9d3b3 --- /dev/null +++ b/infra/terraform/aws/modules/project-storage/v001/outputs.tf @@ -0,0 +1,11 @@ +output "artifacts_bucket_name" { + value = aws_s3_bucket.artifacts.bucket +} + +output "artifacts_bucket_arn" { + value = aws_s3_bucket.artifacts.arn +} + +output "codebuild_project_names" { + value = { for k, v in module.ecr_repos : k => v.codebuild_project_name } +} diff --git a/infra/terraform/aws/modules/project-storage/v001/s3.tf b/infra/terraform/aws/modules/project-storage/v001/s3.tf index b61e47794..7978bab23 100644 --- a/infra/terraform/aws/modules/project-storage/v001/s3.tf +++ b/infra/terraform/aws/modules/project-storage/v001/s3.tf @@ -3,6 +3,18 @@ resource "aws_s3_bucket" "artifacts" { force_destroy = var.force_destroy_storage } +resource "aws_s3_bucket_versioning" "artifacts" { + bucket = aws_s3_bucket.artifacts.id + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_notification" "artifacts_eventbridge" { + bucket = aws_s3_bucket.artifacts.id + eventbridge = true +} + resource "aws_s3_bucket" "tfstate" { bucket = "${var.tfstate_bucket_name_prefix}-${local.ID_ENV}" force_destroy = var.force_destroy_storage diff --git a/infra/terraform/aws/modules/project-storage/v001/variables.tf b/infra/terraform/aws/modules/project-storage/v001/variables.tf index 286d580db..f9cf99510 100644 --- a/infra/terraform/aws/modules/project-storage/v001/variables.tf +++ b/infra/terraform/aws/modules/project-storage/v001/variables.tf @@ -3,7 +3,8 @@ variable "env_id" {} variable "artifacts_bucket_name_prefix" {} variable "tfstate_bucket_name_prefix" {} variable "force_destroy_storage" { - type = bool - default = false + type = bool + default = false } -variable "max_image_storage_count" {} \ No newline at end of file +variable "max_image_storage_count" {} +variable "codebuild_compute_type" {} \ No newline at end of file From 492d98c857980dffa07167824a53b24b5010e26f Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:34:07 -0800 Subject: [PATCH 05/25] add integration test build project --- .../aws/modules/project-storage/v001/integ.tf | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 infra/terraform/aws/modules/project-storage/v001/integ.tf diff --git a/infra/terraform/aws/modules/project-storage/v001/integ.tf b/infra/terraform/aws/modules/project-storage/v001/integ.tf new file mode 100644 index 000000000..aad65b11f --- /dev/null +++ b/infra/terraform/aws/modules/project-storage/v001/integ.tf @@ -0,0 +1,153 @@ +# integration test codebuild project +# runs full test suite with all services via docker compose + +resource "aws_codebuild_project" "integ" { + name = "mxfactorial-integ-${local.ID_ENV}" + service_role = aws_iam_role.integ.arn + + artifacts { + type = "NO_ARTIFACTS" + } + + environment { + compute_type = "BUILD_GENERAL1_LARGE" + image = "aws/codebuild/standard:7.0" + type = "LINUX_CONTAINER" + privileged_mode = true + + environment_variable { + name = "ENV_ID" + value = var.env_id + } + environment_variable { + name = "ENV" + value = var.env + } + environment_variable { + name = "AWS_ACCOUNT_ID" + value = data.aws_caller_identity.current.account_id + } + environment_variable { + name = "REGION" + value = data.aws_region.current.id + } + } + + source { + type = "S3" + location = "${aws_s3_bucket.artifacts.bucket}/${local.INTEG_SOURCE_LOCATION}" + buildspec = <<-EOF + version: 0.2 + phases: + install: + # codebuild standard:7.0 has node 18, but aws-sdk requires node >= 20 + runtime-versions: + nodejs: 20 + pre_build: + commands: + # install test dependencies + - apt-get update && apt-get install -y postgresql-client + - curl -L https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq + - | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + . $HOME/.cargo/env + rustup component add rustfmt clippy + - npx playwright install-deps + # pull pre-built service images from ecr and tag as latest for docker compose + - aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com + - export ECR_URI=$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$ENV_ID/$ENV + - | + for SVC in ${join(" ", local.ECR_REPOS)}; do + echo "pulling $SVC..." + TAG=$(aws ecr describe-images --repository-name $ENV_ID/$ENV/$SVC --region $REGION --query 'sort_by(imageDetails,&imagePushedAt)[-1].imageTags[0]' --output text 2>/dev/null || echo "") + if [ -n "$TAG" ] && [ "$TAG" != "None" ]; then + docker pull $ECR_URI/$SVC:$TAG + docker tag $ECR_URI/$SVC:$TAG $SVC:latest + else + echo "skipping $SVC (no images)" + fi + done + build: + commands: + # build storage (postgres), then start all services with pre-built images + - docker compose -f docker/storage.yaml build + - docker compose -f docker/storage.yaml -f docker/services.yaml up -d --no-build + - until docker exec mxf-postgres-1 pg_isready -U postgres; do sleep 1; done + # run tests + - make --no-print-directory -C crates/pg test-db + - make --no-print-directory -C crates/redisclient test-cache + - make --no-print-directory -C tests test-local + - make --no-print-directory -C client test + post_build: + commands: + - docker compose -f docker/storage.yaml -f docker/services.yaml down + EOF + } +} + +resource "aws_iam_role" "integ" { + name = "mxfactorial-integ-${local.ID_ENV}" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "codebuild.amazonaws.com" + } + } + ] + }) +} + +resource "aws_iam_role_policy" "integ" { + name = "mxfactorial-integ-${local.ID_ENV}" + role = aws_iam_role.integ.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = "*" + }, + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion" + ] + Resource = "${aws_s3_bucket.artifacts.arn}/*" + }, + { + Effect = "Allow" + Action = ["s3:ListBucket"] + Resource = aws_s3_bucket.artifacts.arn + }, + { + Effect = "Allow" + Action = [ + "ecr:GetAuthorizationToken" + ] + Resource = "*" + }, + { + Effect = "Allow" + Action = [ + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:DescribeImages" + ] + Resource = "arn:aws:ecr:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:repository/${var.env_id}/${var.env}/*" + } + ] + }) +} From 462ce3bad833e8d34af18ca85d5e9a36e7381573 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:35:52 -0800 Subject: [PATCH 06/25] remove deprecation warning --- infra/terraform/aws/modules/provided-lambda/v001/lambda.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/terraform/aws/modules/provided-lambda/v001/lambda.tf b/infra/terraform/aws/modules/provided-lambda/v001/lambda.tf index f7f754aa5..b4d480bb0 100644 --- a/infra/terraform/aws/modules/provided-lambda/v001/lambda.tf +++ b/infra/terraform/aws/modules/provided-lambda/v001/lambda.tf @@ -85,7 +85,7 @@ resource "aws_iam_policy" "default" { Action = [ "logs:CreateLogGroup" ], - Resource = "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*" + Resource = "arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:*" }, { Sid = "${local.SERVICE_NAME_TITLE}LogEventPolicy${local.TITLED_ID_ENV}" @@ -95,7 +95,7 @@ resource "aws_iam_policy" "default" { "logs:PutLogEvents" ], Resource = [ - "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:${local.LOG_GROUP_NAME}:*" + "arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:${local.LOG_GROUP_NAME}:*" ] }], }) From d21199214443eb7a5b44285288bd1f69c0918053 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:36:39 -0800 Subject: [PATCH 07/25] configurable build runner size --- infra/terraform/aws/environments/init-dev/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/terraform/aws/environments/init-dev/main.tf b/infra/terraform/aws/environments/init-dev/main.tf index c1508415e..50a040b9d 100644 --- a/infra/terraform/aws/environments/init-dev/main.tf +++ b/infra/terraform/aws/environments/init-dev/main.tf @@ -28,4 +28,5 @@ module "project_storage_dev" { artifacts_bucket_name_prefix = local.STORAGE_ENV_VAR.ARTIFACTS_BUCKET_PREFIX.default tfstate_bucket_name_prefix = local.STORAGE_ENV_VAR.TFSTATE_BUCKET_PREFIX.default max_image_storage_count = 10 + codebuild_compute_type = "BUILD_GENERAL1_MEDIUM" # SMALL, MEDIUM, LARGE, 2XLARGE } From a24a0809dbf1855c2bd347bcecab9c86e57cf7b9 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:37:19 -0800 Subject: [PATCH 08/25] var naming --- infra/terraform/aws/environments/dev/main.tf | 4 ++-- infra/terraform/aws/environments/prod/main.tf | 4 ++-- .../aws/modules/region/v001/apigw-logging.tf | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/infra/terraform/aws/environments/dev/main.tf b/infra/terraform/aws/environments/dev/main.tf index b4a1249ae..717a1c0c5 100644 --- a/infra/terraform/aws/environments/dev/main.tf +++ b/infra/terraform/aws/environments/dev/main.tf @@ -7,7 +7,7 @@ locals { ARTIFACTS_PREFIX = local.STORAGE_ENV_VAR.ARTIFACTS_BUCKET_PREFIX.default TFSTATE_PREFIX = local.STORAGE_ENV_VAR.TFSTATE_BUCKET_PREFIX.default INFRA_ENV_VAR = local.PROJECT_CONF.infra.terraform.aws.modules.environment.env_var.set - RDS_PREFIX = local.INFRA_ENV_VAR.RDS_PREFIX.default + NAME_PREFIX = local.INFRA_ENV_VAR.NAME_PREFIX.default REGION = local.INFRA_ENV_VAR.REGION.default ENV_ID = module.env_id.ENV_ID ID_ENV = "${local.ENV_ID}-${local.ENV}" @@ -57,7 +57,7 @@ module "dev" { rds_instance_class = "db.t3.micro" rds_parameter_group = "default.postgres14" rds_engine_version = "14.17" - rds_instance_name = "${local.RDS_PREFIX}-${local.ID_ENV}" + rds_instance_name = "${local.NAME_PREFIX}-${local.ID_ENV}" db_snapshot_id = null ############### api gateway ############### diff --git a/infra/terraform/aws/environments/prod/main.tf b/infra/terraform/aws/environments/prod/main.tf index 393d373a2..b7be562c9 100644 --- a/infra/terraform/aws/environments/prod/main.tf +++ b/infra/terraform/aws/environments/prod/main.tf @@ -15,7 +15,7 @@ locals { ARTIFACTS_PREFIX = local.STORAGE_ENV_VAR.ARTIFACTS_BUCKET_PREFIX.default TFSTATE_PREFIX = local.STORAGE_ENV_VAR.TFSTATE_BUCKET_PREFIX.default INFRA_ENV_VAR = local.PROJECT_CONF.infra.terraform.aws.modules.environment.env_var.set - RDS_PREFIX = local.INFRA_ENV_VAR.RDS_PREFIX.default + NAME_PREFIX = local.INFRA_ENV_VAR.NAME_PREFIX.default REGION = local.INFRA_ENV_VAR.REGION.default ENV_ID = local.PROJECT_CONF.env_var.set.PROD_ENV_ID.default ID_ENV = "${local.ENV_ID}-${local.ENV}" @@ -67,7 +67,7 @@ module "prod" { rds_instance_class = "db.t3.micro" rds_parameter_group = "default.postgres13" rds_engine_version = "13.20" - rds_instance_name = "${local.RDS_PREFIX}-${local.ID_ENV}" + rds_instance_name = "${local.NAME_PREFIX}-${local.ID_ENV}" db_snapshot_id = null ############### api gateway ############### diff --git a/infra/terraform/aws/modules/region/v001/apigw-logging.tf b/infra/terraform/aws/modules/region/v001/apigw-logging.tf index 65b3ab034..4371df7fb 100644 --- a/infra/terraform/aws/modules/region/v001/apigw-logging.tf +++ b/infra/terraform/aws/modules/region/v001/apigw-logging.tf @@ -1,6 +1,6 @@ locals { - PROJECT_CONF = yamldecode(file("../../../../../project.yaml")) - GITHUB_REPO_NAME = local.PROJECT_CONF[".github"].env_var.set.GITHUB_REPO_NAME.default + PROJECT_CONF = yamldecode(file("../../../../../project.yaml")) + NAME_PREFIX = local.PROJECT_CONF.infra.terraform.aws.modules.environment.env_var.set.NAME_PREFIX.default } resource "aws_api_gateway_account" "apigw_logging" { @@ -13,7 +13,7 @@ resource "aws_iam_role" "apigw_logging" { Version = "2012-10-17" Statement = [ { - Sid = "${title(local.GITHUB_REPO_NAME)}APIGwTrustPolicy" + Sid = "${title(local.NAME_PREFIX)}APIGwTrustPolicy" Action = "sts:AssumeRole" Effect = "Allow" Principal = { @@ -25,13 +25,13 @@ resource "aws_iam_role" "apigw_logging" { } resource "aws_iam_policy" "apigw_logging" { - name = "${local.GITHUB_REPO_NAME}-apigw-logging" - description = "${local.GITHUB_REPO_NAME} apigw logging permission" + name = "${local.NAME_PREFIX}-apigw-logging" + description = "${local.NAME_PREFIX} apigw logging permission" policy = jsonencode({ Version = "2012-10-17" Statement = [ { - Sid = "${title(local.GITHUB_REPO_NAME)}LoggingPolicy" + Sid = "${title(local.NAME_PREFIX)}LoggingPolicy" Action = [ "logs:CreateLogGroup", "logs:CreateLogStream", From cd6be4d3c24654b5f1e132e6f949790f87e009a3 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:41:16 -0800 Subject: [PATCH 09/25] separate measure services in compose --- docker/measure.yaml | 48 +++++++++++++++++++++++++++++++++++++++++++ docker/services.yaml | 49 +++----------------------------------------- make/docker.mk | 4 ++++ migrations/makefile | 33 ++++++++--------------------- scripts/compose.sh | 14 +++++++++++-- 5 files changed, 75 insertions(+), 73 deletions(-) create mode 100644 docker/measure.yaml diff --git a/docker/measure.yaml b/docker/measure.yaml new file mode 100644 index 000000000..047ff9d9e --- /dev/null +++ b/docker/measure.yaml @@ -0,0 +1,48 @@ +name: mxf +services: + event: + image: event:latest + build: + context: ../ + dockerfile: ./docker/event.Dockerfile + environment: + PGDATABASE: mxfactorial + PGUSER: test + PGPASSWORD: test + PGHOST: postgres + PGPORT: 5432 + REDIS_DB: 0 + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_USERNAME: default + REDIS_PASSWORD: test + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + measure: + image: measure:latest + build: + context: ../ + dockerfile: ./docker/measure.Dockerfile + ports: + - "10010:10010" + environment: + PGDATABASE: mxfactorial + PGUSER: test + PGPASSWORD: test + PGHOST: postgres + PGPORT: 5432 + REDIS_DB: 0 + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_USERNAME: default + REDIS_PASSWORD: test + MEASURE_PORT: 10010 + READINESS_CHECK_PATH: "/healthz" + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy diff --git a/docker/services.yaml b/docker/services.yaml index 259c530e8..34f420b35 100644 --- a/docker/services.yaml +++ b/docker/services.yaml @@ -19,6 +19,8 @@ services: READINESS_CHECK_PATH: "/healthz" RUST_LOG: info GRAPHQL_PORT: 10000 + GRAPHQL_RESOURCE: query + GRAPHQL_WS_RESOURCE: ws balance-by-account: image: balance-by-account:latest build: @@ -196,53 +198,8 @@ services: TRANSACTIONS_BY_ACCOUNT_PORT: 10008 depends_on: - postgres - event: - image: event:latest - build: - context: ../ - dockerfile: ./docker/event.Dockerfile - environment: - PGDATABASE: mxfactorial - PGUSER: test - PGPASSWORD: test - PGHOST: postgres - PGPORT: 5432 - REDIS_DB: 0 - REDIS_HOST: redis - REDIS_PORT: 6379 - REDIS_USERNAME: default - REDIS_PASSWORD: test - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - measure: - image: measure:latest - build: - context: ../ - dockerfile: ./docker/measure.Dockerfile - ports: - - "10010:10010" - environment: - PGDATABASE: mxfactorial - PGUSER: test - PGPASSWORD: test - PGHOST: postgres - PGPORT: 5432 - REDIS_DB: 0 - REDIS_HOST: redis - REDIS_PORT: 6379 - REDIS_USERNAME: default - REDIS_PASSWORD: test - MEASURE_PORT: 10010 - READINESS_CHECK_PATH: "/healthz" - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy client: + image: client:latest build: context: ../ dockerfile: ./docker/client.Dockerfile diff --git a/make/docker.mk b/make/docker.mk index bae475f19..b1f364f18 100644 --- a/make/docker.mk +++ b/make/docker.mk @@ -13,6 +13,10 @@ compose-up-build: compose-down: bash scripts/compose.sh --down +compose-add-measure: + @COMPOSE_IGNORE_ORPHANS=true \ + docker compose -f docker/storage.yaml -f docker/measure.yaml up -d + rebuild-db: @$(MAKE) -C migrations rebuild @$(MAKE) -C migrations run diff --git a/migrations/makefile b/migrations/makefile index b6c601b99..30d934566 100644 --- a/migrations/makefile +++ b/migrations/makefile @@ -64,46 +64,29 @@ create: ###################### local/docker migrate commands ###################### resetdocker: - @docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd reset - -rl: - $(MAKE) resetdocker + @COMPOSE_IGNORE_ORPHANS=true \ + docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd reset 2>&1 | grep -v "No services to build" downdocker: - @docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd down + @COMPOSE_IGNORE_ORPHANS=true \ + docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd down 2>&1 | grep -v "No services to build" testdocker: $(MAKE) resetdocker $(MAKE) downdocker updocker: - @docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd up + @COMPOSE_IGNORE_ORPHANS=true \ + docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd up 2>&1 | grep -v "No services to build" dropdocker: - @docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd drop + @COMPOSE_IGNORE_ORPHANS=true \ + docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd drop 2>&1 | grep -v "No services to build" insert: @cd $(RELATIVE_PROJECT_ROOT_PATH); \ bash scripts/insert-transactions.sh -###################### rds migrate commands ###################### - -resetrds: - @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd reset - -downrds: - @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd down - -testrds: - @$(MAKE) resetrds - @$(MAKE) downrds - -uprds: - @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd up - -droprds: - @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd drop - ###################### arg tests ###################### test-dir-arg: diff --git a/scripts/compose.sh b/scripts/compose.sh index 32e58cd70..46add2f32 100644 --- a/scripts/compose.sh +++ b/scripts/compose.sh @@ -36,14 +36,24 @@ COMPOSE_DIR=./docker INIT_CMD="GRAPHQL_URI=$B64_GRAPHQL_URI \\ docker compose \\ -f $COMPOSE_DIR/storage.yaml \\ - -f $COMPOSE_DIR/services.yaml" + -f $COMPOSE_DIR/services.yaml \\ + -f $COMPOSE_DIR/measure.yaml" if [[ $UP ]]; then + if [[ $BUILD ]]; then + # build storage first for reliable startup + STORAGE_BUILD_CMD="docker compose -f $COMPOSE_DIR/storage.yaml build" + echo "$STORAGE_BUILD_CMD" + echo "" + eval "$STORAGE_BUILD_CMD" + fi + UP_CMD=$(printf '%s \\\n up \\\n -d \\\n --renew-anon-volumes' "$INIT_CMD") + # use --no-build when build was already done above if [[ $BUILD ]]; then - UP_CMD=$(printf '%s \\\n --build' "$UP_CMD") + UP_CMD=$(printf '%s \\\n --no-build' "$UP_CMD") fi echo "$UP_CMD" From fee7e34b645770941aa8421929096004bce1776c Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:41:38 -0800 Subject: [PATCH 10/25] registry switch --- docker/storage.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/storage.yaml b/docker/storage.yaml index e25bae7cd..c26ed804c 100644 --- a/docker/storage.yaml +++ b/docker/storage.yaml @@ -13,7 +13,7 @@ services: retries: 5 start_period: 10s redis: - image: bitnami/redis:latest + image: public.ecr.aws/bitnami/redis:latest ports: - "6379:6379" healthcheck: From bfe9ee09618a0f9d660533f29f15cfdd71c973b8 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:41:58 -0800 Subject: [PATCH 11/25] use bash shell --- crates/pg/makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/pg/makefile b/crates/pg/makefile index 65286c81d..68be1ee75 100644 --- a/crates/pg/makefile +++ b/crates/pg/makefile @@ -1,3 +1,4 @@ +SHELL := /bin/bash RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") APP_NAME=$(shell basename $(CURDIR)) PROJECT_CONF_FILE_NAME=project.yaml From a9f2c4dc5230daaf5eba5bd992fe0c123703a182 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:44:12 -0800 Subject: [PATCH 12/25] make rds commands in go-migrate dir --- .github/workflows/dev-integration.yaml | 2 +- .github/workflows/warm-cache.yaml | 2 +- migrations/go-migrate/makefile | 20 +++++++++++++++++++- tests/makefile | 2 +- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev-integration.yaml b/.github/workflows/dev-integration.yaml index 53e4aa7e9..372f1c976 100644 --- a/.github/workflows/dev-integration.yaml +++ b/.github/workflows/dev-integration.yaml @@ -49,7 +49,7 @@ jobs: - name: warm up lambdas with availability script run: bash scripts/test-availability.sh - name: reset dev rds - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make --no-print-directory -C ./migrations resetrds ENV=dev DB=test + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make --no-print-directory -C ./migrations/go-migrate resetrds ENV=dev DB=test - name: warm cache run: ENV_ID=${{ secrets.DEV_ENV_ID }} bash scripts/invoke-warm-cache.sh --env dev - name: run dev cloud integration tests diff --git a/.github/workflows/warm-cache.yaml b/.github/workflows/warm-cache.yaml index 4f010cabb..600f57fc8 100644 --- a/.github/workflows/warm-cache.yaml +++ b/.github/workflows/warm-cache.yaml @@ -47,7 +47,7 @@ jobs: echo "redis key count: $(wc -l < /tmp/redis_keys.txt)" - name: reset rds # reset dev rds to match local postgres seed data - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C migrations resetrds + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C migrations/go-migrate resetrds - name: warm ddb cache run: | # temporarily add env vars to warm-cache get array so make env fetches them diff --git a/migrations/go-migrate/makefile b/migrations/go-migrate/makefile index 14c712dc1..93e84c0b6 100644 --- a/migrations/go-migrate/makefile +++ b/migrations/go-migrate/makefile @@ -1,3 +1,21 @@ RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk -include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk \ No newline at end of file +include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk + +###################### rds migrate commands ###################### + +resetrds: + @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd reset + +downrds: + @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd down + +testrds: + @$(MAKE) resetrds + @$(MAKE) downrds + +uprds: + @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd up + +droprds: + @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd drop \ No newline at end of file diff --git a/tests/makefile b/tests/makefile index cd9a237ba..dab3927c9 100644 --- a/tests/makefile +++ b/tests/makefile @@ -17,7 +17,7 @@ test-docker: test-cloud: @$(MAKE) -s test-env-arg - @$(MAKE) --no-print-directory -C ../migrations resetrds ENV=$(ENV) DB=test + @$(MAKE) --no-print-directory -C ../migrations/go-migrate resetrds ENV=$(ENV) DB=test @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/invoke-warm-cache.sh --env $(ENV) @$(MAKE) --no-print-directory -S get-secrets ENV=$(ENV) cargo test --features integration_tests -- --test-threads=1 From 08b72e82229a8833cfb7c44044ed5f9f300baa0d Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:44:39 -0800 Subject: [PATCH 13/25] add codepipeline module to cue file --- cue/project_conf.cue | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cue/project_conf.cue b/cue/project_conf.cue index 42c3aa84b..021dbc288 100644 --- a/cue/project_conf.cue +++ b/cue/project_conf.cue @@ -86,6 +86,9 @@ infra: { } modules: { env_var!: #EnvVars + codepipeline: { + env_var!: #EnvVars + } environment: { env_var!: #EnvVars } From 31de467cd0f024b3271456358ae7d45b3f768788 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:46:20 -0800 Subject: [PATCH 14/25] optionally test while building image --- docker/auto-confirm.Dockerfile | 9 ++++++++- docker/balance-by-account.Dockerfile | 11 +++++++++-- docker/event.Dockerfile | 11 +++++++++-- docker/graphql.Dockerfile | 11 +++++++++-- docker/measure.Dockerfile | 11 +++++++++-- docker/request-approve.Dockerfile | 11 +++++++++-- docker/request-by-id.Dockerfile | 11 +++++++++-- docker/request-create.Dockerfile | 11 +++++++++-- docker/requests-by-account.Dockerfile | 11 +++++++++-- docker/rule.Dockerfile | 11 +++++++++-- docker/transaction-by-id.Dockerfile | 11 +++++++++-- docker/transactions-by-account.Dockerfile | 11 +++++++++-- 12 files changed, 107 insertions(+), 23 deletions(-) diff --git a/docker/auto-confirm.Dockerfile b/docker/auto-confirm.Dockerfile index 21cd70ff7..b9c082d61 100644 --- a/docker/auto-confirm.Dockerfile +++ b/docker/auto-confirm.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,6 +10,11 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/auto-confirm/Cargo.toml \ --target x86_64-unknown-linux-musl \ diff --git a/docker/balance-by-account.Dockerfile b/docker/balance-by-account.Dockerfile index 814416c20..f0a4b50c4 100644 --- a/docker/balance-by-account.Dockerfile +++ b/docker/balance-by-account.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/balance-by-account/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/balance-by-account /app/balance-by-account diff --git a/docker/event.Dockerfile b/docker/event.Dockerfile index 0b018ad86..802c50dd6 100644 --- a/docker/event.Dockerfile +++ b/docker/event.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/event/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine:latest +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/event /app/event diff --git a/docker/graphql.Dockerfile b/docker/graphql.Dockerfile index 3fd22d6e6..2a0d28ea6 100644 --- a/docker/graphql.Dockerfile +++ b/docker/graphql.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/graphql/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/graphql /app/graphql diff --git a/docker/measure.Dockerfile b/docker/measure.Dockerfile index a3c57366c..0fa0dd5d8 100644 --- a/docker/measure.Dockerfile +++ b/docker/measure.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/measure/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine:latest +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/measure /app/measure diff --git a/docker/request-approve.Dockerfile b/docker/request-approve.Dockerfile index 1db7c5219..b93b59ebc 100644 --- a/docker/request-approve.Dockerfile +++ b/docker/request-approve.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/request-approve/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/request-approve /app/request-approve diff --git a/docker/request-by-id.Dockerfile b/docker/request-by-id.Dockerfile index f169dff19..09374675c 100644 --- a/docker/request-by-id.Dockerfile +++ b/docker/request-by-id.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/request-by-id/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/request-by-id /app/request-by-id diff --git a/docker/request-create.Dockerfile b/docker/request-create.Dockerfile index bc9d1a1d9..837e9f4ff 100644 --- a/docker/request-create.Dockerfile +++ b/docker/request-create.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/request-create/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/request-create /app/request-create diff --git a/docker/requests-by-account.Dockerfile b/docker/requests-by-account.Dockerfile index 82175bbe2..6fbc153f8 100644 --- a/docker/requests-by-account.Dockerfile +++ b/docker/requests-by-account.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/requests-by-account/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/requests-by-account /app/requests-by-account diff --git a/docker/rule.Dockerfile b/docker/rule.Dockerfile index b549aa43b..d739642ed 100644 --- a/docker/rule.Dockerfile +++ b/docker/rule.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/rule/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/rule /app/rule diff --git a/docker/transaction-by-id.Dockerfile b/docker/transaction-by-id.Dockerfile index d40ada425..342c90e6f 100644 --- a/docker/transaction-by-id.Dockerfile +++ b/docker/transaction-by-id.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/transaction-by-id/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/transaction-by-id /app/transaction-by-id diff --git a/docker/transactions-by-account.Dockerfile b/docker/transactions-by-account.Dockerfile index ab3f0906b..35239e87b 100644 --- a/docker/transactions-by-account.Dockerfile +++ b/docker/transactions-by-account.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/transactions-by-account/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/transactions-by-account /app/transactions-by-account From 519b70f6d5882909dbc2ddd09a55606db2a68348 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:46:57 -0800 Subject: [PATCH 15/25] switch image registry --- docker/bitnami-postgres.Dockerfile | 2 +- docker/client.Dockerfile | 19 +++++++++---------- docker/go-migrate.Dockerfile | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/docker/bitnami-postgres.Dockerfile b/docker/bitnami-postgres.Dockerfile index 336b39cb1..d383cccc7 100644 --- a/docker/bitnami-postgres.Dockerfile +++ b/docker/bitnami-postgres.Dockerfile @@ -1,4 +1,4 @@ -FROM bitnamilegacy/postgresql:15.3.0 +FROM public.ecr.aws/bitnami/postgresql:15 USER root diff --git a/docker/client.Dockerfile b/docker/client.Dockerfile index 1bdf36724..2972fa878 100644 --- a/docker/client.Dockerfile +++ b/docker/client.Dockerfile @@ -1,4 +1,4 @@ -FROM mxfactorial/client-base:v1 AS builder1 +FROM public.ecr.aws/docker/library/node:lts-alpine AS builder ARG PUBLIC_POOL_ID ARG PUBLIC_CLIENT_ID @@ -10,19 +10,18 @@ ARG PORT WORKDIR /app -COPY client . +COPY client/package*.json ./ +RUN npm install -# WARNING: docker build was failing to COPY package*.json with unstaged -# changes on macos so npm install may be required before npm run build: -# RUN npm install +COPY client . RUN npm run build -FROM node:lts-alpine +FROM public.ecr.aws/docker/library/node:lts-alpine COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter -COPY --from=builder1 /app/build /app -COPY --from=builder1 /app/package.json /app -COPY --from=builder1 /app/package-lock.json /app +COPY --from=builder /app/build /app +COPY --from=builder /app/package.json /app +COPY --from=builder /app/package-lock.json /app WORKDIR /app -CMD ["node", "index.js"] \ No newline at end of file +CMD ["node", "index.js"] diff --git a/docker/go-migrate.Dockerfile b/docker/go-migrate.Dockerfile index 313486761..9b7979121 100644 --- a/docker/go-migrate.Dockerfile +++ b/docker/go-migrate.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:latest AS builder +FROM public.ecr.aws/docker/library/alpine:latest AS builder ARG VERSION=v4.17.0 ARG ARCH=linux-amd64 From c7d580dec74575ab7437e8ef032682f3dea6ac65 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:47:15 -0800 Subject: [PATCH 16/25] remove unused client base image --- docker/client-base.Dockerfile | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 docker/client-base.Dockerfile diff --git a/docker/client-base.Dockerfile b/docker/client-base.Dockerfile deleted file mode 100644 index f38348b79..000000000 --- a/docker/client-base.Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM node:lts-alpine - -WORKDIR /app - -COPY client/package*.json ./ - -RUN npm install \ No newline at end of file From 3cc34767aa7f7e25bf1b6320eff34f3b6ce8a64e Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:48:14 -0800 Subject: [PATCH 17/25] standalone curl --- scripts/go-migrate-rds.sh | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/scripts/go-migrate-rds.sh b/scripts/go-migrate-rds.sh index 038f0328d..5e763afec 100644 --- a/scripts/go-migrate-rds.sh +++ b/scripts/go-migrate-rds.sh @@ -26,14 +26,26 @@ PROJECT_CONF=project.yaml ENV_ID=$(source scripts/print-env-id.sh) SSM_VERSION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.SSM_VERSION.default' $PROJECT_CONF) REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) +SSM_PREFIX="$ENV_ID/$SSM_VERSION/$ENV" -PASSPHRASE=$(aws ssm get-parameter \ - --name "/$ENV_ID/$SSM_VERSION/$ENV/tool/lambda/go_migrate/passphrase" \ +echo "*** fetching go-migrate url and passphrase from ssm" + +GO_MIGRATE_URL=$(aws ssm get-parameter \ + --name "/$SSM_PREFIX/service/lambda/go_migrate/url" \ + --with-decryption \ --query 'Parameter.Value' \ - --region $REGION \ + --output text \ + --region $REGION) + +PASSPHRASE=$(aws ssm get-parameter \ + --name "/$SSM_PREFIX/tool/lambda/go_migrate/passphrase" \ --with-decryption \ - --output text) + --query 'Parameter.Value' \ + --output text \ + --region $REGION) -PAYLOAD="{\"db_type\":\"test\",\"cmd\":\"$CMD\",\"passphrase\":\"$PASSPHRASE\"}" +echo "*** invoking go-migrate lambda" -bash scripts/invoke-function-url.sh --app-name go-migrate --payload "$PAYLOAD" --env $ENV +curl -s -X POST "$GO_MIGRATE_URL" \ + -H 'Content-Type: application/json' \ + -d "{\"db_type\":\"test\",\"cmd\":\"$CMD\",\"passphrase\":\"$PASSPHRASE\"}" | yq -o=json From 77175a721e138562f868aa7b032996d797e2af51 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:48:28 -0800 Subject: [PATCH 18/25] var naming --- scripts/manage-rds.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/manage-rds.sh b/scripts/manage-rds.sh index 0cd3d1ee6..9db368f17 100644 --- a/scripts/manage-rds.sh +++ b/scripts/manage-rds.sh @@ -19,8 +19,8 @@ PROJECT_CONF=project.yaml REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) ENV_ID=$(source scripts/print-env-id.sh) -RDS_INSTANCE_NAME_PREFIX=$(yq '.infra.terraform.aws.modules.environment.env_var.set.RDS_INSTANCE_NAME_PREFIX.default' $PROJECT_CONF) -RDS_INSTANCE_NAME="$RDS_INSTANCE_NAME_PREFIX-$ENV_ID-$ENV" +NAME_PREFIX=$(yq '.infra.terraform.aws.modules.environment.env_var.set.NAME_PREFIX.default' $PROJECT_CONF) +RDS_INSTANCE_NAME="$NAME_PREFIX-$ENV_ID-$ENV" if [[ "$START" -eq 1 ]]; then aws rds start-db-instance \ From 674de545fcd8ea15d062cc32921d05ab49bb76d6 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:49:39 -0800 Subject: [PATCH 19/25] include test data and all migrations --- scripts/zip-services.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/zip-services.sh b/scripts/zip-services.sh index b6947a79a..d3445c137 100644 --- a/scripts/zip-services.sh +++ b/scripts/zip-services.sh @@ -25,14 +25,9 @@ zip -r $SERVICES_ZIP \ services \ crates \ tests \ - migrations/go-migrate \ - migrations/warm-cache \ - migrations/schema \ - migrations/seed \ - migrations/testseed \ + migrations \ --exclude='*/.env' \ --exclude='*/README.md' \ - --exclude='tests/testdata/*' \ --exclude='tests/thunder-tests/*' \ --exclude='services/graphql/postman/*' \ --exclude='client/node_modules/*' \ From 8b9955898b0c1a1eb45d4720094db997e0913a3a Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:49:58 -0800 Subject: [PATCH 20/25] ecr image convenience script --- scripts/ecr-images.sh | 296 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 scripts/ecr-images.sh diff --git a/scripts/ecr-images.sh b/scripts/ecr-images.sh new file mode 100644 index 000000000..cee75ecb7 --- /dev/null +++ b/scripts/ecr-images.sh @@ -0,0 +1,296 @@ +#!/bin/bash + +set -e + +YELLOW='\033[0;33m' +RED='\033[0;31m' +RESET='\033[0m' + +if [[ "$#" -lt 1 ]]; then + cat <<- 'EOF' + use: + bash scripts/ecr-images.sh --build # build + test all + bash scripts/ecr-images.sh --build --no-test # build all (skip tests) + bash scripts/ecr-images.sh --build --push # build + test + push all + bash scripts/ecr-images.sh --build --push --deploy # build + test + push + deploy all + bash scripts/ecr-images.sh --pull # pull all from ecr + bash scripts/ecr-images.sh --integ # run integration tests + + --service works with any flag: + bash scripts/ecr-images.sh --build --service graphql + bash scripts/ecr-images.sh --build --push --deploy --service graphql + bash scripts/ecr-images.sh --pull --service graphql + EOF + exit 1 +fi + +# defaults +BUILD=false +TEST=true +PUSH=false +DEPLOY=false +PULL=false +INTEG=false +SERVICE="" + +while [[ "$#" -gt 0 ]]; do + case $1 in + --build) BUILD=true; shift ;; + --test) TEST=true; shift ;; + --no-test) TEST=false; shift ;; + --push) PUSH=true; shift ;; + --deploy) DEPLOY=true; shift ;; + --pull) PULL=true; shift ;; + --integ) INTEG=true; shift ;; + --service) SERVICE="$2"; shift; shift ;; + *) echo "unknown parameter passed: $1"; exit 1 ;; + esac +done + +if [[ $(basename $(pwd)) != "mxfactorial" ]]; then + echo "error: run from project root" + exit 1 +fi + +ENV=dev +PROJECT_CONF=project.yaml +ENV_FILE_NAME=$(yq '.env_var.set.ENV_FILE_NAME.default' $PROJECT_CONF) + +if [[ ! -f $ENV_FILE_NAME ]]; then + echo "error: $ENV_FILE_NAME not found. run 'make env-id' first" + exit 1 +fi + +if ! grep -q "ENV_ID=" $ENV_FILE_NAME; then + echo "error: ENV_ID not found in $ENV_FILE_NAME. run 'make env-id' first" + exit 1 +fi + +# validate service if provided +if [[ -n "$SERVICE" ]]; then + if [[ $(bash scripts/list-dir-paths.sh --type all | grep --color=never "$SERVICE$" >/dev/null 2>&1; echo $?) -ne 0 ]]; then + echo "error: \"$SERVICE\" not in $PROJECT_CONF" + exit 1 + fi +fi + +ENV_ID=$(grep "ENV_ID=" $ENV_FILE_NAME | cut -d'=' -f2) +ID_ENV="$ENV_ID-$ENV" +REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +ECR_URI="$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$ENV_ID/$ENV" + +function get_services() { + if [[ -n "$SERVICE" ]]; then + echo "$SERVICE" + else + yq '.. | select(has("type") and has("deploy") and .type == "app" and .deploy == true) | path | .[-1]' $PROJECT_CONF + fi +} + +function start_builds() { + local RUN_TESTS=$1 + local PUSH_IMAGE=$2 + local DO_DEPLOY=$3 + + ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) + ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" + SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) + BUILD_OBJECT_KEY_PATH=$(yq '.infra.terraform.aws.modules.codepipeline.env_var.set.BUILD_OBJECT_KEY_PATH.default' $PROJECT_CONF) + + source scripts/zip-services.sh + + echo '*** uploading archive to s3' + aws s3 cp $SERVICES_ZIP s3://$ARTIFACTS_BUCKET/$BUILD_OBJECT_KEY_PATH/ --region $REGION + rm $SERVICES_ZIP + + echo "*** starting builds (RUN_TESTS=$RUN_TESTS, PUSH_IMAGE=$PUSH_IMAGE, DEPLOY=$DO_DEPLOY)" + echo "" + + BUILD_IDS="" + for SVC in $(get_services); do + PROJECT_NAME="mxfactorial-$SVC-$ID_ENV" + echo -e "${YELLOW}starting $PROJECT_NAME...${RESET}" + BUILD_ID=$(aws codebuild start-build \ + --project-name $PROJECT_NAME \ + --region $REGION \ + --source-type-override S3 \ + --source-location-override "$ARTIFACTS_BUCKET/$BUILD_OBJECT_KEY_PATH/$SERVICES_ZIP" \ + --artifacts-override type=NO_ARTIFACTS \ + --environment-variables-override \ + name=RUN_TESTS,value=$RUN_TESTS \ + name=PUSH_IMAGE,value=$PUSH_IMAGE \ + name=DEPLOY,value=$DO_DEPLOY \ + --query 'build.id' \ + --output text) + BUILD_IDS="$BUILD_IDS $BUILD_ID" + # URL encode the build ID (replace : with %3A) + BUILD_ID_ENCODED=$(printf '%s' "$BUILD_ID" | sed 's/:/%3A/g') + printf 'https://%s.console.aws.amazon.com/codesuite/codebuild/%s/projects/%s/build/%s/?region=%s\n' "$REGION" "$AWS_ACCOUNT_ID" "$PROJECT_NAME" "$BUILD_ID_ENCODED" "$REGION" + echo "" + done + + echo "*** waiting for builds to complete" + echo -e "${YELLOW}(ctrl+c to exit - builds will continue in background)${RESET}" + echo "" + + for BUILD_ID in $BUILD_IDS; do + while true; do + STATUS=$(aws codebuild batch-get-builds \ + --ids $BUILD_ID \ + --region $REGION \ + --query 'builds[0].buildStatus' \ + --output text) + if [[ $STATUS != "IN_PROGRESS" ]]; then + echo "$BUILD_ID: $STATUS" + break + fi + sleep 10 + printf '%s' '.' + done + done +} + +function trigger_pipeline() { + ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) + ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" + SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) + PIPELINE_NAME="mxfactorial-build-$ID_ENV" + + source scripts/zip-services.sh + + echo '*** uploading archive to s3' + aws s3 cp $SERVICES_ZIP s3://$ARTIFACTS_BUCKET/build/ --region $REGION + rm $SERVICES_ZIP + + echo "*** codepipeline $PIPELINE_NAME triggered via eventbridge" + echo "https://$REGION.console.aws.amazon.com/codesuite/codepipeline/pipelines/$PIPELINE_NAME/view/?region=$REGION" + + sleep 5 + + echo "*** waiting for codepipeline to complete" + echo -e "${YELLOW}(ctrl+c to exit - pipeline will continue in background)${RESET}" + + function get_pipeline_state() { + PIPELINE_STATE=$(aws codepipeline get-pipeline-state \ + --name $PIPELINE_NAME \ + --region $REGION \ + --query 'stageStates[?stageName==`Build`].latestExecution.status' \ + --output text) + } + + if [[ $(uname) == "Darwin" ]]; then + TIMEOUT_MAX=$(date -u -v+20M "+%s") + else + TIMEOUT_MAX=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 20 minutes" +'%s') + fi + + PIPELINE_STATE='InProgress' + + while [[ $PIPELINE_STATE == 'InProgress' && $(date +%s) -lt $TIMEOUT_MAX ]]; do + sleep 10 + get_pipeline_state + printf '%s' '.' + done + + echo "" + + if [[ $PIPELINE_STATE == 'Succeeded' ]]; then + echo "*** build succeeded" + else + echo "*** build status: $PIPELINE_STATE" + exit 1 + fi +} + +# handle pull separately +if [[ $PULL == true ]]; then + echo "*** logging into ecr" + source scripts/auth-ecr.sh + + echo "*** pulling images from ecr" + for SVC in $(get_services); do + echo "pulling $SVC..." + docker pull $ECR_URI/$SVC:latest || echo "skipping $SVC (not found)" + docker tag $ECR_URI/$SVC:latest $SVC:latest 2>/dev/null || true + done + + echo "*** done" + exit 0 +fi + +# handle integ tests +if [[ $INTEG == true ]]; then + ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) + ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" + SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) + INTEG_TEST_OBJECT_KEY_PATH=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.INTEG_TEST_OBJECT_KEY_PATH.default' $PROJECT_CONF) + + source scripts/zip-services.sh + + echo '*** uploading archive to s3' + aws s3 cp $SERVICES_ZIP s3://$ARTIFACTS_BUCKET/$INTEG_TEST_OBJECT_KEY_PATH/ --region $REGION + rm $SERVICES_ZIP + + PROJECT_NAME="mxfactorial-integ-$ID_ENV" + echo -e "${YELLOW}*** starting integration tests ($PROJECT_NAME)${RESET}" + echo "" + + BUILD_ID=$(aws codebuild start-build \ + --project-name $PROJECT_NAME \ + --region $REGION \ + --query 'build.id' \ + --output text) + + # URL encode the build ID (replace : with %3A) + BUILD_ID_ENCODED=$(printf '%s' "$BUILD_ID" | sed 's/:/%3A/g') + printf 'https://%s.console.aws.amazon.com/codesuite/codebuild/%s/projects/%s/build/%s/?region=%s\n' "$REGION" "$AWS_ACCOUNT_ID" "$PROJECT_NAME" "$BUILD_ID_ENCODED" "$REGION" + echo "" + + echo "*** waiting for integ tests to complete" + echo -e "${YELLOW}(ctrl+c to exit - build will continue in background)${RESET}" + echo "" + + while true; do + STATUS=$(aws codebuild batch-get-builds \ + --ids $BUILD_ID \ + --region $REGION \ + --query 'builds[0].buildStatus' \ + --output text) + if [[ $STATUS != "IN_PROGRESS" ]]; then + echo "$BUILD_ID: $STATUS" + break + fi + sleep 10 + printf '%s' '.' + done + + if [[ $STATUS == "SUCCEEDED" ]]; then + echo "*** integration tests passed" + else + echo "*** integration tests failed" + exit 1 + fi + + exit 0 +fi + +# handle build +if [[ $BUILD == true ]]; then + RUN_TESTS=$TEST + PUSH_IMAGE=$PUSH + DO_DEPLOY=$DEPLOY + + # warn if deploying without pushing + if [[ $DEPLOY == true && $PUSH == false ]]; then + echo -e "${RED}warning: deploying without pushing (will use existing ecr images)${RESET}" + echo "" + fi + + # use pipeline for full builds (all services + push + no deploy), otherwise direct codebuild + if [[ -z "$SERVICE" && $PUSH == true && $TEST == true && $DEPLOY == false ]]; then + trigger_pipeline + else + start_builds "$RUN_TESTS" "$PUSH_IMAGE" "$DO_DEPLOY" + fi +fi From b62ac7ecd667c060ea5e82cdd1ed2caa395b96ee Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:52:22 -0800 Subject: [PATCH 21/25] build and push images while building cloud dev env --- scripts/build-dev-env.sh | 13 +++++++------ scripts/delete-dev-env.sh | 31 +++++++++++-------------------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/scripts/build-dev-env.sh b/scripts/build-dev-env.sh index b690893d4..8b9db6ca4 100644 --- a/scripts/build-dev-env.sh +++ b/scripts/build-dev-env.sh @@ -40,6 +40,9 @@ terraform init printf "\n${YELLOW}*** provisioning artifact, cache origin and terraform state storage${NOCOLOR}\n\n" terraform apply +printf "\n${YELLOW}*** waiting for IAM roles to propagate${NOCOLOR}\n\n" +sleep 20 + popd function apply_agigw_logging_perm() { @@ -79,8 +82,8 @@ else apply_agigw_logging_perm fi -printf "\n${YELLOW}*** compiling app binaries and pushing to artifact bucket${NOCOLOR}\n\n" -make --no-print-directory all CMD=initial-deploy ENV=dev +printf "\n${YELLOW}*** building and pushing images to ECR${NOCOLOR}\n\n" +bash scripts/ecr-images.sh --build --push source ./scripts/terraform-init-dev.sh \ --key "$TFSTATE_ENV" \ @@ -116,12 +119,10 @@ if [[ $(yq '.scripts.env_var.set.BUILD_DB.default' "../../../../../$PROJECT_CONF exit 0 fi -pushd ../../../../../migrations +popd printf "\n${YELLOW}*** deploying migrations to rds in $ID_ENV${NOCOLOR}\n\n" -make --no-print-directory uprds DB=test ENV=dev - -popd; popd; +make --no-print-directory -C migrations/go-migrate uprds DB=test ENV=dev printf "\n${YELLOW}*** warming cache in $ID_ENV${NOCOLOR}\n\n" bash scripts/invoke-warm-cache.sh --env dev diff --git a/scripts/delete-dev-env.sh b/scripts/delete-dev-env.sh index e0b111dd0..d9bf41613 100644 --- a/scripts/delete-dev-env.sh +++ b/scripts/delete-dev-env.sh @@ -1,49 +1,40 @@ #!/bin/bash -while [[ "$#" -gt 0 ]]; do - case $1 in - --force) FORCE=1; shift ;; - *) echo "unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - - -if [[ ! "$FORCE" ]]; then - set -e -fi +set -e YELLOW='\033[0;33m' NOCOLOR='\033[0m' ENV=dev PROJECT_CONF=project.yaml -ENV_FILE_NAME=$(yq '.env_var.set.ENV_FILE_NAME.default' $PROJECT_CONF) -ENV_FILE=$ENV_FILE_NAME ENV_ID=$(source scripts/print-env-id.sh) if [[ -z $ENV_ID ]]; then - echo "ENV_ID not found in $ENV_FILE" - echo "bash scripts/set-custom-env-id.sh --env-id 12345 before continuing" + echo "ENV_ID not found. run 'make env-id' first" exit 1 fi pushd infra/terraform/aws/environments/dev +printf "\n${YELLOW}*** destroying $ENV_ID-$ENV environment${NOCOLOR}\n\n" terraform destroy --auto-approve && rm -rf .terraform* .tfplan* popd pushd infra/terraform/aws/environments/region -# skip if api gateway logging permission not managed by local terraform if [[ -f terraform.tfstate ]]; then + printf "\n${YELLOW}*** destroying api gateway logging permission${NOCOLOR}\n\n" terraform destroy --auto-approve && rm -rf .terraform* terraform.tfstate -# todo: elif manually delete logging permission if current resource matches naming convention fi popd -source scripts/delete-dev-storage.sh # --env arg not available here, dev only +pushd infra/terraform/aws/environments/init-dev + +printf "\n${YELLOW}*** destroying $ENV_ID-$ENV storage${NOCOLOR}\n\n" +terraform destroy --auto-approve && rm -rf .terraform* terraform.tfstate + +popd -printf "\n${YELLOW}*** ${ENV_ID}-${ENV} env deleted. you may now delete your workspace${NOCOLOR}\n" \ No newline at end of file +printf "\n${YELLOW}*** $ENV_ID-$ENV deleted${NOCOLOR}\n" From c17e83663ceb60537dc9821c76a8f63095f3facb Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:52:59 -0800 Subject: [PATCH 22/25] delete unused scripts and workflows after adding codebuild --- .github/workflows/build-all-images.yaml | 331 ----------------------- .github/workflows/deploy-all-images.yaml | 124 --------- scripts/README.md | 30 +- scripts/build-all-images.sh | 86 ------ scripts/build-image-job.sh | 40 --- scripts/delete-dev-images.sh | 41 --- scripts/delete-dev-storage.sh | 56 ---- scripts/delete-ecr-repos.sh | 11 - scripts/deploy-all-images.sh | 85 ------ scripts/deploy-image-job.sh | 45 --- scripts/pull-all-images.sh | 23 -- 11 files changed, 5 insertions(+), 867 deletions(-) delete mode 100644 .github/workflows/build-all-images.yaml delete mode 100644 .github/workflows/deploy-all-images.yaml delete mode 100644 scripts/build-all-images.sh delete mode 100644 scripts/build-image-job.sh delete mode 100644 scripts/delete-dev-images.sh delete mode 100644 scripts/delete-dev-storage.sh delete mode 100644 scripts/delete-ecr-repos.sh delete mode 100644 scripts/deploy-all-images.sh delete mode 100644 scripts/deploy-image-job.sh delete mode 100644 scripts/pull-all-images.sh diff --git a/.github/workflows/build-all-images.yaml b/.github/workflows/build-all-images.yaml deleted file mode 100644 index e581dcb13..000000000 --- a/.github/workflows/build-all-images.yaml +++ /dev/null @@ -1,331 +0,0 @@ -name: build-all-images - -on: - workflow_dispatch: - -env: - ENV_ID: ${{ secrets.DEV_ENV_ID }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_REGISTRY: ghcr.io - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - rule: - name: rule - runs-on: ubuntu-latest - env: - SERVICE_NAME: rule - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - balance_by_account: - name: balance-by-account - runs-on: ubuntu-latest - env: - SERVICE_NAME: balance-by-account - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - graphql: - name: graphql - runs-on: ubuntu-latest - env: - SERVICE_NAME: graphql - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - request_create: - name: request-create - runs-on: ubuntu-latest - env: - SERVICE_NAME: request-create - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - request_approve: - name: request-approve - runs-on: ubuntu-latest - env: - SERVICE_NAME: request-approve - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - request_by_id: - name: request-by-id - runs-on: ubuntu-latest - env: - SERVICE_NAME: request-by-id - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - requests_by_account: - name: requests-by-account - runs-on: ubuntu-latest - env: - SERVICE_NAME: requests-by-account - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - transaction_by_id: - name: transaction-by-id - runs-on: ubuntu-latest - env: - SERVICE_NAME: transaction-by-id - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - transactions_by_account: - name: transactions-by-account - runs-on: ubuntu-latest - env: - SERVICE_NAME: transactions-by-account - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - event: - name: event - runs-on: ubuntu-latest - env: - SERVICE_NAME: event - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - measure: - name: measure - runs-on: ubuntu-latest - env: - SERVICE_NAME: measure - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - auto_confirm: - name: auto-confirm - runs-on: ubuntu-latest - env: - SERVICE_NAME: auto-confirm - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - go_migrate: - name: go-migrate - runs-on: ubuntu-latest - env: - SERVICE_NAME: go-migrate - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx ./migrations/go-migrate - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - warm_cache: - name: warm-cache - runs-on: ubuntu-latest - env: - SERVICE_NAME: warm-cache - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - client: - name: client - runs-on: ubuntu-latest - env: - SERVICE_NAME: client - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME \ No newline at end of file diff --git a/.github/workflows/deploy-all-images.yaml b/.github/workflows/deploy-all-images.yaml deleted file mode 100644 index a039eba27..000000000 --- a/.github/workflows/deploy-all-images.yaml +++ /dev/null @@ -1,124 +0,0 @@ -name: deploy-all-images - -on: - workflow_dispatch: - -env: - ENV_ID: ${{ secrets.DEV_ENV_ID }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - rule: - name: rule - runs-on: ubuntu-latest - env: - SERVICE_NAME: rule - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - balance_by_account: - name: balance-by-account - runs-on: ubuntu-latest - env: - SERVICE_NAME: balance-by-account - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - graphql: - name: graphql - runs-on: ubuntu-latest - env: - SERVICE_NAME: graphql - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - request_create: - name: request-create - runs-on: ubuntu-latest - env: - SERVICE_NAME: request-create - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - request_approve: - name: request-approve - runs-on: ubuntu-latest - env: - SERVICE_NAME: request-approve - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - request_by_id: - name: request-by-id - runs-on: ubuntu-latest - env: - SERVICE_NAME: request-by-id - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - requests_by_account: - name: requests-by-account - runs-on: ubuntu-latest - env: - SERVICE_NAME: requests-by-account - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - transaction_by_id: - name: transaction-by-id - runs-on: ubuntu-latest - env: - SERVICE_NAME: transaction-by-id - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - transactions_by_account: - name: transactions-by-account - runs-on: ubuntu-latest - env: - SERVICE_NAME: transactions-by-account - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - auto_confirm: - name: auto-confirm - runs-on: ubuntu-latest - env: - SERVICE_NAME: auto-confirm - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - go_migrate: - name: go-migrate - runs-on: ubuntu-latest - env: - SERVICE_NAME: go-migrate - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - warm_cache: - name: warm-cache - runs-on: ubuntu-latest - env: - SERVICE_NAME: warm-cache - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md index ddcbcd488..25f574ad0 100755 --- a/scripts/README.md +++ b/scripts/README.md @@ -144,10 +144,6 @@ inits terraform state for dev envs creates then sets custom env id to enable access to dev env from multiple workspaces -##### `delete-dev-storage.sh` - -deletes storage for cloud dev environment with aws cli instead of terraform when state file not found, e.g. cloud dev env storage was created on different machine - ##### `list-lambdas.sh` lists lambdas created by terraform @@ -216,6 +212,10 @@ invokes go-migrate lambda to run migrations on rds send a http request to the internal `migrations/go-migrate` tool +##### `ecr-images.sh` + +triggers codebuild to build, test and push images to ecr. `--build` builds+tests, `--push` pushes to ecr, `--deploy` updates lambdas, `--no-test` skips tests, `--integ` runs integration tests, `--pull` pulls from ecr, `--service ` targets single service + ##### `auth-ecr.sh` authenticate with ecr @@ -248,10 +248,6 @@ prints uri of ecr repo prints service image tag with ecr repo uri and current git sha added as tag version -##### `delete-ecr-repos.sh` - -convenience script to delete all dev ecr repos - ##### `push-dev-image.sh` pushes local docker image to dev ecr repo (assumes local image already tagged) @@ -282,22 +278,9 @@ used in integration test workflow after cloud integration tests pass 1. adds prod tag if current dev image tagged with merge commit, then pushes to prod ecr 1. exits if current dev image NOT tagged with merge commit (prod image not tagged and pushed) -##### `build-image-job.sh` - -used in `.github/workflows/build-all-images.yaml` to copy zipped code from s3, then build, tag and push service images to github container registry - ##### `zip-services.sh` -adds services to zip file. used by `scripts/build-all-images.sh` before triggering `.github/workflows/build-all-images.yaml` - -##### `build-all-images.sh` -zips and pushes current service code to s3, then triggers `.github/workflows/build-all-images.yaml` to avoid building almost a dozen rust images locally - -##### `pull-all-images.sh` -pulls images built and pushed by `.github/workflows/build-all-images.yaml` - -##### `deploy-all-images.sh` -zips and pushes current service code to s3, then triggers `.github/workflows/deploy-all-images.yaml` to build and deploy services to lambda +adds services to zip file for s3 upload ##### `create-env-id.sh` adds a [$RANDOM](https://tldp.org/LDP/abs/html/randomvar.html) `ENV_ID` variable the `.env` file in project root. the `ENV_ID` variable is used by terraform to provision cloud development environments matched to a developers local machine @@ -332,9 +315,6 @@ gets ssh key from ssm and writes it to disk #### `install.sh` installs project dependencies. only macos supported -#### `delete-dev-images.sh` -deletes all images of single app in dev ecr - #### `test-availability.sh` tests availability of services locally or in cloud diff --git a/scripts/build-all-images.sh b/scripts/build-all-images.sh deleted file mode 100644 index 05d6466b3..000000000 --- a/scripts/build-all-images.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash - -set -e - -if [[ -z $GITHUB_PAT ]]; then - echo "set GITHUB_PAT variable in shell to continue" - exit 1 -fi - -ENV=dev -PROJECT_CONF=project.yaml -ENV_ID=$(source ./scripts/print-env-id.sh) -ID_ENV="$ENV_ID-$ENV" -REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) -ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) -ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" -IMAGE_BUILDER_WORKFLOW=$(yq '.[".github"].workflows.env_var.set.IMAGE_BUILDER_WORKFLOW.default' $PROJECT_CONF) -WORKFLOW_ID=$IMAGE_BUILDER_WORKFLOW -GITHUB_ORG=$(yq '.[".github"].env_var.set.GITHUB_ORG.default' $PROJECT_CONF) -GITHUB_REPO_NAME=$(yq '.[".github"].env_var.set.GITHUB_REPO_NAME.default' $PROJECT_CONF) -SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) - -source scripts/zip-services.sh - -echo '*** uploading archive to s3' -aws s3 cp $SERVICES_ZIP s3://$ARTIFACTS_BUCKET --region $REGION -rm $SERVICES_ZIP - -echo "*** triggering .github/workflows/$WORKFLOW_ID" - -curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_PAT" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/workflows/$WORKFLOW_ID/dispatches \ - -d "{\"ref\":\"develop\"}" - -if [[ $(uname) == "Darwin" ]]; then - # store utc time in iso8601 format minus 10 seconds - CREATED=$(date -u -v-10S "+%Y-%m-%dT%H:%M:%SZ") - # store utc time in unix timestamp format plus 10 minutes - TEN_MIN_MAX=$(date -u -v+10M "+%s") -else - CREATED=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 10 seconds ago" +'%Y-%m-%dT%H:%M:%SZ') - TEN_MIN_MAX=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 10 minutes" +'%s') -fi - -# wait 5 seconds -sleep 5 - -RUN_ID=$(curl -sL \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_PAT" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/workflows/$WORKFLOW_ID/runs?created=>$CREATED" | yq '.workflow_runs[0].id') - -echo "*** waiting for $WORKFLOW_ID github workflow to complete" - -function get_run() { - RUN=$(curl -sL \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_PAT" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/runs/$RUN_ID) - - STATUS=$(echo "$RUN" | yq '.status') - CONCLUSION=$(echo "$RUN" | yq '.conclusion') -} - -STATUS='queued' -CONCLUSION=null - -while [[ $STATUS != 'completed' && $(date +%s) -lt $TEN_MIN_MAX ]]; do - sleep 5 - get_run - printf '%s' '.' -done - -echo "" - -if [[ $CONCLUSION != 'success' ]]; then - echo "build failed, pulling latest from ghcr anyway" -fi - -source scripts/pull-all-images.sh \ No newline at end of file diff --git a/scripts/build-image-job.sh b/scripts/build-image-job.sh deleted file mode 100644 index f0219c9d9..000000000 --- a/scripts/build-image-job.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -e - -# set in .github/workflows/build-all-images.yaml -if [[ -z $ENV_ID ]]; then - echo "ENV_ID is not set" - exit 1 -fi - -if [[ "$#" -ne 4 ]]; then - echo "use: bash scripts/build-image-job.sh --service-name request-create --build-ctx ." - exit 1 -fi - -while [[ "$#" -gt 0 ]]; do - case $1 in - --service-name) SERVICE_NAME="$2"; shift ;; - --build-ctx) BUILD_CTX="$2"; shift ;; - *) echo "unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - -PROJECT_CONF=project.yaml -REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) -ENV=dev -ID_ENV="$ENV_ID-$ENV" -ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) -ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" -SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) -SERVICES_DIR=$(echo $SERVICES_ZIP | sed 's/.zip//') - -aws s3 cp s3://$ARTIFACTS_BUCKET/$SERVICES_ZIP . --region $REGION - -unzip $SERVICES_ZIP -d $SERVICES_DIR - -cd $SERVICES_DIR - -docker build -t $SERVICE_NAME:latest -f ./docker/$SERVICE_NAME.Dockerfile --provenance=false $BUILD_CTX \ No newline at end of file diff --git a/scripts/delete-dev-images.sh b/scripts/delete-dev-images.sh deleted file mode 100644 index ab4b04f00..000000000 --- a/scripts/delete-dev-images.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -e - -if [[ "$#" -ne 2 ]]; then - cat <<- 'EOF' - use: - bash scripts/delete-dev-images.sh --app-name client - EOF - exit 1 -fi - -while [[ "$#" -gt 0 ]]; do - case $1 in - --app-name) APP_NAME="$2"; shift ;; - *) echo "unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - -PROJECT_CONF=project.yaml -ENV=dev -ENV_ID=$(source ./scripts/print-env-id.sh) -ID_ENV_PREFIX="$ENV_ID/$ENV" -REPO_NAME="$ID_ENV_PREFIX/$APP_NAME" -REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) -AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) - -# returns: sha256:a9be998dcb5479d44b471381b79cb26a3b864d0fe996943c7abaf42c99638c9d sha256:ae20a55d89d8cece6f38dad19806637b400888fc6375bc167d6cf252e34dcb93 sha256:67dcb4b0a3bbc0bafb16cc163178f3b6c0a0d103694c26f085197c7015c33914 -IMAGE_DIGESTS=($(aws ecr list-images --repository-name $REPO_NAME --registry-id $AWS_ACCOUNT_ID --region $REGION --query 'imageIds[*].imageDigest' --output text)) - -declare IMAGE_IDS -# add imageDigest assignments to IMAGE_IDS -for i in "${IMAGE_DIGESTS[@]}"; do - IMAGE_IDS+="imageDigest=$i " -done - -# remove trailing whitespace -IMAGE_IDS=$(echo $IMAGE_IDS | xargs) - -aws ecr batch-delete-image --repository-name $REPO_NAME --registry-id $AWS_ACCOUNT_ID --region $REGION --image-ids $IMAGE_IDS \ No newline at end of file diff --git a/scripts/delete-dev-storage.sh b/scripts/delete-dev-storage.sh deleted file mode 100644 index c655a3a5a..000000000 --- a/scripts/delete-dev-storage.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -ENV=dev # hardcoding intended -PROJECT_CONF=project.yaml -REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) -ENV_ID=$(source ./scripts/print-env-id.sh) -ID_ENV="$ENV_ID-$ENV" -ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) -TFSTATE_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.TFSTATE_BUCKET_PREFIX.default' $PROJECT_CONF) -LOCAL_TFSTATE_FILE=terraform.tfstate - -ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" -TFSTATE_BUCKET="$TFSTATE_BUCKET_PREFIX-$ID_ENV" - -INIT_DEV_DIR=infra/terraform/aws/environments/init-dev - -export AWS_DEFAULT_REGION="$REGION" - -function delete_bucket() { - local bucket_name="$1" - - aws s3 rm "s3://$bucket_name" --recursive - - aws s3api delete-bucket --bucket "$bucket_name" -} - -function delete_dev_storage() { - # delete buckets - delete_bucket "$ARTIFACTS_BUCKET" - delete_bucket "$TFSTATE_BUCKET" - - popd - source ./scripts/delete-ecr-repos.sh - pushd $INIT_DEV_DIR -} - -pushd $INIT_DEV_DIR - -if ! [[ -f $LOCAL_TFSTATE_FILE ]]; then - echo "tfstate not found. manually deleting dev storage" - delete_dev_storage -elif [[ $(yq '.resources | length' $LOCAL_TFSTATE_FILE) -eq 0 ]]; then - echo "resources not found in tfstate. manually deleting dev storage" - delete_dev_storage -fi - -set +e -terraform destroy --auto-approve 2>/dev/null - -if [[ "$?" -ne 0 ]]; then - echo "destroy incomplete. manually deleting dev storage" - delete_dev_storage - popd -else - popd -fi diff --git a/scripts/delete-ecr-repos.sh b/scripts/delete-ecr-repos.sh deleted file mode 100644 index 6643df829..000000000 --- a/scripts/delete-ecr-repos.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -PROJECT_CONF=project.yaml -ENV=dev -ENV_ID=$(source scripts/print-env-id.sh) -REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) - -for APP_NAME in $(bash scripts/list-deployments.sh | xargs basename -a); do - IMAGE_NAME="$ENV_ID/$ENV/$APP_NAME" - aws ecr delete-repository --repository-name $IMAGE_NAME --region $REGION --force -done \ No newline at end of file diff --git a/scripts/deploy-all-images.sh b/scripts/deploy-all-images.sh deleted file mode 100644 index c2d172515..000000000 --- a/scripts/deploy-all-images.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash - -set -e - -if [[ -z $GITHUB_PAT ]]; then - echo "set GITHUB_PAT variable in shell to continue" - exit 1 -fi - -ENV=dev -PROJECT_CONF=project.yaml -ENV_ID=$(source ./scripts/print-env-id.sh) -ID_ENV="$ENV_ID-$ENV" -REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) -ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) -ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" -DEPLOY_IMAGE_WORKFLOW=$(yq '.[".github"].workflows.env_var.set.DEPLOY_IMAGE_WORKFLOW.default' $PROJECT_CONF) -WORKFLOW_ID=$DEPLOY_IMAGE_WORKFLOW -GITHUB_ORG=$(yq '.[".github"].env_var.set.GITHUB_ORG.default' $PROJECT_CONF) -GITHUB_REPO_NAME=$(yq '.[".github"].env_var.set.GITHUB_REPO_NAME.default' $PROJECT_CONF) -SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) - -source scripts/zip-services.sh - -echo '*** uploading archive to s3' -aws s3 cp $SERVICES_ZIP s3://$ARTIFACTS_BUCKET --region $REGION -rm $SERVICES_ZIP - -echo "*** triggering .github/workflows/$WORKFLOW_ID" - -curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_PAT" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/workflows/$WORKFLOW_ID/dispatches \ - -d "{\"ref\":\"develop\"}" - -if [[ $(uname) == "Darwin" ]]; then - # store utc time in iso8601 format minus 10 seconds - CREATED=$(date -u -v-10S "+%Y-%m-%dT%H:%M:%SZ") - # store utc time in unix timestamp format plus 10 minutes - TEN_MIN_MAX=$(date -u -v+10M "+%s") -else - CREATED=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 10 seconds ago" +'%Y-%m-%dT%H:%M:%SZ') - TEN_MIN_MAX=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 10 minutes" +'%s') -fi - -# wait 5 seconds -sleep 5 - -RUN_ID=$(curl -sL \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_PAT" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/workflows/$WORKFLOW_ID/runs?created=>$CREATED" | yq '.workflow_runs[0].id') - -echo "*** waiting for $WORKFLOW_ID github workflow to complete" - -function get_run() { - RUN=$(curl -sL \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_PAT" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/runs/$RUN_ID) - - STATUS=$(echo "$RUN" | yq '.status') - CONCLUSION=$(echo "$RUN" | yq '.conclusion') -} - -STATUS='queued' -CONCLUSION=null - -while [[ $STATUS != 'completed' && $(date +%s) -lt $TEN_MIN_MAX ]]; do - sleep 5 - get_run - printf '%s' '.' -done - -echo "" - -if [[ $CONCLUSION != 'success' ]]; then - echo "build failed" - exit 1 -fi \ No newline at end of file diff --git a/scripts/deploy-image-job.sh b/scripts/deploy-image-job.sh deleted file mode 100644 index e9b1ee80d..000000000 --- a/scripts/deploy-image-job.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -set -e - -# set in .github/workflows/deploy-all-images.yaml -if [[ -z $ENV_ID ]]; then - echo "ENV_ID is not set" - exit 1 -fi - -if [[ "$#" -ne 2 ]]; then - echo "use: bash scripts/deploy-image-job.sh --service-name request-create" - exit 1 -fi - -while [[ "$#" -gt 0 ]]; do - case $1 in - --service-name) SERVICE_NAME="$2"; shift ;; - *) echo "unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - -PROJECT_CONF=project.yaml -REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) -ENV=dev -ID_ENV="$ENV_ID-$ENV" -ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) -ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" -SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) -PROJECT_DIR=$(echo $SERVICES_ZIP | sed 's/.zip//') - -source scripts/auth-ecr.sh - -aws s3 cp s3://$ARTIFACTS_BUCKET/$SERVICES_ZIP . --region $REGION - -unzip $SERVICES_ZIP -d $PROJECT_DIR - -cd $PROJECT_DIR - -SERVICE_DIR=$(bash scripts/list-deployments.sh | grep --color=never $SERVICE_NAME) - -cd "$SERVICE_DIR" - -make --no-print-directory deploy \ No newline at end of file diff --git a/scripts/pull-all-images.sh b/scripts/pull-all-images.sh deleted file mode 100644 index 34af58320..000000000 --- a/scripts/pull-all-images.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -e - -PROJECT_CONF=project.yaml -GITHUB_REGISTRY=$(yq '.[".github"].workflows.env_var.set.GITHUB_REGISTRY.default' $PROJECT_CONF) -GITHUB_ORG=$(yq '.[".github"].env_var.set.GITHUB_ORG.default' $PROJECT_CONF) -GITHUB_REPO_NAME=$(yq '.[".github"].env_var.set.GITHUB_REPO_NAME.default' $PROJECT_CONF) -LOCAL_TAG_VERSION=$(yq '.docker.env_var.set.LOCAL_TAG_VERSION.default' $PROJECT_CONF) - -NAMESPACE=$GITHUB_ORG/$GITHUB_REPO_NAME -REGISTRY_URI=$GITHUB_REGISTRY/$NAMESPACE - -SERVICES=($(bash scripts/list-deployments.sh | xargs basename -a)) - -for SERVICE in "${SERVICES[@]}"; do - IMAGE_NAME=$SERVICE:$LOCAL_TAG_VERSION - docker pull $REGISTRY_URI/$IMAGE_NAME - docker tag $REGISTRY_URI/$IMAGE_NAME $IMAGE_NAME -done - -echo "" -echo '*** "make compose-up" to start services in docker' \ No newline at end of file From f1d482469893b227c1de0c34a704ffa2fbc2691f Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:54:07 -0800 Subject: [PATCH 23/25] add codebuild object key path vars --- project.yaml | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/project.yaml b/project.yaml index 08d13b52c..28f4a68a0 100644 --- a/project.yaml +++ b/project.yaml @@ -1,12 +1,6 @@ .github: env_var: - set: - GITHUB_ORG: - ssm: null - default: systemaccounting - GITHUB_REPO_NAME: - ssm: null - default: mxfactorial + set: {} get: [] params: [] codecov: @@ -16,16 +10,7 @@ - ui workflows: env_var: - set: - GITHUB_REGISTRY: - ssm: null - default: ghcr.io - IMAGE_BUILDER_WORKFLOW: - ssm: null - default: build-all-images.yaml - DEPLOY_IMAGE_WORKFLOW: - ssm: null - default: deploy-all-images.yaml + set: {} get: [] params: [] client: @@ -146,9 +131,6 @@ docker: DOCKER_USER: ssm: null default: mxfactorial - LOCAL_TAG_VERSION: - ssm: null - default: latest get: [] infra: terraform: @@ -177,6 +159,13 @@ infra: ssm: null default: provided.al2023 get: [] + codepipeline: + env_var: + set: + BUILD_OBJECT_KEY_PATH: + ssm: null + default: build + get: [] environment: env_var: set: @@ -213,7 +202,7 @@ infra: REGION: ssm: null default: us-east-1 - RDS_PREFIX: + NAME_PREFIX: ssm: null default: mxfactorial SSM_VERSION: @@ -246,9 +235,6 @@ infra: BALANCE_BY_ACCOUNT_URL: ssm: service/lambda/balance_by_account/url default: null - RDS_INSTANCE_NAME_PREFIX: - ssm: null - default: mxfactorial READINESS_CHECK_PATH: ssm: service/lambda/readiness_check_path default: /healthz @@ -305,6 +291,9 @@ infra: TFSTATE_BUCKET_PREFIX: ssm: null default: mxfactorial-tfstate + INTEG_TEST_OBJECT_KEY_PATH: + ssm: null + default: integ get: [] k8s: local: From d40f3e877b1cb0047eca3099330f7232f0f2a676 Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 18:54:44 -0800 Subject: [PATCH 24/25] codebuild and ecr instructions --- .agents/onboard.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.agents/onboard.md b/.agents/onboard.md index 3abc130c2..5daac3c0b 100644 --- a/.agents/onboard.md +++ b/.agents/onboard.md @@ -14,7 +14,7 @@ read scripts/bootcamp.sh to learn how to test services `make start` to start services in docker. services use `cargo-watch` for hot-reloading during development -test the bootcamp commands to learn services requests and responses +test the bootcamp commands to learn services requests and responses. the primary transaction bootcamp commands are `make rule`, `make request-create`, `make request-approve` and `make balance-by-account` project.yaml is project config uniformly sourced in bash, make and terraform code to avoid scattering it across the project. its large so parse when possible. for example, `yq .services.rule.env_var.set project.yaml` to list environment variables set by services/rule and `yq .services.rule.env_var.get project.yaml` to list variables it requires @@ -102,4 +102,16 @@ check cloudwatch logs: `aws logs tail /aws/lambda/SERVICE-ENVID-ENV --since 5m - image tags use `SHORT_GIT_SHA_LENGTH` from project.yaml (default: 7) for git hash length -use yq instead of jq for local json/yaml scripting \ No newline at end of file +use yq instead of jq for local json/yaml scripting + +`bash scripts/ecr-images.sh --build` triggers codebuild to build+test images remotely. add `--push` to push to ECR, `--deploy` to update lambdas, `--no-test` to skip tests, `--service graphql` to target one service + +`bash scripts/ecr-images.sh --integ` runs integration tests in codebuild using pre-built ECR images (test-db, test-cache, test-local, client e2e). requires `--build --push` first + +`bash scripts/ecr-images.sh --pull` pulls images from ECR and retags locally + +codebuild projects are in infra/terraform/aws/modules/project-storage/v001. per-service builds in codepipeline.tf (sources codebuild module), integ tests in integ.tf (buildspec inlined) + +buildspecs use runtime config via env vars (RUN_TESTS, PUSH_IMAGE, DEPLOY) set by `--environment-variables-override` in ecr-images.sh + +s3 upload to artifacts bucket auto-triggers codepipeline via eventbridge when using `--build --push` without `--service` \ No newline at end of file From 242aefbd2f7749276eea1ddd6cf878054a1e4aae Mon Sep 17 00:00:00 2001 From: max funk Date: Mon, 19 Jan 2026 21:01:28 -0800 Subject: [PATCH 25/25] shared service workflow --- .github/workflows/REUSE_service.yaml | 154 ----------- .github/workflows/balance-by-account.yaml | 27 -- .github/workflows/graphql.yaml | 26 -- .github/workflows/mirror-images.yaml | 50 ++++ .github/workflows/request-approve.yaml | 27 -- .github/workflows/request-by-id.yaml | 27 -- .github/workflows/request-create.yaml | 27 -- .github/workflows/requests-by-account.yaml | 27 -- .github/workflows/rule.yaml | 27 -- .github/workflows/services.yaml | 245 ++++++++++++++++++ .github/workflows/transaction-by-id.yaml | 27 -- .../workflows/transactions-by-account.yaml | 27 -- crates/pg/Cargo.toml | 1 + make/ecr-lambda.mk | 2 +- scripts/build-image.sh | 10 +- scripts/start-local.sh | 12 +- 16 files changed, 311 insertions(+), 405 deletions(-) delete mode 100644 .github/workflows/REUSE_service.yaml delete mode 100644 .github/workflows/balance-by-account.yaml delete mode 100644 .github/workflows/graphql.yaml create mode 100644 .github/workflows/mirror-images.yaml delete mode 100644 .github/workflows/request-approve.yaml delete mode 100644 .github/workflows/request-by-id.yaml delete mode 100644 .github/workflows/request-create.yaml delete mode 100644 .github/workflows/requests-by-account.yaml delete mode 100644 .github/workflows/rule.yaml create mode 100644 .github/workflows/services.yaml delete mode 100644 .github/workflows/transaction-by-id.yaml delete mode 100644 .github/workflows/transactions-by-account.yaml diff --git a/.github/workflows/REUSE_service.yaml b/.github/workflows/REUSE_service.yaml deleted file mode 100644 index 55c37abd4..000000000 --- a/.github/workflows/REUSE_service.yaml +++ /dev/null @@ -1,154 +0,0 @@ -name: REUSE_service - -on: - workflow_call: - inputs: - app_dir: - required: true - type: string - secrets: - AWS_ACCESS_KEY_ID: - required: true - AWS_SECRET_ACCESS_KEY: - required: true - AWS_ACCOUNT_ID: - required: true - DEV_ENV_ID: - required: true - -jobs: - lint_test: - name: lint test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: Swatinem/rust-cache@v2 - - name: linting - run: | - cargo fmt -- --check - cargo clippy -- -Dwarnings - - unit_test: - name: unit test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: Swatinem/rust-cache@v2 - - name: unit test - run: cargo test - - database_test: - name: database test in local docker - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: Swatinem/rust-cache@v2 - - name: test database - run: make -C crates/pg test-db - - compile: - name: compile ${{ matrix.service }} - runs-on: ubuntu-latest - strategy: - matrix: - service: - - graphql - - request-create - - request-approve - - rule - - request-by-id - - requests-by-account - - transaction-by-id - - transactions-by-account - - balance-by-account - # TODO: - # - event - # - measure - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - - uses: Swatinem/rust-cache@v2 - with: - shared-key: ${{ matrix.service }} - - name: compile - run: cargo build -p ${{ matrix.service }} - - name: upload binary - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.service }} - path: target/debug/${{ matrix.service }} - retention-days: 1 - - integration_test: - name: integration test in local docker - needs: compile - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: download all binaries - uses: actions/download-artifact@v4 - with: - path: target/debug/ - merge-multiple: true - - name: set permissions - run: chmod +x target/debug/* - - name: start services - run: make start - - name: test service integration - run: make -C ./tests test-local - - name: clean up - run: make stop - - client_test: - name: client test in local docker - needs: compile - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: download all binaries - uses: actions/download-artifact@v4 - with: - path: target/debug/ - merge-multiple: true - - name: set permissions - run: chmod +x target/debug/* - - name: start services - run: make start - - name: e2e test client - run: make -C ./client test-ci - - name: clean up - run: make stop - - push_image: - name: push image to dev ecr - runs-on: ubuntu-latest - needs: [lint_test, unit_test, database_test, integration_test, client_test] - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - APP_DIR: ${{ inputs.app_dir }} - steps: - - uses: actions/checkout@v4 - - name: mask values - run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - - name: build image - run: make -C $APP_DIR build-image - - name: tag image - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR tag-dev-image - - name: push image - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR push-dev-image diff --git a/.github/workflows/balance-by-account.yaml b/.github/workflows/balance-by-account.yaml deleted file mode 100644 index 113e156bc..000000000 --- a/.github/workflows/balance-by-account.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: balance-by-account - -on: - workflow_dispatch: - push: - paths: - - 'services/balance-by-account/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/balance-by-account - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/graphql.yaml b/.github/workflows/graphql.yaml deleted file mode 100644 index 8fb34a146..000000000 --- a/.github/workflows/graphql.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: graphql - -on: - workflow_dispatch: - push: - paths: - - 'services/graphql/**' - - 'crates/**' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/graphql - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/mirror-images.yaml b/.github/workflows/mirror-images.yaml new file mode 100644 index 000000000..377d88c17 --- /dev/null +++ b/.github/workflows/mirror-images.yaml @@ -0,0 +1,50 @@ +name: mirror-images + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * 0' # weekly + +jobs: + mirror: + name: mirror base images to ghcr + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - name: login to ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: mirror rust + run: | + docker pull public.ecr.aws/docker/library/rust:latest + docker tag public.ecr.aws/docker/library/rust:latest ghcr.io/${{ github.repository_owner }}/rust:latest + docker push ghcr.io/${{ github.repository_owner }}/rust:latest + - name: mirror lambda-provided + run: | + docker pull public.ecr.aws/lambda/provided:al2023 + docker tag public.ecr.aws/lambda/provided:al2023 ghcr.io/${{ github.repository_owner }}/lambda-provided:al2023 + docker push ghcr.io/${{ github.repository_owner }}/lambda-provided:al2023 + - name: mirror postgresql + run: | + docker pull public.ecr.aws/bitnami/postgresql:15 + docker tag public.ecr.aws/bitnami/postgresql:15 ghcr.io/${{ github.repository_owner }}/postgresql:15 + docker push ghcr.io/${{ github.repository_owner }}/postgresql:15 + - name: mirror redis + run: | + docker pull public.ecr.aws/bitnami/redis:latest + docker tag public.ecr.aws/bitnami/redis:latest ghcr.io/${{ github.repository_owner }}/redis:latest + docker push ghcr.io/${{ github.repository_owner }}/redis:latest + - name: mirror alpine + run: | + docker pull public.ecr.aws/docker/library/alpine:latest + docker tag public.ecr.aws/docker/library/alpine:latest ghcr.io/${{ github.repository_owner }}/alpine:latest + docker push ghcr.io/${{ github.repository_owner }}/alpine:latest + - name: mirror node + run: | + docker pull public.ecr.aws/docker/library/node:lts-alpine + docker tag public.ecr.aws/docker/library/node:lts-alpine ghcr.io/${{ github.repository_owner }}/node:lts-alpine + docker push ghcr.io/${{ github.repository_owner }}/node:lts-alpine diff --git a/.github/workflows/request-approve.yaml b/.github/workflows/request-approve.yaml deleted file mode 100644 index 7e449f2dc..000000000 --- a/.github/workflows/request-approve.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: request-approve - -on: - workflow_dispatch: - push: - paths: - - 'services/request-approve/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/request-approve - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/request-by-id.yaml b/.github/workflows/request-by-id.yaml deleted file mode 100644 index 780e9db9f..000000000 --- a/.github/workflows/request-by-id.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: request-by-id - -on: - workflow_dispatch: - push: - paths: - - 'services/request-by-id/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/request-by-id - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/request-create.yaml b/.github/workflows/request-create.yaml deleted file mode 100644 index b85ca3e20..000000000 --- a/.github/workflows/request-create.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: request-create - -on: - workflow_dispatch: - push: - paths: - - 'services/request-create/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/request-create - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/requests-by-account.yaml b/.github/workflows/requests-by-account.yaml deleted file mode 100644 index e2048e16e..000000000 --- a/.github/workflows/requests-by-account.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: requests-by-account - -on: - workflow_dispatch: - push: - paths: - - 'services/requests-by-account/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/requests-by-account - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/rule.yaml b/.github/workflows/rule.yaml deleted file mode 100644 index 6d105cf33..000000000 --- a/.github/workflows/rule.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: rule - -on: - workflow_dispatch: - push: - paths: - - 'services/rule/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/rule - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/services.yaml b/.github/workflows/services.yaml new file mode 100644 index 000000000..4920d83da --- /dev/null +++ b/.github/workflows/services.yaml @@ -0,0 +1,245 @@ +name: services + +on: + workflow_dispatch: + push: + paths: + - 'services/graphql/**' + - 'services/request-create/**' + - 'services/request-approve/**' + - 'services/rule/**' + - 'services/request-by-id/**' + - 'services/requests-by-account/**' + - 'services/transaction-by-id/**' + - 'services/transactions-by-account/**' + - 'services/balance-by-account/**' + - 'crates/**' + - 'migrations/schema/*' + branches-ignore: + - 'master' + - 'develop' + +concurrency: + group: services-${{ github.ref }} + cancel-in-progress: true + +jobs: + cache_images: + name: cache base images + runs-on: ubuntu-latest + permissions: + packages: read + steps: + - name: login to ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: restore image cache + id: cache + uses: actions/cache@v4 + with: + path: ~/image-cache + key: base-images-v1 + - name: pull and retag images + if: steps.cache.outputs.cache-hit != 'true' + run: | + mkdir -p ~/image-cache + docker pull ghcr.io/systemaccounting/rust:latest + docker tag ghcr.io/systemaccounting/rust:latest public.ecr.aws/docker/library/rust:latest + docker pull ghcr.io/systemaccounting/lambda-provided:al2023 + docker tag ghcr.io/systemaccounting/lambda-provided:al2023 public.ecr.aws/lambda/provided:al2023 + docker pull ghcr.io/systemaccounting/postgresql:15 + docker tag ghcr.io/systemaccounting/postgresql:15 public.ecr.aws/bitnami/postgresql:15 + docker pull ghcr.io/systemaccounting/redis:latest + docker tag ghcr.io/systemaccounting/redis:latest public.ecr.aws/bitnami/redis:latest + docker pull ghcr.io/systemaccounting/alpine:latest + docker tag ghcr.io/systemaccounting/alpine:latest public.ecr.aws/docker/library/alpine:latest + docker pull ghcr.io/systemaccounting/node:lts-alpine + docker tag ghcr.io/systemaccounting/node:lts-alpine public.ecr.aws/docker/library/node:lts-alpine + docker save -o ~/image-cache/images.tar \ + public.ecr.aws/docker/library/rust:latest \ + public.ecr.aws/lambda/provided:al2023 \ + public.ecr.aws/bitnami/postgresql:15 \ + public.ecr.aws/bitnami/redis:latest \ + public.ecr.aws/docker/library/alpine:latest \ + public.ecr.aws/docker/library/node:lts-alpine + + lint_test: + name: lint test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - uses: Swatinem/rust-cache@v2 + - name: linting + run: | + cargo fmt -- --check + cargo clippy -- -Dwarnings + + unit_test: + name: unit test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - uses: Swatinem/rust-cache@v2 + - name: unit test + run: cargo test + + database_test: + name: database test in local docker + needs: cache_images + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: restore image cache + uses: actions/cache@v4 + with: + path: ~/image-cache + key: base-images-v1 + - name: load cached images + run: docker load -i ~/image-cache/images.tar + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - uses: Swatinem/rust-cache@v2 + - name: test database + run: make -C crates/pg test-db + + compile: + name: compile ${{ matrix.service }} + runs-on: ubuntu-latest + strategy: + matrix: + service: + - graphql + - request-create + - request-approve + - rule + - request-by-id + - requests-by-account + - transaction-by-id + - transactions-by-account + - balance-by-account + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - uses: Swatinem/rust-cache@v2 + with: + shared-key: ${{ matrix.service }} + - name: compile + run: cargo build -p ${{ matrix.service }} + - name: upload binary + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.service }} + path: target/debug/${{ matrix.service }} + retention-days: 1 + + integration_test: + name: integration test in local docker + needs: [cache_images, compile] + runs-on: ubuntu-latest + env: + SERVICES_WORKFLOW: true + steps: + - uses: actions/checkout@v4 + - name: restore image cache + uses: actions/cache@v4 + with: + path: ~/image-cache + key: base-images-v1 + - name: load cached images + run: docker load -i ~/image-cache/images.tar + - name: download all binaries + uses: actions/download-artifact@v4 + with: + path: target/debug/ + merge-multiple: true + - name: set permissions + run: chmod +x target/debug/* + - name: start services + run: make start + - name: test service integration + run: make -C ./tests test-local + - name: clean up + run: make stop + + client_test: + name: client test in local docker + needs: [cache_images, compile] + runs-on: ubuntu-latest + env: + SERVICES_WORKFLOW: true + steps: + - uses: actions/checkout@v4 + - name: restore image cache + uses: actions/cache@v4 + with: + path: ~/image-cache + key: base-images-v1 + - name: load cached images + run: docker load -i ~/image-cache/images.tar + - name: download all binaries + uses: actions/download-artifact@v4 + with: + path: target/debug/ + merge-multiple: true + - name: set permissions + run: chmod +x target/debug/* + - name: start services + run: make start + - name: e2e test client + run: make -C ./client test-ci + - name: clean up + run: make stop + + push_images: + name: push ${{ matrix.service }} image + runs-on: ubuntu-latest + needs: [cache_images, lint_test, unit_test, database_test, integration_test, client_test] + strategy: + matrix: + service: + - graphql + - request-create + - request-approve + - rule + - request-by-id + - requests-by-account + - transaction-by-id + - transactions-by-account + - balance-by-account + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + NO_TEST: true + steps: + - uses: actions/checkout@v4 + - name: restore image cache + uses: actions/cache@v4 + with: + path: ~/image-cache + key: base-images-v1 + - name: load cached images + run: docker load -i ~/image-cache/images.tar + - name: mask values + run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" + - name: build image + run: make -C services/${{ matrix.service }} build-image + - name: tag image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C services/${{ matrix.service }} tag-dev-image + - name: push image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C services/${{ matrix.service }} push-dev-image diff --git a/.github/workflows/transaction-by-id.yaml b/.github/workflows/transaction-by-id.yaml deleted file mode 100644 index 1a5b85568..000000000 --- a/.github/workflows/transaction-by-id.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: transaction-by-id - -on: - workflow_dispatch: - push: - paths: - - 'services/transaction-by-id/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/transaction-by-id - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/transactions-by-account.yaml b/.github/workflows/transactions-by-account.yaml deleted file mode 100644 index 34ed7eb34..000000000 --- a/.github/workflows/transactions-by-account.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: transactions-by-account - -on: - workflow_dispatch: - push: - paths: - - 'services/transactions-by-account/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/transactions-by-account - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/crates/pg/Cargo.toml b/crates/pg/Cargo.toml index be77f3aed..92f207eb6 100644 --- a/crates/pg/Cargo.toml +++ b/crates/pg/Cargo.toml @@ -2,6 +2,7 @@ name = "pg" version = "0.1.0" edition = "2021" +description = "postgresql db access" [dependencies] bb8 = "0.9" diff --git a/make/ecr-lambda.mk b/make/ecr-lambda.mk index 2351dd5ec..85a6e7c43 100644 --- a/make/ecr-lambda.mk +++ b/make/ecr-lambda.mk @@ -2,7 +2,7 @@ BUILD_CTX?=. build-image: @cd $(RELATIVE_PROJECT_ROOT_PATH); \ - bash scripts/build-image.sh --app-name $(APP_NAME) --build-ctx $(BUILD_CTX) + bash scripts/build-image.sh --app-name $(APP_NAME) --build-ctx $(BUILD_CTX) $(if $(NO_TEST),--no-test) tag-dev-image: @cd $(RELATIVE_PROJECT_ROOT_PATH); \ diff --git a/scripts/build-image.sh b/scripts/build-image.sh index 25990b1f3..a8cf19df2 100644 --- a/scripts/build-image.sh +++ b/scripts/build-image.sh @@ -1,13 +1,16 @@ #!/bin/bash -if [[ "$#" -ne 4 ]]; then +if [[ "$#" -lt 4 ]]; then cat <<-'EOF' use: bash scripts/build-image.sh --app-name rule --build-ctx . + bash scripts/build-image.sh --app-name rule --build-ctx . --no-test EOF exit 1 fi +BUILD_ARGS="" + while [[ "$#" -gt 0 ]]; do case $1 in --app-name) @@ -18,6 +21,9 @@ while [[ "$#" -gt 0 ]]; do BUILD_CTX="$2" shift ;; + --no-test) + BUILD_ARGS="--build-arg RUN_TESTS=false" + ;; *) echo "unknown parameter passed: $1" exit 1 @@ -32,4 +38,4 @@ HASH=$(git rev-parse --short=$SHORT_GIT_SHA_LENGTH HEAD) IMAGE_TAG="$APP_NAME:$HASH" DOCKERFILE_PATH=./docker/$APP_NAME.Dockerfile -docker build -f $DOCKERFILE_PATH -t $IMAGE_TAG --provenance=false "$BUILD_CTX" \ No newline at end of file +docker build -f $DOCKERFILE_PATH -t $IMAGE_TAG --provenance=false $BUILD_ARGS "$BUILD_CTX" \ No newline at end of file diff --git a/scripts/start-local.sh b/scripts/start-local.sh index dec176902..f6204bb3e 100644 --- a/scripts/start-local.sh +++ b/scripts/start-local.sh @@ -44,24 +44,24 @@ for d in "${APP_DIRS[@]}"; do RUNTIME=$(yq "$CONF_PATH.runtime" $PROJECT_CONF) BUILD_SRC_PATH=$(yq "$CONF_PATH.build_src_path" $PROJECT_CONF) - # compile rust before starting (skip in CI - uses pre-built binaries) - if [[ -z "$CI" ]] && [[ "$RUNTIME" == "$RUST_RUNTIME" ]]; then + # compile rust before starting (skip when using pre-built binaries from services workflow artifacts) + if [[ -z "$SERVICES_WORKFLOW" ]] && [[ "$RUNTIME" == "$RUST_RUNTIME" ]]; then echo -e -n "\n${GREEN}*** compiling $d${RESET}\n" make --no-print-directory -C "$d" compile fi - # skip starting client in workflows - if [[ "$CI" ]] && [[ "$d" == 'client' ]]; then + # skip starting client in services workflows + if [[ "$SERVICES_WORKFLOW" ]] && [[ "$d" == 'client' ]]; then continue fi echo -e -n "\n${GREEN}*** starting $d${RESET}\n" - if [[ "$CI" ]]; then + if [[ "$SERVICES_WORKFLOW" ]]; then make --no-print-directory -C "$d" get-secrets ENV=local > /dev/null # &; \ disown fails in make so backgrounding kept in bash if [[ "$RUNTIME" == "$RUST_RUNTIME" ]]; then - # run pre-built binary from CI matrix compile job + # run pre-built binary from services workflow compile job artifacts # skips cargo fingerprint check which would recompile BINARY_PATH="$(pwd)/target/debug/$(basename "$d")" (cd "$d"; eval $(cat $ENV_FILE_NAME) $BINARY_PATH > /dev/null 2>&1 & disown $!)