diff --git a/.agents/onboard.md b/.agents/onboard.md index 3abc130c2..5daac3c0b 100644 --- a/.agents/onboard.md +++ b/.agents/onboard.md @@ -14,7 +14,7 @@ read scripts/bootcamp.sh to learn how to test services `make start` to start services in docker. services use `cargo-watch` for hot-reloading during development -test the bootcamp commands to learn services requests and responses +test the bootcamp commands to learn services requests and responses. the primary transaction bootcamp commands are `make rule`, `make request-create`, `make request-approve` and `make balance-by-account` project.yaml is project config uniformly sourced in bash, make and terraform code to avoid scattering it across the project. its large so parse when possible. for example, `yq .services.rule.env_var.set project.yaml` to list environment variables set by services/rule and `yq .services.rule.env_var.get project.yaml` to list variables it requires @@ -102,4 +102,16 @@ check cloudwatch logs: `aws logs tail /aws/lambda/SERVICE-ENVID-ENV --since 5m - image tags use `SHORT_GIT_SHA_LENGTH` from project.yaml (default: 7) for git hash length -use yq instead of jq for local json/yaml scripting \ No newline at end of file +use yq instead of jq for local json/yaml scripting + +`bash scripts/ecr-images.sh --build` triggers codebuild to build+test images remotely. add `--push` to push to ECR, `--deploy` to update lambdas, `--no-test` to skip tests, `--service graphql` to target one service + +`bash scripts/ecr-images.sh --integ` runs integration tests in codebuild using pre-built ECR images (test-db, test-cache, test-local, client e2e). requires `--build --push` first + +`bash scripts/ecr-images.sh --pull` pulls images from ECR and retags locally + +codebuild projects are in infra/terraform/aws/modules/project-storage/v001. per-service builds in codepipeline.tf (sources codebuild module), integ tests in integ.tf (buildspec inlined) + +buildspecs use runtime config via env vars (RUN_TESTS, PUSH_IMAGE, DEPLOY) set by `--environment-variables-override` in ecr-images.sh + +s3 upload to artifacts bucket auto-triggers codepipeline via eventbridge when using `--build --push` without `--service` \ No newline at end of file diff --git a/.github/workflows/REUSE_service.yaml b/.github/workflows/REUSE_service.yaml deleted file mode 100644 index 55c37abd4..000000000 --- a/.github/workflows/REUSE_service.yaml +++ /dev/null @@ -1,154 +0,0 @@ -name: REUSE_service - -on: - workflow_call: - inputs: - app_dir: - required: true - type: string - secrets: - AWS_ACCESS_KEY_ID: - required: true - AWS_SECRET_ACCESS_KEY: - required: true - AWS_ACCOUNT_ID: - required: true - DEV_ENV_ID: - required: true - -jobs: - lint_test: - name: lint test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: Swatinem/rust-cache@v2 - - name: linting - run: | - cargo fmt -- --check - cargo clippy -- -Dwarnings - - unit_test: - name: unit test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: Swatinem/rust-cache@v2 - - name: unit test - run: cargo test - - database_test: - name: database test in local docker - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - components: clippy, rustfmt - - uses: Swatinem/rust-cache@v2 - - name: test database - run: make -C crates/pg test-db - - compile: - name: compile ${{ matrix.service }} - runs-on: ubuntu-latest - strategy: - matrix: - service: - - graphql - - request-create - - request-approve - - rule - - request-by-id - - requests-by-account - - transaction-by-id - - transactions-by-account - - balance-by-account - # TODO: - # - event - # - measure - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - - uses: Swatinem/rust-cache@v2 - with: - shared-key: ${{ matrix.service }} - - name: compile - run: cargo build -p ${{ matrix.service }} - - name: upload binary - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.service }} - path: target/debug/${{ matrix.service }} - retention-days: 1 - - integration_test: - name: integration test in local docker - needs: compile - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: download all binaries - uses: actions/download-artifact@v4 - with: - path: target/debug/ - merge-multiple: true - - name: set permissions - run: chmod +x target/debug/* - - name: start services - run: make start - - name: test service integration - run: make -C ./tests test-local - - name: clean up - run: make stop - - client_test: - name: client test in local docker - needs: compile - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: download all binaries - uses: actions/download-artifact@v4 - with: - path: target/debug/ - merge-multiple: true - - name: set permissions - run: chmod +x target/debug/* - - name: start services - run: make start - - name: e2e test client - run: make -C ./client test-ci - - name: clean up - run: make stop - - push_image: - name: push image to dev ecr - runs-on: ubuntu-latest - needs: [lint_test, unit_test, database_test, integration_test, client_test] - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - APP_DIR: ${{ inputs.app_dir }} - steps: - - uses: actions/checkout@v4 - - name: mask values - run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" - - name: build image - run: make -C $APP_DIR build-image - - name: tag image - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR tag-dev-image - - name: push image - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C $APP_DIR push-dev-image diff --git a/.github/workflows/balance-by-account.yaml b/.github/workflows/balance-by-account.yaml deleted file mode 100644 index 113e156bc..000000000 --- a/.github/workflows/balance-by-account.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: balance-by-account - -on: - workflow_dispatch: - push: - paths: - - 'services/balance-by-account/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/balance-by-account - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/build-all-images.yaml b/.github/workflows/build-all-images.yaml deleted file mode 100644 index e581dcb13..000000000 --- a/.github/workflows/build-all-images.yaml +++ /dev/null @@ -1,331 +0,0 @@ -name: build-all-images - -on: - workflow_dispatch: - -env: - ENV_ID: ${{ secrets.DEV_ENV_ID }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GITHUB_REGISTRY: ghcr.io - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - rule: - name: rule - runs-on: ubuntu-latest - env: - SERVICE_NAME: rule - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - balance_by_account: - name: balance-by-account - runs-on: ubuntu-latest - env: - SERVICE_NAME: balance-by-account - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - graphql: - name: graphql - runs-on: ubuntu-latest - env: - SERVICE_NAME: graphql - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - request_create: - name: request-create - runs-on: ubuntu-latest - env: - SERVICE_NAME: request-create - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - request_approve: - name: request-approve - runs-on: ubuntu-latest - env: - SERVICE_NAME: request-approve - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - request_by_id: - name: request-by-id - runs-on: ubuntu-latest - env: - SERVICE_NAME: request-by-id - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - requests_by_account: - name: requests-by-account - runs-on: ubuntu-latest - env: - SERVICE_NAME: requests-by-account - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - transaction_by_id: - name: transaction-by-id - runs-on: ubuntu-latest - env: - SERVICE_NAME: transaction-by-id - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - transactions_by_account: - name: transactions-by-account - runs-on: ubuntu-latest - env: - SERVICE_NAME: transactions-by-account - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - event: - name: event - runs-on: ubuntu-latest - env: - SERVICE_NAME: event - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - measure: - name: measure - runs-on: ubuntu-latest - env: - SERVICE_NAME: measure - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - auto_confirm: - name: auto-confirm - runs-on: ubuntu-latest - env: - SERVICE_NAME: auto-confirm - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - go_migrate: - name: go-migrate - runs-on: ubuntu-latest - env: - SERVICE_NAME: go-migrate - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx ./migrations/go-migrate - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - warm_cache: - name: warm-cache - runs-on: ubuntu-latest - env: - SERVICE_NAME: warm-cache - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME - client: - name: client - runs-on: ubuntu-latest - env: - SERVICE_NAME: client - steps: - - uses: actions/checkout@v4 - - name: build image - run: bash scripts/build-image-job.sh --service-name $SERVICE_NAME --build-ctx . - - name: tag image with current short sha and latest - run: | - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:${GITHUB_SHA:0:7} - docker tag $SERVICE_NAME:latest ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME:latest - - name: log into container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: push images - run: docker push --all-tags ${{ env.GITHUB_REGISTRY }}/${{ github.repository }}/$SERVICE_NAME \ No newline at end of file diff --git a/.github/workflows/deploy-all-images.yaml b/.github/workflows/deploy-all-images.yaml deleted file mode 100644 index a039eba27..000000000 --- a/.github/workflows/deploy-all-images.yaml +++ /dev/null @@ -1,124 +0,0 @@ -name: deploy-all-images - -on: - workflow_dispatch: - -env: - ENV_ID: ${{ secrets.DEV_ENV_ID }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - rule: - name: rule - runs-on: ubuntu-latest - env: - SERVICE_NAME: rule - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - balance_by_account: - name: balance-by-account - runs-on: ubuntu-latest - env: - SERVICE_NAME: balance-by-account - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - graphql: - name: graphql - runs-on: ubuntu-latest - env: - SERVICE_NAME: graphql - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - request_create: - name: request-create - runs-on: ubuntu-latest - env: - SERVICE_NAME: request-create - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - request_approve: - name: request-approve - runs-on: ubuntu-latest - env: - SERVICE_NAME: request-approve - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - request_by_id: - name: request-by-id - runs-on: ubuntu-latest - env: - SERVICE_NAME: request-by-id - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - requests_by_account: - name: requests-by-account - runs-on: ubuntu-latest - env: - SERVICE_NAME: requests-by-account - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - transaction_by_id: - name: transaction-by-id - runs-on: ubuntu-latest - env: - SERVICE_NAME: transaction-by-id - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - transactions_by_account: - name: transactions-by-account - runs-on: ubuntu-latest - env: - SERVICE_NAME: transactions-by-account - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - auto_confirm: - name: auto-confirm - runs-on: ubuntu-latest - env: - SERVICE_NAME: auto-confirm - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - go_migrate: - name: go-migrate - runs-on: ubuntu-latest - env: - SERVICE_NAME: go-migrate - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME - warm_cache: - name: warm-cache - runs-on: ubuntu-latest - env: - SERVICE_NAME: warm-cache - steps: - - uses: actions/checkout@v4 - - name: build and deploy image - run: bash scripts/deploy-image-job.sh --service-name $SERVICE_NAME \ No newline at end of file diff --git a/.github/workflows/dev-integration.yaml b/.github/workflows/dev-integration.yaml index 53e4aa7e9..372f1c976 100644 --- a/.github/workflows/dev-integration.yaml +++ b/.github/workflows/dev-integration.yaml @@ -49,7 +49,7 @@ jobs: - name: warm up lambdas with availability script run: bash scripts/test-availability.sh - name: reset dev rds - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make --no-print-directory -C ./migrations resetrds ENV=dev DB=test + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make --no-print-directory -C ./migrations/go-migrate resetrds ENV=dev DB=test - name: warm cache run: ENV_ID=${{ secrets.DEV_ENV_ID }} bash scripts/invoke-warm-cache.sh --env dev - name: run dev cloud integration tests diff --git a/.github/workflows/graphql.yaml b/.github/workflows/graphql.yaml deleted file mode 100644 index 8fb34a146..000000000 --- a/.github/workflows/graphql.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: graphql - -on: - workflow_dispatch: - push: - paths: - - 'services/graphql/**' - - 'crates/**' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/graphql - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/mirror-images.yaml b/.github/workflows/mirror-images.yaml new file mode 100644 index 000000000..377d88c17 --- /dev/null +++ b/.github/workflows/mirror-images.yaml @@ -0,0 +1,50 @@ +name: mirror-images + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * 0' # weekly + +jobs: + mirror: + name: mirror base images to ghcr + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - name: login to ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: mirror rust + run: | + docker pull public.ecr.aws/docker/library/rust:latest + docker tag public.ecr.aws/docker/library/rust:latest ghcr.io/${{ github.repository_owner }}/rust:latest + docker push ghcr.io/${{ github.repository_owner }}/rust:latest + - name: mirror lambda-provided + run: | + docker pull public.ecr.aws/lambda/provided:al2023 + docker tag public.ecr.aws/lambda/provided:al2023 ghcr.io/${{ github.repository_owner }}/lambda-provided:al2023 + docker push ghcr.io/${{ github.repository_owner }}/lambda-provided:al2023 + - name: mirror postgresql + run: | + docker pull public.ecr.aws/bitnami/postgresql:15 + docker tag public.ecr.aws/bitnami/postgresql:15 ghcr.io/${{ github.repository_owner }}/postgresql:15 + docker push ghcr.io/${{ github.repository_owner }}/postgresql:15 + - name: mirror redis + run: | + docker pull public.ecr.aws/bitnami/redis:latest + docker tag public.ecr.aws/bitnami/redis:latest ghcr.io/${{ github.repository_owner }}/redis:latest + docker push ghcr.io/${{ github.repository_owner }}/redis:latest + - name: mirror alpine + run: | + docker pull public.ecr.aws/docker/library/alpine:latest + docker tag public.ecr.aws/docker/library/alpine:latest ghcr.io/${{ github.repository_owner }}/alpine:latest + docker push ghcr.io/${{ github.repository_owner }}/alpine:latest + - name: mirror node + run: | + docker pull public.ecr.aws/docker/library/node:lts-alpine + docker tag public.ecr.aws/docker/library/node:lts-alpine ghcr.io/${{ github.repository_owner }}/node:lts-alpine + docker push ghcr.io/${{ github.repository_owner }}/node:lts-alpine diff --git a/.github/workflows/request-approve.yaml b/.github/workflows/request-approve.yaml deleted file mode 100644 index 7e449f2dc..000000000 --- a/.github/workflows/request-approve.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: request-approve - -on: - workflow_dispatch: - push: - paths: - - 'services/request-approve/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/request-approve - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/request-by-id.yaml b/.github/workflows/request-by-id.yaml deleted file mode 100644 index 780e9db9f..000000000 --- a/.github/workflows/request-by-id.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: request-by-id - -on: - workflow_dispatch: - push: - paths: - - 'services/request-by-id/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/request-by-id - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/request-create.yaml b/.github/workflows/request-create.yaml deleted file mode 100644 index b85ca3e20..000000000 --- a/.github/workflows/request-create.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: request-create - -on: - workflow_dispatch: - push: - paths: - - 'services/request-create/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/request-create - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/requests-by-account.yaml b/.github/workflows/requests-by-account.yaml deleted file mode 100644 index e2048e16e..000000000 --- a/.github/workflows/requests-by-account.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: requests-by-account - -on: - workflow_dispatch: - push: - paths: - - 'services/requests-by-account/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/requests-by-account - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/rule.yaml b/.github/workflows/rule.yaml deleted file mode 100644 index 6d105cf33..000000000 --- a/.github/workflows/rule.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: rule - -on: - workflow_dispatch: - push: - paths: - - 'services/rule/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/rule - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/services.yaml b/.github/workflows/services.yaml new file mode 100644 index 000000000..4920d83da --- /dev/null +++ b/.github/workflows/services.yaml @@ -0,0 +1,245 @@ +name: services + +on: + workflow_dispatch: + push: + paths: + - 'services/graphql/**' + - 'services/request-create/**' + - 'services/request-approve/**' + - 'services/rule/**' + - 'services/request-by-id/**' + - 'services/requests-by-account/**' + - 'services/transaction-by-id/**' + - 'services/transactions-by-account/**' + - 'services/balance-by-account/**' + - 'crates/**' + - 'migrations/schema/*' + branches-ignore: + - 'master' + - 'develop' + +concurrency: + group: services-${{ github.ref }} + cancel-in-progress: true + +jobs: + cache_images: + name: cache base images + runs-on: ubuntu-latest + permissions: + packages: read + steps: + - name: login to ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: restore image cache + id: cache + uses: actions/cache@v4 + with: + path: ~/image-cache + key: base-images-v1 + - name: pull and retag images + if: steps.cache.outputs.cache-hit != 'true' + run: | + mkdir -p ~/image-cache + docker pull ghcr.io/systemaccounting/rust:latest + docker tag ghcr.io/systemaccounting/rust:latest public.ecr.aws/docker/library/rust:latest + docker pull ghcr.io/systemaccounting/lambda-provided:al2023 + docker tag ghcr.io/systemaccounting/lambda-provided:al2023 public.ecr.aws/lambda/provided:al2023 + docker pull ghcr.io/systemaccounting/postgresql:15 + docker tag ghcr.io/systemaccounting/postgresql:15 public.ecr.aws/bitnami/postgresql:15 + docker pull ghcr.io/systemaccounting/redis:latest + docker tag ghcr.io/systemaccounting/redis:latest public.ecr.aws/bitnami/redis:latest + docker pull ghcr.io/systemaccounting/alpine:latest + docker tag ghcr.io/systemaccounting/alpine:latest public.ecr.aws/docker/library/alpine:latest + docker pull ghcr.io/systemaccounting/node:lts-alpine + docker tag ghcr.io/systemaccounting/node:lts-alpine public.ecr.aws/docker/library/node:lts-alpine + docker save -o ~/image-cache/images.tar \ + public.ecr.aws/docker/library/rust:latest \ + public.ecr.aws/lambda/provided:al2023 \ + public.ecr.aws/bitnami/postgresql:15 \ + public.ecr.aws/bitnami/redis:latest \ + public.ecr.aws/docker/library/alpine:latest \ + public.ecr.aws/docker/library/node:lts-alpine + + lint_test: + name: lint test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - uses: Swatinem/rust-cache@v2 + - name: linting + run: | + cargo fmt -- --check + cargo clippy -- -Dwarnings + + unit_test: + name: unit test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - uses: Swatinem/rust-cache@v2 + - name: unit test + run: cargo test + + database_test: + name: database test in local docker + needs: cache_images + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: restore image cache + uses: actions/cache@v4 + with: + path: ~/image-cache + key: base-images-v1 + - name: load cached images + run: docker load -i ~/image-cache/images.tar + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + components: clippy, rustfmt + - uses: Swatinem/rust-cache@v2 + - name: test database + run: make -C crates/pg test-db + + compile: + name: compile ${{ matrix.service }} + runs-on: ubuntu-latest + strategy: + matrix: + service: + - graphql + - request-create + - request-approve + - rule + - request-by-id + - requests-by-account + - transaction-by-id + - transactions-by-account + - balance-by-account + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - uses: Swatinem/rust-cache@v2 + with: + shared-key: ${{ matrix.service }} + - name: compile + run: cargo build -p ${{ matrix.service }} + - name: upload binary + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.service }} + path: target/debug/${{ matrix.service }} + retention-days: 1 + + integration_test: + name: integration test in local docker + needs: [cache_images, compile] + runs-on: ubuntu-latest + env: + SERVICES_WORKFLOW: true + steps: + - uses: actions/checkout@v4 + - name: restore image cache + uses: actions/cache@v4 + with: + path: ~/image-cache + key: base-images-v1 + - name: load cached images + run: docker load -i ~/image-cache/images.tar + - name: download all binaries + uses: actions/download-artifact@v4 + with: + path: target/debug/ + merge-multiple: true + - name: set permissions + run: chmod +x target/debug/* + - name: start services + run: make start + - name: test service integration + run: make -C ./tests test-local + - name: clean up + run: make stop + + client_test: + name: client test in local docker + needs: [cache_images, compile] + runs-on: ubuntu-latest + env: + SERVICES_WORKFLOW: true + steps: + - uses: actions/checkout@v4 + - name: restore image cache + uses: actions/cache@v4 + with: + path: ~/image-cache + key: base-images-v1 + - name: load cached images + run: docker load -i ~/image-cache/images.tar + - name: download all binaries + uses: actions/download-artifact@v4 + with: + path: target/debug/ + merge-multiple: true + - name: set permissions + run: chmod +x target/debug/* + - name: start services + run: make start + - name: e2e test client + run: make -C ./client test-ci + - name: clean up + run: make stop + + push_images: + name: push ${{ matrix.service }} image + runs-on: ubuntu-latest + needs: [cache_images, lint_test, unit_test, database_test, integration_test, client_test] + strategy: + matrix: + service: + - graphql + - request-create + - request-approve + - rule + - request-by-id + - requests-by-account + - transaction-by-id + - transactions-by-account + - balance-by-account + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + NO_TEST: true + steps: + - uses: actions/checkout@v4 + - name: restore image cache + uses: actions/cache@v4 + with: + path: ~/image-cache + key: base-images-v1 + - name: load cached images + run: docker load -i ~/image-cache/images.tar + - name: mask values + run: echo "::add-mask::${{ secrets.AWS_ACCOUNT_ID }}" + - name: build image + run: make -C services/${{ matrix.service }} build-image + - name: tag image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C services/${{ matrix.service }} tag-dev-image + - name: push image + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C services/${{ matrix.service }} push-dev-image diff --git a/.github/workflows/transaction-by-id.yaml b/.github/workflows/transaction-by-id.yaml deleted file mode 100644 index 1a5b85568..000000000 --- a/.github/workflows/transaction-by-id.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: transaction-by-id - -on: - workflow_dispatch: - push: - paths: - - 'services/transaction-by-id/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/transaction-by-id - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/transactions-by-account.yaml b/.github/workflows/transactions-by-account.yaml deleted file mode 100644 index 34ed7eb34..000000000 --- a/.github/workflows/transactions-by-account.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: transactions-by-account - -on: - workflow_dispatch: - push: - paths: - - 'services/transactions-by-account/**' - - 'crates/**' - - 'migrations/schema/*' - branches-ignore: - - 'master' - - 'develop' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/REUSE_service.yaml - with: - app_dir: services/transactions-by-account - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - DEV_ENV_ID: ${{ secrets.DEV_ENV_ID }} diff --git a/.github/workflows/warm-cache.yaml b/.github/workflows/warm-cache.yaml index 4f010cabb..600f57fc8 100644 --- a/.github/workflows/warm-cache.yaml +++ b/.github/workflows/warm-cache.yaml @@ -47,7 +47,7 @@ jobs: echo "redis key count: $(wc -l < /tmp/redis_keys.txt)" - name: reset rds # reset dev rds to match local postgres seed data - run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C migrations resetrds + run: ENV_ID=${{ secrets.DEV_ENV_ID }} make -C migrations/go-migrate resetrds - name: warm ddb cache run: | # temporarily add env vars to warm-cache get array so make env fetches them diff --git a/crates/pg/Cargo.toml b/crates/pg/Cargo.toml index be77f3aed..92f207eb6 100644 --- a/crates/pg/Cargo.toml +++ b/crates/pg/Cargo.toml @@ -2,6 +2,7 @@ name = "pg" version = "0.1.0" edition = "2021" +description = "postgresql db access" [dependencies] bb8 = "0.9" diff --git a/crates/pg/makefile b/crates/pg/makefile index 65286c81d..68be1ee75 100644 --- a/crates/pg/makefile +++ b/crates/pg/makefile @@ -1,3 +1,4 @@ +SHELL := /bin/bash RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") APP_NAME=$(shell basename $(CURDIR)) PROJECT_CONF_FILE_NAME=project.yaml diff --git a/cue/project_conf.cue b/cue/project_conf.cue index 42c3aa84b..021dbc288 100644 --- a/cue/project_conf.cue +++ b/cue/project_conf.cue @@ -86,6 +86,9 @@ infra: { } modules: { env_var!: #EnvVars + codepipeline: { + env_var!: #EnvVars + } environment: { env_var!: #EnvVars } diff --git a/docker/auto-confirm.Dockerfile b/docker/auto-confirm.Dockerfile index 21cd70ff7..b9c082d61 100644 --- a/docker/auto-confirm.Dockerfile +++ b/docker/auto-confirm.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,6 +10,11 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/auto-confirm/Cargo.toml \ --target x86_64-unknown-linux-musl \ diff --git a/docker/balance-by-account.Dockerfile b/docker/balance-by-account.Dockerfile index 814416c20..f0a4b50c4 100644 --- a/docker/balance-by-account.Dockerfile +++ b/docker/balance-by-account.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/balance-by-account/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/balance-by-account /app/balance-by-account diff --git a/docker/bitnami-postgres.Dockerfile b/docker/bitnami-postgres.Dockerfile index 336b39cb1..d383cccc7 100644 --- a/docker/bitnami-postgres.Dockerfile +++ b/docker/bitnami-postgres.Dockerfile @@ -1,4 +1,4 @@ -FROM bitnamilegacy/postgresql:15.3.0 +FROM public.ecr.aws/bitnami/postgresql:15 USER root diff --git a/docker/client-base.Dockerfile b/docker/client-base.Dockerfile deleted file mode 100644 index f38348b79..000000000 --- a/docker/client-base.Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM node:lts-alpine - -WORKDIR /app - -COPY client/package*.json ./ - -RUN npm install \ No newline at end of file diff --git a/docker/client.Dockerfile b/docker/client.Dockerfile index 1bdf36724..2972fa878 100644 --- a/docker/client.Dockerfile +++ b/docker/client.Dockerfile @@ -1,4 +1,4 @@ -FROM mxfactorial/client-base:v1 AS builder1 +FROM public.ecr.aws/docker/library/node:lts-alpine AS builder ARG PUBLIC_POOL_ID ARG PUBLIC_CLIENT_ID @@ -10,19 +10,18 @@ ARG PORT WORKDIR /app -COPY client . +COPY client/package*.json ./ +RUN npm install -# WARNING: docker build was failing to COPY package*.json with unstaged -# changes on macos so npm install may be required before npm run build: -# RUN npm install +COPY client . RUN npm run build -FROM node:lts-alpine +FROM public.ecr.aws/docker/library/node:lts-alpine COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter -COPY --from=builder1 /app/build /app -COPY --from=builder1 /app/package.json /app -COPY --from=builder1 /app/package-lock.json /app +COPY --from=builder /app/build /app +COPY --from=builder /app/package.json /app +COPY --from=builder /app/package-lock.json /app WORKDIR /app -CMD ["node", "index.js"] \ No newline at end of file +CMD ["node", "index.js"] diff --git a/docker/event.Dockerfile b/docker/event.Dockerfile index 0b018ad86..802c50dd6 100644 --- a/docker/event.Dockerfile +++ b/docker/event.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/event/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine:latest +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/event /app/event diff --git a/docker/go-migrate.Dockerfile b/docker/go-migrate.Dockerfile index 313486761..9b7979121 100644 --- a/docker/go-migrate.Dockerfile +++ b/docker/go-migrate.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:latest AS builder +FROM public.ecr.aws/docker/library/alpine:latest AS builder ARG VERSION=v4.17.0 ARG ARCH=linux-amd64 diff --git a/docker/graphql.Dockerfile b/docker/graphql.Dockerfile index 3fd22d6e6..2a0d28ea6 100644 --- a/docker/graphql.Dockerfile +++ b/docker/graphql.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/graphql/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/graphql /app/graphql diff --git a/docker/measure.Dockerfile b/docker/measure.Dockerfile index a3c57366c..0fa0dd5d8 100644 --- a/docker/measure.Dockerfile +++ b/docker/measure.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/measure/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine:latest +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/measure /app/measure diff --git a/docker/measure.yaml b/docker/measure.yaml new file mode 100644 index 000000000..047ff9d9e --- /dev/null +++ b/docker/measure.yaml @@ -0,0 +1,48 @@ +name: mxf +services: + event: + image: event:latest + build: + context: ../ + dockerfile: ./docker/event.Dockerfile + environment: + PGDATABASE: mxfactorial + PGUSER: test + PGPASSWORD: test + PGHOST: postgres + PGPORT: 5432 + REDIS_DB: 0 + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_USERNAME: default + REDIS_PASSWORD: test + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + measure: + image: measure:latest + build: + context: ../ + dockerfile: ./docker/measure.Dockerfile + ports: + - "10010:10010" + environment: + PGDATABASE: mxfactorial + PGUSER: test + PGPASSWORD: test + PGHOST: postgres + PGPORT: 5432 + REDIS_DB: 0 + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_USERNAME: default + REDIS_PASSWORD: test + MEASURE_PORT: 10010 + READINESS_CHECK_PATH: "/healthz" + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy diff --git a/docker/request-approve.Dockerfile b/docker/request-approve.Dockerfile index 1db7c5219..b93b59ebc 100644 --- a/docker/request-approve.Dockerfile +++ b/docker/request-approve.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/request-approve/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/request-approve /app/request-approve diff --git a/docker/request-by-id.Dockerfile b/docker/request-by-id.Dockerfile index f169dff19..09374675c 100644 --- a/docker/request-by-id.Dockerfile +++ b/docker/request-by-id.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/request-by-id/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/request-by-id /app/request-by-id diff --git a/docker/request-create.Dockerfile b/docker/request-create.Dockerfile index bc9d1a1d9..837e9f4ff 100644 --- a/docker/request-create.Dockerfile +++ b/docker/request-create.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/request-create/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/request-create /app/request-create diff --git a/docker/requests-by-account.Dockerfile b/docker/requests-by-account.Dockerfile index 82175bbe2..6fbc153f8 100644 --- a/docker/requests-by-account.Dockerfile +++ b/docker/requests-by-account.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/requests-by-account/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/requests-by-account /app/requests-by-account diff --git a/docker/rule.Dockerfile b/docker/rule.Dockerfile index b549aa43b..d739642ed 100644 --- a/docker/rule.Dockerfile +++ b/docker/rule.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/rule/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/rule /app/rule diff --git a/docker/services.yaml b/docker/services.yaml index 259c530e8..34f420b35 100644 --- a/docker/services.yaml +++ b/docker/services.yaml @@ -19,6 +19,8 @@ services: READINESS_CHECK_PATH: "/healthz" RUST_LOG: info GRAPHQL_PORT: 10000 + GRAPHQL_RESOURCE: query + GRAPHQL_WS_RESOURCE: ws balance-by-account: image: balance-by-account:latest build: @@ -196,53 +198,8 @@ services: TRANSACTIONS_BY_ACCOUNT_PORT: 10008 depends_on: - postgres - event: - image: event:latest - build: - context: ../ - dockerfile: ./docker/event.Dockerfile - environment: - PGDATABASE: mxfactorial - PGUSER: test - PGPASSWORD: test - PGHOST: postgres - PGPORT: 5432 - REDIS_DB: 0 - REDIS_HOST: redis - REDIS_PORT: 6379 - REDIS_USERNAME: default - REDIS_PASSWORD: test - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - measure: - image: measure:latest - build: - context: ../ - dockerfile: ./docker/measure.Dockerfile - ports: - - "10010:10010" - environment: - PGDATABASE: mxfactorial - PGUSER: test - PGPASSWORD: test - PGHOST: postgres - PGPORT: 5432 - REDIS_DB: 0 - REDIS_HOST: redis - REDIS_PORT: 6379 - REDIS_USERNAME: default - REDIS_PASSWORD: test - MEASURE_PORT: 10010 - READINESS_CHECK_PATH: "/healthz" - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy client: + image: client:latest build: context: ../ dockerfile: ./docker/client.Dockerfile diff --git a/docker/storage.yaml b/docker/storage.yaml index e25bae7cd..c26ed804c 100644 --- a/docker/storage.yaml +++ b/docker/storage.yaml @@ -13,7 +13,7 @@ services: retries: 5 start_period: 10s redis: - image: bitnami/redis:latest + image: public.ecr.aws/bitnami/redis:latest ports: - "6379:6379" healthcheck: diff --git a/docker/transaction-by-id.Dockerfile b/docker/transaction-by-id.Dockerfile index d40ada425..342c90e6f 100644 --- a/docker/transaction-by-id.Dockerfile +++ b/docker/transaction-by-id.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/transaction-by-id/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/transaction-by-id /app/transaction-by-id diff --git a/docker/transactions-by-account.Dockerfile b/docker/transactions-by-account.Dockerfile index ab3f0906b..35239e87b 100644 --- a/docker/transactions-by-account.Dockerfile +++ b/docker/transactions-by-account.Dockerfile @@ -1,4 +1,6 @@ -FROM rust:latest AS builder +FROM public.ecr.aws/docker/library/rust:latest AS builder + +ARG RUN_TESTS=true WORKDIR /app @@ -8,12 +10,17 @@ RUN rustup target add x86_64-unknown-linux-musl RUN apt update && \ apt install -y musl-tools perl make +RUN if [ "$RUN_TESTS" = "true" ]; then rustup component add rustfmt clippy; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo fmt --check; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo clippy -- -D warnings; fi +RUN if [ "$RUN_TESTS" = "true" ]; then cargo test; fi + RUN USER=root cargo build \ --manifest-path=services/transactions-by-account/Cargo.toml \ --target x86_64-unknown-linux-musl \ --release -FROM alpine +FROM public.ecr.aws/docker/library/alpine:latest COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.9.0 /lambda-adapter /opt/extensions/lambda-adapter COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/transactions-by-account /app/transactions-by-account diff --git a/infra/terraform/aws/environments/dev/main.tf b/infra/terraform/aws/environments/dev/main.tf index b4a1249ae..717a1c0c5 100644 --- a/infra/terraform/aws/environments/dev/main.tf +++ b/infra/terraform/aws/environments/dev/main.tf @@ -7,7 +7,7 @@ locals { ARTIFACTS_PREFIX = local.STORAGE_ENV_VAR.ARTIFACTS_BUCKET_PREFIX.default TFSTATE_PREFIX = local.STORAGE_ENV_VAR.TFSTATE_BUCKET_PREFIX.default INFRA_ENV_VAR = local.PROJECT_CONF.infra.terraform.aws.modules.environment.env_var.set - RDS_PREFIX = local.INFRA_ENV_VAR.RDS_PREFIX.default + NAME_PREFIX = local.INFRA_ENV_VAR.NAME_PREFIX.default REGION = local.INFRA_ENV_VAR.REGION.default ENV_ID = module.env_id.ENV_ID ID_ENV = "${local.ENV_ID}-${local.ENV}" @@ -57,7 +57,7 @@ module "dev" { rds_instance_class = "db.t3.micro" rds_parameter_group = "default.postgres14" rds_engine_version = "14.17" - rds_instance_name = "${local.RDS_PREFIX}-${local.ID_ENV}" + rds_instance_name = "${local.NAME_PREFIX}-${local.ID_ENV}" db_snapshot_id = null ############### api gateway ############### diff --git a/infra/terraform/aws/environments/init-dev/main.tf b/infra/terraform/aws/environments/init-dev/main.tf index c1508415e..50a040b9d 100644 --- a/infra/terraform/aws/environments/init-dev/main.tf +++ b/infra/terraform/aws/environments/init-dev/main.tf @@ -28,4 +28,5 @@ module "project_storage_dev" { artifacts_bucket_name_prefix = local.STORAGE_ENV_VAR.ARTIFACTS_BUCKET_PREFIX.default tfstate_bucket_name_prefix = local.STORAGE_ENV_VAR.TFSTATE_BUCKET_PREFIX.default max_image_storage_count = 10 + codebuild_compute_type = "BUILD_GENERAL1_MEDIUM" # SMALL, MEDIUM, LARGE, 2XLARGE } diff --git a/infra/terraform/aws/environments/prod/main.tf b/infra/terraform/aws/environments/prod/main.tf index 393d373a2..b7be562c9 100644 --- a/infra/terraform/aws/environments/prod/main.tf +++ b/infra/terraform/aws/environments/prod/main.tf @@ -15,7 +15,7 @@ locals { ARTIFACTS_PREFIX = local.STORAGE_ENV_VAR.ARTIFACTS_BUCKET_PREFIX.default TFSTATE_PREFIX = local.STORAGE_ENV_VAR.TFSTATE_BUCKET_PREFIX.default INFRA_ENV_VAR = local.PROJECT_CONF.infra.terraform.aws.modules.environment.env_var.set - RDS_PREFIX = local.INFRA_ENV_VAR.RDS_PREFIX.default + NAME_PREFIX = local.INFRA_ENV_VAR.NAME_PREFIX.default REGION = local.INFRA_ENV_VAR.REGION.default ENV_ID = local.PROJECT_CONF.env_var.set.PROD_ENV_ID.default ID_ENV = "${local.ENV_ID}-${local.ENV}" @@ -67,7 +67,7 @@ module "prod" { rds_instance_class = "db.t3.micro" rds_parameter_group = "default.postgres13" rds_engine_version = "13.20" - rds_instance_name = "${local.RDS_PREFIX}-${local.ID_ENV}" + rds_instance_name = "${local.NAME_PREFIX}-${local.ID_ENV}" db_snapshot_id = null ############### api gateway ############### diff --git a/infra/terraform/aws/modules/codebuild/v001/codebuild.tf b/infra/terraform/aws/modules/codebuild/v001/codebuild.tf new file mode 100644 index 000000000..00075e5d3 --- /dev/null +++ b/infra/terraform/aws/modules/codebuild/v001/codebuild.tf @@ -0,0 +1,125 @@ +resource "aws_codebuild_project" "default" { + name = var.project_name + service_role = aws_iam_role.codebuild.arn + + artifacts { + type = "CODEPIPELINE" + } + + environment { + compute_type = var.compute_type + image = "aws/codebuild/standard:7.0" + type = "LINUX_CONTAINER" + privileged_mode = true + + environment_variable { + name = "SERVICE_NAME" + value = var.service_name + } + environment_variable { + name = "ENV_ID" + value = var.env_id + } + environment_variable { + name = "ENV" + value = var.env + } + environment_variable { + name = "AWS_ACCOUNT_ID" + value = var.aws_account_id + } + environment_variable { + name = "REGION" + value = var.aws_region + } + environment_variable { + name = "RUN_TESTS" + value = "true" + } + environment_variable { + name = "PUSH_IMAGE" + value = "true" + } + environment_variable { + name = "DEPLOY" + value = "false" + } + } + + source { + type = "CODEPIPELINE" + buildspec = var.buildspec + } +} + +resource "aws_iam_role" "codebuild" { + name = var.project_name + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "codebuild.amazonaws.com" + } + } + ] + }) +} + +resource "aws_iam_role_policy" "codebuild" { + name = var.project_name + role = aws_iam_role.codebuild.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = "*" + }, + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion" + ] + Resource = "${var.artifacts_bucket_arn}/*" + }, + { + Effect = "Allow" + Action = [ + "ecr:GetAuthorizationToken" + ] + Resource = "*" + }, + { + Effect = "Allow" + Action = [ + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:PutImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload" + ] + Resource = var.ecr_repository_arn + }, + { + Effect = "Allow" + Action = [ + "lambda:UpdateFunctionCode" + ] + Resource = var.lambda_function_arn + } + ] + }) +} diff --git a/infra/terraform/aws/modules/codebuild/v001/outputs.tf b/infra/terraform/aws/modules/codebuild/v001/outputs.tf new file mode 100644 index 000000000..3efe80117 --- /dev/null +++ b/infra/terraform/aws/modules/codebuild/v001/outputs.tf @@ -0,0 +1,7 @@ +output "project_name" { + value = aws_codebuild_project.default.name +} + +output "project_arn" { + value = aws_codebuild_project.default.arn +} diff --git a/infra/terraform/aws/modules/codebuild/v001/variables.tf b/infra/terraform/aws/modules/codebuild/v001/variables.tf new file mode 100644 index 000000000..e10579a69 --- /dev/null +++ b/infra/terraform/aws/modules/codebuild/v001/variables.tf @@ -0,0 +1,11 @@ +variable "project_name" {} +variable "service_name" {} +variable "env" {} +variable "env_id" {} +variable "aws_region" {} +variable "aws_account_id" {} +variable "compute_type" {} +variable "buildspec" {} +variable "artifacts_bucket_arn" {} +variable "ecr_repository_arn" {} +variable "lambda_function_arn" {} diff --git a/infra/terraform/aws/modules/codepipeline/v001/codepipeline.tf b/infra/terraform/aws/modules/codepipeline/v001/codepipeline.tf new file mode 100644 index 000000000..e1906e72b --- /dev/null +++ b/infra/terraform/aws/modules/codepipeline/v001/codepipeline.tf @@ -0,0 +1,158 @@ +locals { + ID_ENV = "${var.env_id}-${var.env}" + PROJECT_CONF_FILE_NAME = "project.yaml" + PROJECT_CONF = yamldecode(file("../../../../../${local.PROJECT_CONF_FILE_NAME}")) + CODEPIPELINE_ENV_VAR = local.PROJECT_CONF.infra.terraform.aws.modules.codepipeline.env_var.set + SERVICES_ZIP = local.PROJECT_CONF.scripts.env_var.set.SERVICES_ZIP.default + BUILD_OBJECT_KEY_PATH = local.CODEPIPELINE_ENV_VAR.BUILD_OBJECT_KEY_PATH.default + BUILD_SOURCE_LOCATION = "${local.BUILD_OBJECT_KEY_PATH}/${local.SERVICES_ZIP}" +} + +resource "aws_codepipeline" "build" { + name = "mxfactorial-build-${local.ID_ENV}" + role_arn = aws_iam_role.codepipeline.arn + pipeline_type = "V2" + + artifact_store { + location = var.artifacts_bucket_name + type = "S3" + } + + stage { + name = "Source" + action { + name = "S3Source" + category = "Source" + owner = "AWS" + provider = "S3" + version = "1" + output_artifacts = ["source_output"] + configuration = { + S3Bucket = var.artifacts_bucket_name + S3ObjectKey = local.BUILD_SOURCE_LOCATION + PollForSourceChanges = false + } + } + } + + stage { + name = "Build" + + dynamic "action" { + for_each = var.codebuild_project_names + content { + name = action.key + category = "Build" + owner = "AWS" + provider = "CodeBuild" + input_artifacts = ["source_output"] + version = "1" + run_order = 1 + configuration = { + ProjectName = action.value + } + } + } + } +} + +resource "aws_cloudwatch_event_rule" "s3_trigger" { + name = "mxfactorial-build-trigger-${local.ID_ENV}" + + event_pattern = jsonencode({ + source = ["aws.s3"] + detail-type = ["Object Created"] + detail = { + bucket = { name = [var.artifacts_bucket_name] } + object = { key = [{ prefix = local.BUILD_SOURCE_LOCATION }] } + } + }) +} + +resource "aws_cloudwatch_event_target" "codepipeline" { + rule = aws_cloudwatch_event_rule.s3_trigger.name + arn = aws_codepipeline.build.arn + role_arn = aws_iam_role.eventbridge_pipeline.arn +} + +resource "aws_iam_role" "codepipeline" { + name = "codepipeline-build-${local.ID_ENV}" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "codepipeline.amazonaws.com" + } + } + ] + }) +} + +resource "aws_iam_role_policy" "codepipeline" { + name = "codepipeline-build-${local.ID_ENV}" + role = aws_iam_role.codepipeline.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:GetBucketVersioning", + "s3:PutObject" + ] + Resource = [ + var.artifacts_bucket_arn, + "${var.artifacts_bucket_arn}/*" + ] + }, + { + Effect = "Allow" + Action = [ + "codebuild:BatchGetBuilds", + "codebuild:StartBuild" + ] + Resource = "*" + } + ] + }) +} + +resource "aws_iam_role" "eventbridge_pipeline" { + name = "eventbridge-pipeline-${local.ID_ENV}" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "events.amazonaws.com" + } + } + ] + }) +} + +resource "aws_iam_role_policy" "eventbridge_pipeline" { + name = "eventbridge-pipeline-${local.ID_ENV}" + role = aws_iam_role.eventbridge_pipeline.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = "codepipeline:StartPipelineExecution" + Resource = aws_codepipeline.build.arn + } + ] + }) +} diff --git a/infra/terraform/aws/modules/codepipeline/v001/variables.tf b/infra/terraform/aws/modules/codepipeline/v001/variables.tf new file mode 100644 index 000000000..a0831601e --- /dev/null +++ b/infra/terraform/aws/modules/codepipeline/v001/variables.tf @@ -0,0 +1,7 @@ +variable "env" {} +variable "env_id" {} +variable "artifacts_bucket_name" {} +variable "artifacts_bucket_arn" {} +variable "codebuild_project_names" { + type = map(string) +} diff --git a/infra/terraform/aws/modules/ecr/v001/buildspecs/build.yaml b/infra/terraform/aws/modules/ecr/v001/buildspecs/build.yaml new file mode 100644 index 000000000..921613e0b --- /dev/null +++ b/infra/terraform/aws/modules/ecr/v001/buildspecs/build.yaml @@ -0,0 +1,32 @@ +version: 0.2 + +phases: + pre_build: + commands: + - aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com + - export SHORT_SHA=$(echo ${CODEBUILD_RESOLVED_SOURCE_VERSION:-$CODEBUILD_BUILD_ID} | sed 's/.*://' | cut -c1-7) + - export ECR_URI=$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$ENV_ID/$ENV/$SERVICE_NAME + build: + commands: + - echo "building $SERVICE_NAME (RUN_TESTS=$RUN_TESTS, PUSH_IMAGE=$PUSH_IMAGE)..." + - docker build --build-arg RUN_TESTS=$RUN_TESTS -t $SERVICE_NAME:$SHORT_SHA -f docker/$SERVICE_NAME.Dockerfile . + - docker tag $SERVICE_NAME:$SHORT_SHA $ECR_URI:$SHORT_SHA + - docker tag $SERVICE_NAME:$SHORT_SHA $ECR_URI:latest + post_build: + commands: + - | + if [ "$PUSH_IMAGE" = "true" ]; then + echo "pushing $SERVICE_NAME..." + docker push $ECR_URI:$SHORT_SHA + docker push $ECR_URI:latest + else + echo "skipping push (PUSH_IMAGE=$PUSH_IMAGE)" + fi + - | + if [ "$DEPLOY" = "true" ]; then + echo "deploying $SERVICE_NAME to lambda..." + aws lambda update-function-code \ + --function-name $SERVICE_NAME-$ENV_ID-$ENV \ + --image-uri $ECR_URI:latest \ + --region $REGION + fi diff --git a/infra/terraform/aws/modules/ecr/v001/codebuild.tf b/infra/terraform/aws/modules/ecr/v001/codebuild.tf new file mode 100644 index 000000000..7dd30cc70 --- /dev/null +++ b/infra/terraform/aws/modules/ecr/v001/codebuild.tf @@ -0,0 +1,19 @@ +locals { + ID_ENV = "${var.env_id}-${var.env}" +} + +module "codebuild" { + source = "../../codebuild/v001" + + project_name = "mxfactorial-${var.service_name}-${local.ID_ENV}" + service_name = var.service_name + env = var.env + env_id = var.env_id + aws_region = var.aws_region + aws_account_id = var.aws_account_id + compute_type = var.codebuild_compute_type + buildspec = file("${path.module}/buildspecs/build.yaml") + artifacts_bucket_arn = var.artifacts_bucket_arn + ecr_repository_arn = aws_ecr_repository.default.arn + lambda_function_arn = "arn:aws:lambda:${var.aws_region}:${var.aws_account_id}:function:${var.service_name}-${local.ID_ENV}" +} diff --git a/infra/terraform/aws/modules/ecr/v001/outputs.tf b/infra/terraform/aws/modules/ecr/v001/outputs.tf new file mode 100644 index 000000000..ae672401f --- /dev/null +++ b/infra/terraform/aws/modules/ecr/v001/outputs.tf @@ -0,0 +1,11 @@ +output "ecr_repository_url" { + value = aws_ecr_repository.default.repository_url +} + +output "codebuild_project_name" { + value = module.codebuild.project_name +} + +output "codebuild_project_arn" { + value = module.codebuild.project_arn +} diff --git a/infra/terraform/aws/modules/ecr/v001/variables.tf b/infra/terraform/aws/modules/ecr/v001/variables.tf index b841b14be..210fd1e81 100644 --- a/infra/terraform/aws/modules/ecr/v001/variables.tf +++ b/infra/terraform/aws/modules/ecr/v001/variables.tf @@ -1,8 +1,12 @@ variable "env" {} variable "env_id" {} variable "force_delete" { - type = bool + type = bool default = false } variable "service_name" {} -variable "max_image_storage_count" {} \ No newline at end of file +variable "max_image_storage_count" {} +variable "artifacts_bucket_arn" {} +variable "aws_region" {} +variable "aws_account_id" {} +variable "codebuild_compute_type" {} \ No newline at end of file diff --git a/infra/terraform/aws/modules/project-storage/v001/README.md b/infra/terraform/aws/modules/project-storage/v001/README.md new file mode 100644 index 000000000..3b58b67b2 --- /dev/null +++ b/infra/terraform/aws/modules/project-storage/v001/README.md @@ -0,0 +1,131 @@ +

+ systemaccounting +

+ +### project-storage + +resources provisioned in init-dev for dev environment + +#### ENV_ID + +assigned in `.env` at project root + +#### naming conventions + +- `infra/terraform/aws/modules/project-storage/v001/s3.tf` +- `infra/terraform/aws/modules/project-storage/v001/codepipeline.tf` +- `infra/terraform/aws/modules/project-storage/v001/integ.tf` +- `infra/terraform/aws/modules/ecr/v001/ecr.tf` +- `infra/terraform/aws/modules/ecr/v001/codebuild.tf` +- `infra/terraform/aws/modules/codebuild/v001/codebuild.tf` +- `infra/terraform/aws/modules/codepipeline/v001/codepipeline.tf` + +#### manual deletion + +when `terraform destroy` fails, delete in order: + +1. codepipeline +1. eventbridge rule + target +1. codebuild projects (integ + 13 per-service) +1. ecr repos +1. s3 buckets (artifacts, tfstate) +1. iam roles + policies + +#### state list + +``` +module.project_storage_dev.aws_s3_bucket.artifacts +module.project_storage_dev.aws_s3_bucket.tfstate +module.project_storage_dev.aws_s3_bucket_notification.artifacts_eventbridge +module.project_storage_dev.aws_s3_bucket_versioning.artifacts + +module.project_storage_dev.aws_codebuild_project.integ +module.project_storage_dev.aws_iam_role.integ +module.project_storage_dev.aws_iam_role_policy.integ + +module.project_storage_dev.module.codepipeline.aws_codepipeline.build +module.project_storage_dev.module.codepipeline.aws_cloudwatch_event_rule.s3_trigger +module.project_storage_dev.module.codepipeline.aws_cloudwatch_event_target.codepipeline +module.project_storage_dev.module.codepipeline.aws_iam_role.codepipeline +module.project_storage_dev.module.codepipeline.aws_iam_role.eventbridge_pipeline +module.project_storage_dev.module.codepipeline.aws_iam_role_policy.codepipeline +module.project_storage_dev.module.codepipeline.aws_iam_role_policy.eventbridge_pipeline + +module.project_storage_dev.module.ecr_repos["auto-confirm"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["auto-confirm"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["auto-confirm"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["auto-confirm"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["auto-confirm"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["balance-by-account"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["balance-by-account"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["balance-by-account"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["balance-by-account"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["balance-by-account"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["client"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["client"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["client"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["client"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["client"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["go-migrate"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["go-migrate"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["go-migrate"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["go-migrate"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["go-migrate"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["graphql"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["graphql"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["graphql"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["graphql"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["graphql"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["request-approve"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["request-approve"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["request-approve"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["request-approve"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["request-approve"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["request-by-id"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["request-by-id"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["request-by-id"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["request-by-id"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["request-by-id"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["request-create"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["request-create"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["request-create"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["request-create"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["request-create"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["requests-by-account"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["requests-by-account"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["requests-by-account"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["requests-by-account"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["requests-by-account"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["rule"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["rule"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["rule"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["rule"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["rule"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["transaction-by-id"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["transaction-by-id"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["transaction-by-id"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["transaction-by-id"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["transaction-by-id"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["transactions-by-account"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["transactions-by-account"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["transactions-by-account"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["transactions-by-account"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["transactions-by-account"].module.codebuild.aws_iam_role_policy.codebuild + +module.project_storage_dev.module.ecr_repos["warm-cache"].aws_ecr_repository.default +module.project_storage_dev.module.ecr_repos["warm-cache"].aws_ecr_lifecycle_policy.default +module.project_storage_dev.module.ecr_repos["warm-cache"].module.codebuild.aws_codebuild_project.default +module.project_storage_dev.module.ecr_repos["warm-cache"].module.codebuild.aws_iam_role.codebuild +module.project_storage_dev.module.ecr_repos["warm-cache"].module.codebuild.aws_iam_role_policy.codebuild +``` diff --git a/infra/terraform/aws/modules/project-storage/v001/codepipeline.tf b/infra/terraform/aws/modules/project-storage/v001/codepipeline.tf new file mode 100644 index 000000000..a7c7232c1 --- /dev/null +++ b/infra/terraform/aws/modules/project-storage/v001/codepipeline.tf @@ -0,0 +1,8 @@ +module "codepipeline" { + source = "../../codepipeline/v001" + env = var.env + env_id = var.env_id + artifacts_bucket_name = aws_s3_bucket.artifacts.bucket + artifacts_bucket_arn = aws_s3_bucket.artifacts.arn + codebuild_project_names = { for k, v in module.ecr_repos : k => v.codebuild_project_name } +} diff --git a/infra/terraform/aws/modules/project-storage/v001/ecr.tf b/infra/terraform/aws/modules/project-storage/v001/ecr.tf index 5b2967e0b..978918691 100644 --- a/infra/terraform/aws/modules/project-storage/v001/ecr.tf +++ b/infra/terraform/aws/modules/project-storage/v001/ecr.tf @@ -1,3 +1,6 @@ +data "aws_caller_identity" "current" {} +data "aws_region" "current" {} + module "ecr_repos" { for_each = local.ECR_REPOS source = "../../ecr/v001" @@ -6,4 +9,8 @@ module "ecr_repos" { env_id = var.env_id service_name = each.value force_delete = var.force_destroy_storage + artifacts_bucket_arn = aws_s3_bucket.artifacts.arn + aws_region = data.aws_region.current.id + aws_account_id = data.aws_caller_identity.current.account_id + codebuild_compute_type = var.codebuild_compute_type } diff --git a/infra/terraform/aws/modules/project-storage/v001/integ.tf b/infra/terraform/aws/modules/project-storage/v001/integ.tf new file mode 100644 index 000000000..aad65b11f --- /dev/null +++ b/infra/terraform/aws/modules/project-storage/v001/integ.tf @@ -0,0 +1,153 @@ +# integration test codebuild project +# runs full test suite with all services via docker compose + +resource "aws_codebuild_project" "integ" { + name = "mxfactorial-integ-${local.ID_ENV}" + service_role = aws_iam_role.integ.arn + + artifacts { + type = "NO_ARTIFACTS" + } + + environment { + compute_type = "BUILD_GENERAL1_LARGE" + image = "aws/codebuild/standard:7.0" + type = "LINUX_CONTAINER" + privileged_mode = true + + environment_variable { + name = "ENV_ID" + value = var.env_id + } + environment_variable { + name = "ENV" + value = var.env + } + environment_variable { + name = "AWS_ACCOUNT_ID" + value = data.aws_caller_identity.current.account_id + } + environment_variable { + name = "REGION" + value = data.aws_region.current.id + } + } + + source { + type = "S3" + location = "${aws_s3_bucket.artifacts.bucket}/${local.INTEG_SOURCE_LOCATION}" + buildspec = <<-EOF + version: 0.2 + phases: + install: + # codebuild standard:7.0 has node 18, but aws-sdk requires node >= 20 + runtime-versions: + nodejs: 20 + pre_build: + commands: + # install test dependencies + - apt-get update && apt-get install -y postgresql-client + - curl -L https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq + - | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + . $HOME/.cargo/env + rustup component add rustfmt clippy + - npx playwright install-deps + # pull pre-built service images from ecr and tag as latest for docker compose + - aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com + - export ECR_URI=$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$ENV_ID/$ENV + - | + for SVC in ${join(" ", local.ECR_REPOS)}; do + echo "pulling $SVC..." + TAG=$(aws ecr describe-images --repository-name $ENV_ID/$ENV/$SVC --region $REGION --query 'sort_by(imageDetails,&imagePushedAt)[-1].imageTags[0]' --output text 2>/dev/null || echo "") + if [ -n "$TAG" ] && [ "$TAG" != "None" ]; then + docker pull $ECR_URI/$SVC:$TAG + docker tag $ECR_URI/$SVC:$TAG $SVC:latest + else + echo "skipping $SVC (no images)" + fi + done + build: + commands: + # build storage (postgres), then start all services with pre-built images + - docker compose -f docker/storage.yaml build + - docker compose -f docker/storage.yaml -f docker/services.yaml up -d --no-build + - until docker exec mxf-postgres-1 pg_isready -U postgres; do sleep 1; done + # run tests + - make --no-print-directory -C crates/pg test-db + - make --no-print-directory -C crates/redisclient test-cache + - make --no-print-directory -C tests test-local + - make --no-print-directory -C client test + post_build: + commands: + - docker compose -f docker/storage.yaml -f docker/services.yaml down + EOF + } +} + +resource "aws_iam_role" "integ" { + name = "mxfactorial-integ-${local.ID_ENV}" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "codebuild.amazonaws.com" + } + } + ] + }) +} + +resource "aws_iam_role_policy" "integ" { + name = "mxfactorial-integ-${local.ID_ENV}" + role = aws_iam_role.integ.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = "*" + }, + { + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion" + ] + Resource = "${aws_s3_bucket.artifacts.arn}/*" + }, + { + Effect = "Allow" + Action = ["s3:ListBucket"] + Resource = aws_s3_bucket.artifacts.arn + }, + { + Effect = "Allow" + Action = [ + "ecr:GetAuthorizationToken" + ] + Resource = "*" + }, + { + Effect = "Allow" + Action = [ + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:DescribeImages" + ] + Resource = "arn:aws:ecr:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:repository/${var.env_id}/${var.env}/*" + } + ] + }) +} diff --git a/infra/terraform/aws/modules/project-storage/v001/locals.tf b/infra/terraform/aws/modules/project-storage/v001/locals.tf index 331d4efc2..2c7beeb3f 100644 --- a/infra/terraform/aws/modules/project-storage/v001/locals.tf +++ b/infra/terraform/aws/modules/project-storage/v001/locals.tf @@ -1,9 +1,12 @@ locals { - ID_ENV = "${var.env_id}-${var.env}" - PROJECT_CONF_FILE_NAME = "project.yaml" - PROJECT_CONF = yamldecode(file("../../../../../${local.PROJECT_CONF_FILE_NAME}")) - STORAGE_ENV_VAR = local.PROJECT_CONF.infra.terraform.aws.modules.project-storage.env_var.set - ID_ENV_PREFIX = "${var.env_id}/${var.env}" + ID_ENV = "${var.env_id}-${var.env}" + PROJECT_CONF_FILE_NAME = "project.yaml" + PROJECT_CONF = yamldecode(file("../../../../../${local.PROJECT_CONF_FILE_NAME}")) + STORAGE_ENV_VAR = local.PROJECT_CONF.infra.terraform.aws.modules.project-storage.env_var.set + ID_ENV_PREFIX = "${var.env_id}/${var.env}" + SERVICES_ZIP = local.PROJECT_CONF.scripts.env_var.set.SERVICES_ZIP.default + INTEG_TEST_OBJECT_KEY_PATH = local.STORAGE_ENV_VAR.INTEG_TEST_OBJECT_KEY_PATH.default + INTEG_SOURCE_LOCATION = "${local.INTEG_TEST_OBJECT_KEY_PATH}/${local.SERVICES_ZIP}" // add a terraform_data precondition to fail // if a service is not found in project.yaml diff --git a/infra/terraform/aws/modules/project-storage/v001/outputs.tf b/infra/terraform/aws/modules/project-storage/v001/outputs.tf new file mode 100644 index 000000000..9bed9d3b3 --- /dev/null +++ b/infra/terraform/aws/modules/project-storage/v001/outputs.tf @@ -0,0 +1,11 @@ +output "artifacts_bucket_name" { + value = aws_s3_bucket.artifacts.bucket +} + +output "artifacts_bucket_arn" { + value = aws_s3_bucket.artifacts.arn +} + +output "codebuild_project_names" { + value = { for k, v in module.ecr_repos : k => v.codebuild_project_name } +} diff --git a/infra/terraform/aws/modules/project-storage/v001/s3.tf b/infra/terraform/aws/modules/project-storage/v001/s3.tf index b61e47794..7978bab23 100644 --- a/infra/terraform/aws/modules/project-storage/v001/s3.tf +++ b/infra/terraform/aws/modules/project-storage/v001/s3.tf @@ -3,6 +3,18 @@ resource "aws_s3_bucket" "artifacts" { force_destroy = var.force_destroy_storage } +resource "aws_s3_bucket_versioning" "artifacts" { + bucket = aws_s3_bucket.artifacts.id + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_notification" "artifacts_eventbridge" { + bucket = aws_s3_bucket.artifacts.id + eventbridge = true +} + resource "aws_s3_bucket" "tfstate" { bucket = "${var.tfstate_bucket_name_prefix}-${local.ID_ENV}" force_destroy = var.force_destroy_storage diff --git a/infra/terraform/aws/modules/project-storage/v001/variables.tf b/infra/terraform/aws/modules/project-storage/v001/variables.tf index 286d580db..f9cf99510 100644 --- a/infra/terraform/aws/modules/project-storage/v001/variables.tf +++ b/infra/terraform/aws/modules/project-storage/v001/variables.tf @@ -3,7 +3,8 @@ variable "env_id" {} variable "artifacts_bucket_name_prefix" {} variable "tfstate_bucket_name_prefix" {} variable "force_destroy_storage" { - type = bool - default = false + type = bool + default = false } -variable "max_image_storage_count" {} \ No newline at end of file +variable "max_image_storage_count" {} +variable "codebuild_compute_type" {} \ No newline at end of file diff --git a/infra/terraform/aws/modules/provided-lambda/v001/lambda.tf b/infra/terraform/aws/modules/provided-lambda/v001/lambda.tf index f7f754aa5..b4d480bb0 100644 --- a/infra/terraform/aws/modules/provided-lambda/v001/lambda.tf +++ b/infra/terraform/aws/modules/provided-lambda/v001/lambda.tf @@ -85,7 +85,7 @@ resource "aws_iam_policy" "default" { Action = [ "logs:CreateLogGroup" ], - Resource = "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*" + Resource = "arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:*" }, { Sid = "${local.SERVICE_NAME_TITLE}LogEventPolicy${local.TITLED_ID_ENV}" @@ -95,7 +95,7 @@ resource "aws_iam_policy" "default" { "logs:PutLogEvents" ], Resource = [ - "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:${local.LOG_GROUP_NAME}:*" + "arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:${local.LOG_GROUP_NAME}:*" ] }], }) diff --git a/infra/terraform/aws/modules/region/v001/apigw-logging.tf b/infra/terraform/aws/modules/region/v001/apigw-logging.tf index 65b3ab034..4371df7fb 100644 --- a/infra/terraform/aws/modules/region/v001/apigw-logging.tf +++ b/infra/terraform/aws/modules/region/v001/apigw-logging.tf @@ -1,6 +1,6 @@ locals { - PROJECT_CONF = yamldecode(file("../../../../../project.yaml")) - GITHUB_REPO_NAME = local.PROJECT_CONF[".github"].env_var.set.GITHUB_REPO_NAME.default + PROJECT_CONF = yamldecode(file("../../../../../project.yaml")) + NAME_PREFIX = local.PROJECT_CONF.infra.terraform.aws.modules.environment.env_var.set.NAME_PREFIX.default } resource "aws_api_gateway_account" "apigw_logging" { @@ -13,7 +13,7 @@ resource "aws_iam_role" "apigw_logging" { Version = "2012-10-17" Statement = [ { - Sid = "${title(local.GITHUB_REPO_NAME)}APIGwTrustPolicy" + Sid = "${title(local.NAME_PREFIX)}APIGwTrustPolicy" Action = "sts:AssumeRole" Effect = "Allow" Principal = { @@ -25,13 +25,13 @@ resource "aws_iam_role" "apigw_logging" { } resource "aws_iam_policy" "apigw_logging" { - name = "${local.GITHUB_REPO_NAME}-apigw-logging" - description = "${local.GITHUB_REPO_NAME} apigw logging permission" + name = "${local.NAME_PREFIX}-apigw-logging" + description = "${local.NAME_PREFIX} apigw logging permission" policy = jsonencode({ Version = "2012-10-17" Statement = [ { - Sid = "${title(local.GITHUB_REPO_NAME)}LoggingPolicy" + Sid = "${title(local.NAME_PREFIX)}LoggingPolicy" Action = [ "logs:CreateLogGroup", "logs:CreateLogStream", diff --git a/make/docker.mk b/make/docker.mk index bae475f19..b1f364f18 100644 --- a/make/docker.mk +++ b/make/docker.mk @@ -13,6 +13,10 @@ compose-up-build: compose-down: bash scripts/compose.sh --down +compose-add-measure: + @COMPOSE_IGNORE_ORPHANS=true \ + docker compose -f docker/storage.yaml -f docker/measure.yaml up -d + rebuild-db: @$(MAKE) -C migrations rebuild @$(MAKE) -C migrations run diff --git a/make/ecr-lambda.mk b/make/ecr-lambda.mk index 2351dd5ec..85a6e7c43 100644 --- a/make/ecr-lambda.mk +++ b/make/ecr-lambda.mk @@ -2,7 +2,7 @@ BUILD_CTX?=. build-image: @cd $(RELATIVE_PROJECT_ROOT_PATH); \ - bash scripts/build-image.sh --app-name $(APP_NAME) --build-ctx $(BUILD_CTX) + bash scripts/build-image.sh --app-name $(APP_NAME) --build-ctx $(BUILD_CTX) $(if $(NO_TEST),--no-test) tag-dev-image: @cd $(RELATIVE_PROJECT_ROOT_PATH); \ diff --git a/migrations/go-migrate/makefile b/migrations/go-migrate/makefile index 14c712dc1..93e84c0b6 100644 --- a/migrations/go-migrate/makefile +++ b/migrations/go-migrate/makefile @@ -1,3 +1,21 @@ RELATIVE_PROJECT_ROOT_PATH=$(shell REL_PATH="."; while [ $$(ls "$$REL_PATH" | grep project.yaml | wc -l | xargs) -eq 0 ]; do REL_PATH="$$REL_PATH./.."; done; printf '%s' "$$REL_PATH") include $(RELATIVE_PROJECT_ROOT_PATH)/make/shared.mk -include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk \ No newline at end of file +include $(RELATIVE_PROJECT_ROOT_PATH)/make/ecr-lambda.mk + +###################### rds migrate commands ###################### + +resetrds: + @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd reset + +downrds: + @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd down + +testrds: + @$(MAKE) resetrds + @$(MAKE) downrds + +uprds: + @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd up + +droprds: + @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd drop \ No newline at end of file diff --git a/migrations/makefile b/migrations/makefile index b6c601b99..30d934566 100644 --- a/migrations/makefile +++ b/migrations/makefile @@ -64,46 +64,29 @@ create: ###################### local/docker migrate commands ###################### resetdocker: - @docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd reset - -rl: - $(MAKE) resetdocker + @COMPOSE_IGNORE_ORPHANS=true \ + docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd reset 2>&1 | grep -v "No services to build" downdocker: - @docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd down + @COMPOSE_IGNORE_ORPHANS=true \ + docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd down 2>&1 | grep -v "No services to build" testdocker: $(MAKE) resetdocker $(MAKE) downdocker updocker: - @docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd up + @COMPOSE_IGNORE_ORPHANS=true \ + docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd up 2>&1 | grep -v "No services to build" dropdocker: - @docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd drop + @COMPOSE_IGNORE_ORPHANS=true \ + docker compose -f $(RELATIVE_PROJECT_ROOT_PATH)/docker/storage.yaml run --rm go-migrate --db_type test --cmd drop 2>&1 | grep -v "No services to build" insert: @cd $(RELATIVE_PROJECT_ROOT_PATH); \ bash scripts/insert-transactions.sh -###################### rds migrate commands ###################### - -resetrds: - @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd reset - -downrds: - @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd down - -testrds: - @$(MAKE) resetrds - @$(MAKE) downrds - -uprds: - @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd up - -droprds: - @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/go-migrate-rds.sh --env dev --cmd drop - ###################### arg tests ###################### test-dir-arg: diff --git a/project.yaml b/project.yaml index 08d13b52c..28f4a68a0 100644 --- a/project.yaml +++ b/project.yaml @@ -1,12 +1,6 @@ .github: env_var: - set: - GITHUB_ORG: - ssm: null - default: systemaccounting - GITHUB_REPO_NAME: - ssm: null - default: mxfactorial + set: {} get: [] params: [] codecov: @@ -16,16 +10,7 @@ - ui workflows: env_var: - set: - GITHUB_REGISTRY: - ssm: null - default: ghcr.io - IMAGE_BUILDER_WORKFLOW: - ssm: null - default: build-all-images.yaml - DEPLOY_IMAGE_WORKFLOW: - ssm: null - default: deploy-all-images.yaml + set: {} get: [] params: [] client: @@ -146,9 +131,6 @@ docker: DOCKER_USER: ssm: null default: mxfactorial - LOCAL_TAG_VERSION: - ssm: null - default: latest get: [] infra: terraform: @@ -177,6 +159,13 @@ infra: ssm: null default: provided.al2023 get: [] + codepipeline: + env_var: + set: + BUILD_OBJECT_KEY_PATH: + ssm: null + default: build + get: [] environment: env_var: set: @@ -213,7 +202,7 @@ infra: REGION: ssm: null default: us-east-1 - RDS_PREFIX: + NAME_PREFIX: ssm: null default: mxfactorial SSM_VERSION: @@ -246,9 +235,6 @@ infra: BALANCE_BY_ACCOUNT_URL: ssm: service/lambda/balance_by_account/url default: null - RDS_INSTANCE_NAME_PREFIX: - ssm: null - default: mxfactorial READINESS_CHECK_PATH: ssm: service/lambda/readiness_check_path default: /healthz @@ -305,6 +291,9 @@ infra: TFSTATE_BUCKET_PREFIX: ssm: null default: mxfactorial-tfstate + INTEG_TEST_OBJECT_KEY_PATH: + ssm: null + default: integ get: [] k8s: local: diff --git a/scripts/README.md b/scripts/README.md index ddcbcd488..25f574ad0 100755 --- a/scripts/README.md +++ b/scripts/README.md @@ -144,10 +144,6 @@ inits terraform state for dev envs creates then sets custom env id to enable access to dev env from multiple workspaces -##### `delete-dev-storage.sh` - -deletes storage for cloud dev environment with aws cli instead of terraform when state file not found, e.g. cloud dev env storage was created on different machine - ##### `list-lambdas.sh` lists lambdas created by terraform @@ -216,6 +212,10 @@ invokes go-migrate lambda to run migrations on rds send a http request to the internal `migrations/go-migrate` tool +##### `ecr-images.sh` + +triggers codebuild to build, test and push images to ecr. `--build` builds+tests, `--push` pushes to ecr, `--deploy` updates lambdas, `--no-test` skips tests, `--integ` runs integration tests, `--pull` pulls from ecr, `--service ` targets single service + ##### `auth-ecr.sh` authenticate with ecr @@ -248,10 +248,6 @@ prints uri of ecr repo prints service image tag with ecr repo uri and current git sha added as tag version -##### `delete-ecr-repos.sh` - -convenience script to delete all dev ecr repos - ##### `push-dev-image.sh` pushes local docker image to dev ecr repo (assumes local image already tagged) @@ -282,22 +278,9 @@ used in integration test workflow after cloud integration tests pass 1. adds prod tag if current dev image tagged with merge commit, then pushes to prod ecr 1. exits if current dev image NOT tagged with merge commit (prod image not tagged and pushed) -##### `build-image-job.sh` - -used in `.github/workflows/build-all-images.yaml` to copy zipped code from s3, then build, tag and push service images to github container registry - ##### `zip-services.sh` -adds services to zip file. used by `scripts/build-all-images.sh` before triggering `.github/workflows/build-all-images.yaml` - -##### `build-all-images.sh` -zips and pushes current service code to s3, then triggers `.github/workflows/build-all-images.yaml` to avoid building almost a dozen rust images locally - -##### `pull-all-images.sh` -pulls images built and pushed by `.github/workflows/build-all-images.yaml` - -##### `deploy-all-images.sh` -zips and pushes current service code to s3, then triggers `.github/workflows/deploy-all-images.yaml` to build and deploy services to lambda +adds services to zip file for s3 upload ##### `create-env-id.sh` adds a [$RANDOM](https://tldp.org/LDP/abs/html/randomvar.html) `ENV_ID` variable the `.env` file in project root. the `ENV_ID` variable is used by terraform to provision cloud development environments matched to a developers local machine @@ -332,9 +315,6 @@ gets ssh key from ssm and writes it to disk #### `install.sh` installs project dependencies. only macos supported -#### `delete-dev-images.sh` -deletes all images of single app in dev ecr - #### `test-availability.sh` tests availability of services locally or in cloud diff --git a/scripts/build-all-images.sh b/scripts/build-all-images.sh deleted file mode 100644 index 05d6466b3..000000000 --- a/scripts/build-all-images.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash - -set -e - -if [[ -z $GITHUB_PAT ]]; then - echo "set GITHUB_PAT variable in shell to continue" - exit 1 -fi - -ENV=dev -PROJECT_CONF=project.yaml -ENV_ID=$(source ./scripts/print-env-id.sh) -ID_ENV="$ENV_ID-$ENV" -REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) -ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) -ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" -IMAGE_BUILDER_WORKFLOW=$(yq '.[".github"].workflows.env_var.set.IMAGE_BUILDER_WORKFLOW.default' $PROJECT_CONF) -WORKFLOW_ID=$IMAGE_BUILDER_WORKFLOW -GITHUB_ORG=$(yq '.[".github"].env_var.set.GITHUB_ORG.default' $PROJECT_CONF) -GITHUB_REPO_NAME=$(yq '.[".github"].env_var.set.GITHUB_REPO_NAME.default' $PROJECT_CONF) -SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) - -source scripts/zip-services.sh - -echo '*** uploading archive to s3' -aws s3 cp $SERVICES_ZIP s3://$ARTIFACTS_BUCKET --region $REGION -rm $SERVICES_ZIP - -echo "*** triggering .github/workflows/$WORKFLOW_ID" - -curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_PAT" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/workflows/$WORKFLOW_ID/dispatches \ - -d "{\"ref\":\"develop\"}" - -if [[ $(uname) == "Darwin" ]]; then - # store utc time in iso8601 format minus 10 seconds - CREATED=$(date -u -v-10S "+%Y-%m-%dT%H:%M:%SZ") - # store utc time in unix timestamp format plus 10 minutes - TEN_MIN_MAX=$(date -u -v+10M "+%s") -else - CREATED=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 10 seconds ago" +'%Y-%m-%dT%H:%M:%SZ') - TEN_MIN_MAX=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 10 minutes" +'%s') -fi - -# wait 5 seconds -sleep 5 - -RUN_ID=$(curl -sL \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_PAT" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/workflows/$WORKFLOW_ID/runs?created=>$CREATED" | yq '.workflow_runs[0].id') - -echo "*** waiting for $WORKFLOW_ID github workflow to complete" - -function get_run() { - RUN=$(curl -sL \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_PAT" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/runs/$RUN_ID) - - STATUS=$(echo "$RUN" | yq '.status') - CONCLUSION=$(echo "$RUN" | yq '.conclusion') -} - -STATUS='queued' -CONCLUSION=null - -while [[ $STATUS != 'completed' && $(date +%s) -lt $TEN_MIN_MAX ]]; do - sleep 5 - get_run - printf '%s' '.' -done - -echo "" - -if [[ $CONCLUSION != 'success' ]]; then - echo "build failed, pulling latest from ghcr anyway" -fi - -source scripts/pull-all-images.sh \ No newline at end of file diff --git a/scripts/build-dev-env.sh b/scripts/build-dev-env.sh index b690893d4..8b9db6ca4 100644 --- a/scripts/build-dev-env.sh +++ b/scripts/build-dev-env.sh @@ -40,6 +40,9 @@ terraform init printf "\n${YELLOW}*** provisioning artifact, cache origin and terraform state storage${NOCOLOR}\n\n" terraform apply +printf "\n${YELLOW}*** waiting for IAM roles to propagate${NOCOLOR}\n\n" +sleep 20 + popd function apply_agigw_logging_perm() { @@ -79,8 +82,8 @@ else apply_agigw_logging_perm fi -printf "\n${YELLOW}*** compiling app binaries and pushing to artifact bucket${NOCOLOR}\n\n" -make --no-print-directory all CMD=initial-deploy ENV=dev +printf "\n${YELLOW}*** building and pushing images to ECR${NOCOLOR}\n\n" +bash scripts/ecr-images.sh --build --push source ./scripts/terraform-init-dev.sh \ --key "$TFSTATE_ENV" \ @@ -116,12 +119,10 @@ if [[ $(yq '.scripts.env_var.set.BUILD_DB.default' "../../../../../$PROJECT_CONF exit 0 fi -pushd ../../../../../migrations +popd printf "\n${YELLOW}*** deploying migrations to rds in $ID_ENV${NOCOLOR}\n\n" -make --no-print-directory uprds DB=test ENV=dev - -popd; popd; +make --no-print-directory -C migrations/go-migrate uprds DB=test ENV=dev printf "\n${YELLOW}*** warming cache in $ID_ENV${NOCOLOR}\n\n" bash scripts/invoke-warm-cache.sh --env dev diff --git a/scripts/build-image-job.sh b/scripts/build-image-job.sh deleted file mode 100644 index f0219c9d9..000000000 --- a/scripts/build-image-job.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -e - -# set in .github/workflows/build-all-images.yaml -if [[ -z $ENV_ID ]]; then - echo "ENV_ID is not set" - exit 1 -fi - -if [[ "$#" -ne 4 ]]; then - echo "use: bash scripts/build-image-job.sh --service-name request-create --build-ctx ." - exit 1 -fi - -while [[ "$#" -gt 0 ]]; do - case $1 in - --service-name) SERVICE_NAME="$2"; shift ;; - --build-ctx) BUILD_CTX="$2"; shift ;; - *) echo "unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - -PROJECT_CONF=project.yaml -REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) -ENV=dev -ID_ENV="$ENV_ID-$ENV" -ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) -ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" -SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) -SERVICES_DIR=$(echo $SERVICES_ZIP | sed 's/.zip//') - -aws s3 cp s3://$ARTIFACTS_BUCKET/$SERVICES_ZIP . --region $REGION - -unzip $SERVICES_ZIP -d $SERVICES_DIR - -cd $SERVICES_DIR - -docker build -t $SERVICE_NAME:latest -f ./docker/$SERVICE_NAME.Dockerfile --provenance=false $BUILD_CTX \ No newline at end of file diff --git a/scripts/build-image.sh b/scripts/build-image.sh index 25990b1f3..a8cf19df2 100644 --- a/scripts/build-image.sh +++ b/scripts/build-image.sh @@ -1,13 +1,16 @@ #!/bin/bash -if [[ "$#" -ne 4 ]]; then +if [[ "$#" -lt 4 ]]; then cat <<-'EOF' use: bash scripts/build-image.sh --app-name rule --build-ctx . + bash scripts/build-image.sh --app-name rule --build-ctx . --no-test EOF exit 1 fi +BUILD_ARGS="" + while [[ "$#" -gt 0 ]]; do case $1 in --app-name) @@ -18,6 +21,9 @@ while [[ "$#" -gt 0 ]]; do BUILD_CTX="$2" shift ;; + --no-test) + BUILD_ARGS="--build-arg RUN_TESTS=false" + ;; *) echo "unknown parameter passed: $1" exit 1 @@ -32,4 +38,4 @@ HASH=$(git rev-parse --short=$SHORT_GIT_SHA_LENGTH HEAD) IMAGE_TAG="$APP_NAME:$HASH" DOCKERFILE_PATH=./docker/$APP_NAME.Dockerfile -docker build -f $DOCKERFILE_PATH -t $IMAGE_TAG --provenance=false "$BUILD_CTX" \ No newline at end of file +docker build -f $DOCKERFILE_PATH -t $IMAGE_TAG --provenance=false $BUILD_ARGS "$BUILD_CTX" \ No newline at end of file diff --git a/scripts/compose.sh b/scripts/compose.sh index 32e58cd70..46add2f32 100644 --- a/scripts/compose.sh +++ b/scripts/compose.sh @@ -36,14 +36,24 @@ COMPOSE_DIR=./docker INIT_CMD="GRAPHQL_URI=$B64_GRAPHQL_URI \\ docker compose \\ -f $COMPOSE_DIR/storage.yaml \\ - -f $COMPOSE_DIR/services.yaml" + -f $COMPOSE_DIR/services.yaml \\ + -f $COMPOSE_DIR/measure.yaml" if [[ $UP ]]; then + if [[ $BUILD ]]; then + # build storage first for reliable startup + STORAGE_BUILD_CMD="docker compose -f $COMPOSE_DIR/storage.yaml build" + echo "$STORAGE_BUILD_CMD" + echo "" + eval "$STORAGE_BUILD_CMD" + fi + UP_CMD=$(printf '%s \\\n up \\\n -d \\\n --renew-anon-volumes' "$INIT_CMD") + # use --no-build when build was already done above if [[ $BUILD ]]; then - UP_CMD=$(printf '%s \\\n --build' "$UP_CMD") + UP_CMD=$(printf '%s \\\n --no-build' "$UP_CMD") fi echo "$UP_CMD" diff --git a/scripts/delete-dev-env.sh b/scripts/delete-dev-env.sh index e0b111dd0..d9bf41613 100644 --- a/scripts/delete-dev-env.sh +++ b/scripts/delete-dev-env.sh @@ -1,49 +1,40 @@ #!/bin/bash -while [[ "$#" -gt 0 ]]; do - case $1 in - --force) FORCE=1; shift ;; - *) echo "unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - - -if [[ ! "$FORCE" ]]; then - set -e -fi +set -e YELLOW='\033[0;33m' NOCOLOR='\033[0m' ENV=dev PROJECT_CONF=project.yaml -ENV_FILE_NAME=$(yq '.env_var.set.ENV_FILE_NAME.default' $PROJECT_CONF) -ENV_FILE=$ENV_FILE_NAME ENV_ID=$(source scripts/print-env-id.sh) if [[ -z $ENV_ID ]]; then - echo "ENV_ID not found in $ENV_FILE" - echo "bash scripts/set-custom-env-id.sh --env-id 12345 before continuing" + echo "ENV_ID not found. run 'make env-id' first" exit 1 fi pushd infra/terraform/aws/environments/dev +printf "\n${YELLOW}*** destroying $ENV_ID-$ENV environment${NOCOLOR}\n\n" terraform destroy --auto-approve && rm -rf .terraform* .tfplan* popd pushd infra/terraform/aws/environments/region -# skip if api gateway logging permission not managed by local terraform if [[ -f terraform.tfstate ]]; then + printf "\n${YELLOW}*** destroying api gateway logging permission${NOCOLOR}\n\n" terraform destroy --auto-approve && rm -rf .terraform* terraform.tfstate -# todo: elif manually delete logging permission if current resource matches naming convention fi popd -source scripts/delete-dev-storage.sh # --env arg not available here, dev only +pushd infra/terraform/aws/environments/init-dev + +printf "\n${YELLOW}*** destroying $ENV_ID-$ENV storage${NOCOLOR}\n\n" +terraform destroy --auto-approve && rm -rf .terraform* terraform.tfstate + +popd -printf "\n${YELLOW}*** ${ENV_ID}-${ENV} env deleted. you may now delete your workspace${NOCOLOR}\n" \ No newline at end of file +printf "\n${YELLOW}*** $ENV_ID-$ENV deleted${NOCOLOR}\n" diff --git a/scripts/delete-dev-images.sh b/scripts/delete-dev-images.sh deleted file mode 100644 index ab4b04f00..000000000 --- a/scripts/delete-dev-images.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -e - -if [[ "$#" -ne 2 ]]; then - cat <<- 'EOF' - use: - bash scripts/delete-dev-images.sh --app-name client - EOF - exit 1 -fi - -while [[ "$#" -gt 0 ]]; do - case $1 in - --app-name) APP_NAME="$2"; shift ;; - *) echo "unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - -PROJECT_CONF=project.yaml -ENV=dev -ENV_ID=$(source ./scripts/print-env-id.sh) -ID_ENV_PREFIX="$ENV_ID/$ENV" -REPO_NAME="$ID_ENV_PREFIX/$APP_NAME" -REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) -AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) - -# returns: sha256:a9be998dcb5479d44b471381b79cb26a3b864d0fe996943c7abaf42c99638c9d sha256:ae20a55d89d8cece6f38dad19806637b400888fc6375bc167d6cf252e34dcb93 sha256:67dcb4b0a3bbc0bafb16cc163178f3b6c0a0d103694c26f085197c7015c33914 -IMAGE_DIGESTS=($(aws ecr list-images --repository-name $REPO_NAME --registry-id $AWS_ACCOUNT_ID --region $REGION --query 'imageIds[*].imageDigest' --output text)) - -declare IMAGE_IDS -# add imageDigest assignments to IMAGE_IDS -for i in "${IMAGE_DIGESTS[@]}"; do - IMAGE_IDS+="imageDigest=$i " -done - -# remove trailing whitespace -IMAGE_IDS=$(echo $IMAGE_IDS | xargs) - -aws ecr batch-delete-image --repository-name $REPO_NAME --registry-id $AWS_ACCOUNT_ID --region $REGION --image-ids $IMAGE_IDS \ No newline at end of file diff --git a/scripts/delete-dev-storage.sh b/scripts/delete-dev-storage.sh deleted file mode 100644 index c655a3a5a..000000000 --- a/scripts/delete-dev-storage.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -ENV=dev # hardcoding intended -PROJECT_CONF=project.yaml -REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) -ENV_ID=$(source ./scripts/print-env-id.sh) -ID_ENV="$ENV_ID-$ENV" -ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) -TFSTATE_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.TFSTATE_BUCKET_PREFIX.default' $PROJECT_CONF) -LOCAL_TFSTATE_FILE=terraform.tfstate - -ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" -TFSTATE_BUCKET="$TFSTATE_BUCKET_PREFIX-$ID_ENV" - -INIT_DEV_DIR=infra/terraform/aws/environments/init-dev - -export AWS_DEFAULT_REGION="$REGION" - -function delete_bucket() { - local bucket_name="$1" - - aws s3 rm "s3://$bucket_name" --recursive - - aws s3api delete-bucket --bucket "$bucket_name" -} - -function delete_dev_storage() { - # delete buckets - delete_bucket "$ARTIFACTS_BUCKET" - delete_bucket "$TFSTATE_BUCKET" - - popd - source ./scripts/delete-ecr-repos.sh - pushd $INIT_DEV_DIR -} - -pushd $INIT_DEV_DIR - -if ! [[ -f $LOCAL_TFSTATE_FILE ]]; then - echo "tfstate not found. manually deleting dev storage" - delete_dev_storage -elif [[ $(yq '.resources | length' $LOCAL_TFSTATE_FILE) -eq 0 ]]; then - echo "resources not found in tfstate. manually deleting dev storage" - delete_dev_storage -fi - -set +e -terraform destroy --auto-approve 2>/dev/null - -if [[ "$?" -ne 0 ]]; then - echo "destroy incomplete. manually deleting dev storage" - delete_dev_storage - popd -else - popd -fi diff --git a/scripts/delete-ecr-repos.sh b/scripts/delete-ecr-repos.sh deleted file mode 100644 index 6643df829..000000000 --- a/scripts/delete-ecr-repos.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -PROJECT_CONF=project.yaml -ENV=dev -ENV_ID=$(source scripts/print-env-id.sh) -REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) - -for APP_NAME in $(bash scripts/list-deployments.sh | xargs basename -a); do - IMAGE_NAME="$ENV_ID/$ENV/$APP_NAME" - aws ecr delete-repository --repository-name $IMAGE_NAME --region $REGION --force -done \ No newline at end of file diff --git a/scripts/deploy-all-images.sh b/scripts/deploy-all-images.sh deleted file mode 100644 index c2d172515..000000000 --- a/scripts/deploy-all-images.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash - -set -e - -if [[ -z $GITHUB_PAT ]]; then - echo "set GITHUB_PAT variable in shell to continue" - exit 1 -fi - -ENV=dev -PROJECT_CONF=project.yaml -ENV_ID=$(source ./scripts/print-env-id.sh) -ID_ENV="$ENV_ID-$ENV" -REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) -ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) -ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" -DEPLOY_IMAGE_WORKFLOW=$(yq '.[".github"].workflows.env_var.set.DEPLOY_IMAGE_WORKFLOW.default' $PROJECT_CONF) -WORKFLOW_ID=$DEPLOY_IMAGE_WORKFLOW -GITHUB_ORG=$(yq '.[".github"].env_var.set.GITHUB_ORG.default' $PROJECT_CONF) -GITHUB_REPO_NAME=$(yq '.[".github"].env_var.set.GITHUB_REPO_NAME.default' $PROJECT_CONF) -SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) - -source scripts/zip-services.sh - -echo '*** uploading archive to s3' -aws s3 cp $SERVICES_ZIP s3://$ARTIFACTS_BUCKET --region $REGION -rm $SERVICES_ZIP - -echo "*** triggering .github/workflows/$WORKFLOW_ID" - -curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_PAT" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/workflows/$WORKFLOW_ID/dispatches \ - -d "{\"ref\":\"develop\"}" - -if [[ $(uname) == "Darwin" ]]; then - # store utc time in iso8601 format minus 10 seconds - CREATED=$(date -u -v-10S "+%Y-%m-%dT%H:%M:%SZ") - # store utc time in unix timestamp format plus 10 minutes - TEN_MIN_MAX=$(date -u -v+10M "+%s") -else - CREATED=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 10 seconds ago" +'%Y-%m-%dT%H:%M:%SZ') - TEN_MIN_MAX=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 10 minutes" +'%s') -fi - -# wait 5 seconds -sleep 5 - -RUN_ID=$(curl -sL \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_PAT" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/workflows/$WORKFLOW_ID/runs?created=>$CREATED" | yq '.workflow_runs[0].id') - -echo "*** waiting for $WORKFLOW_ID github workflow to complete" - -function get_run() { - RUN=$(curl -sL \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GITHUB_PAT" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/$GITHUB_ORG/$GITHUB_REPO_NAME/actions/runs/$RUN_ID) - - STATUS=$(echo "$RUN" | yq '.status') - CONCLUSION=$(echo "$RUN" | yq '.conclusion') -} - -STATUS='queued' -CONCLUSION=null - -while [[ $STATUS != 'completed' && $(date +%s) -lt $TEN_MIN_MAX ]]; do - sleep 5 - get_run - printf '%s' '.' -done - -echo "" - -if [[ $CONCLUSION != 'success' ]]; then - echo "build failed" - exit 1 -fi \ No newline at end of file diff --git a/scripts/deploy-image-job.sh b/scripts/deploy-image-job.sh deleted file mode 100644 index e9b1ee80d..000000000 --- a/scripts/deploy-image-job.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -set -e - -# set in .github/workflows/deploy-all-images.yaml -if [[ -z $ENV_ID ]]; then - echo "ENV_ID is not set" - exit 1 -fi - -if [[ "$#" -ne 2 ]]; then - echo "use: bash scripts/deploy-image-job.sh --service-name request-create" - exit 1 -fi - -while [[ "$#" -gt 0 ]]; do - case $1 in - --service-name) SERVICE_NAME="$2"; shift ;; - *) echo "unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - -PROJECT_CONF=project.yaml -REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) -ENV=dev -ID_ENV="$ENV_ID-$ENV" -ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) -ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" -SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) -PROJECT_DIR=$(echo $SERVICES_ZIP | sed 's/.zip//') - -source scripts/auth-ecr.sh - -aws s3 cp s3://$ARTIFACTS_BUCKET/$SERVICES_ZIP . --region $REGION - -unzip $SERVICES_ZIP -d $PROJECT_DIR - -cd $PROJECT_DIR - -SERVICE_DIR=$(bash scripts/list-deployments.sh | grep --color=never $SERVICE_NAME) - -cd "$SERVICE_DIR" - -make --no-print-directory deploy \ No newline at end of file diff --git a/scripts/ecr-images.sh b/scripts/ecr-images.sh new file mode 100644 index 000000000..cee75ecb7 --- /dev/null +++ b/scripts/ecr-images.sh @@ -0,0 +1,296 @@ +#!/bin/bash + +set -e + +YELLOW='\033[0;33m' +RED='\033[0;31m' +RESET='\033[0m' + +if [[ "$#" -lt 1 ]]; then + cat <<- 'EOF' + use: + bash scripts/ecr-images.sh --build # build + test all + bash scripts/ecr-images.sh --build --no-test # build all (skip tests) + bash scripts/ecr-images.sh --build --push # build + test + push all + bash scripts/ecr-images.sh --build --push --deploy # build + test + push + deploy all + bash scripts/ecr-images.sh --pull # pull all from ecr + bash scripts/ecr-images.sh --integ # run integration tests + + --service works with any flag: + bash scripts/ecr-images.sh --build --service graphql + bash scripts/ecr-images.sh --build --push --deploy --service graphql + bash scripts/ecr-images.sh --pull --service graphql + EOF + exit 1 +fi + +# defaults +BUILD=false +TEST=true +PUSH=false +DEPLOY=false +PULL=false +INTEG=false +SERVICE="" + +while [[ "$#" -gt 0 ]]; do + case $1 in + --build) BUILD=true; shift ;; + --test) TEST=true; shift ;; + --no-test) TEST=false; shift ;; + --push) PUSH=true; shift ;; + --deploy) DEPLOY=true; shift ;; + --pull) PULL=true; shift ;; + --integ) INTEG=true; shift ;; + --service) SERVICE="$2"; shift; shift ;; + *) echo "unknown parameter passed: $1"; exit 1 ;; + esac +done + +if [[ $(basename $(pwd)) != "mxfactorial" ]]; then + echo "error: run from project root" + exit 1 +fi + +ENV=dev +PROJECT_CONF=project.yaml +ENV_FILE_NAME=$(yq '.env_var.set.ENV_FILE_NAME.default' $PROJECT_CONF) + +if [[ ! -f $ENV_FILE_NAME ]]; then + echo "error: $ENV_FILE_NAME not found. run 'make env-id' first" + exit 1 +fi + +if ! grep -q "ENV_ID=" $ENV_FILE_NAME; then + echo "error: ENV_ID not found in $ENV_FILE_NAME. run 'make env-id' first" + exit 1 +fi + +# validate service if provided +if [[ -n "$SERVICE" ]]; then + if [[ $(bash scripts/list-dir-paths.sh --type all | grep --color=never "$SERVICE$" >/dev/null 2>&1; echo $?) -ne 0 ]]; then + echo "error: \"$SERVICE\" not in $PROJECT_CONF" + exit 1 + fi +fi + +ENV_ID=$(grep "ENV_ID=" $ENV_FILE_NAME | cut -d'=' -f2) +ID_ENV="$ENV_ID-$ENV" +REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +ECR_URI="$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$ENV_ID/$ENV" + +function get_services() { + if [[ -n "$SERVICE" ]]; then + echo "$SERVICE" + else + yq '.. | select(has("type") and has("deploy") and .type == "app" and .deploy == true) | path | .[-1]' $PROJECT_CONF + fi +} + +function start_builds() { + local RUN_TESTS=$1 + local PUSH_IMAGE=$2 + local DO_DEPLOY=$3 + + ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) + ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" + SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) + BUILD_OBJECT_KEY_PATH=$(yq '.infra.terraform.aws.modules.codepipeline.env_var.set.BUILD_OBJECT_KEY_PATH.default' $PROJECT_CONF) + + source scripts/zip-services.sh + + echo '*** uploading archive to s3' + aws s3 cp $SERVICES_ZIP s3://$ARTIFACTS_BUCKET/$BUILD_OBJECT_KEY_PATH/ --region $REGION + rm $SERVICES_ZIP + + echo "*** starting builds (RUN_TESTS=$RUN_TESTS, PUSH_IMAGE=$PUSH_IMAGE, DEPLOY=$DO_DEPLOY)" + echo "" + + BUILD_IDS="" + for SVC in $(get_services); do + PROJECT_NAME="mxfactorial-$SVC-$ID_ENV" + echo -e "${YELLOW}starting $PROJECT_NAME...${RESET}" + BUILD_ID=$(aws codebuild start-build \ + --project-name $PROJECT_NAME \ + --region $REGION \ + --source-type-override S3 \ + --source-location-override "$ARTIFACTS_BUCKET/$BUILD_OBJECT_KEY_PATH/$SERVICES_ZIP" \ + --artifacts-override type=NO_ARTIFACTS \ + --environment-variables-override \ + name=RUN_TESTS,value=$RUN_TESTS \ + name=PUSH_IMAGE,value=$PUSH_IMAGE \ + name=DEPLOY,value=$DO_DEPLOY \ + --query 'build.id' \ + --output text) + BUILD_IDS="$BUILD_IDS $BUILD_ID" + # URL encode the build ID (replace : with %3A) + BUILD_ID_ENCODED=$(printf '%s' "$BUILD_ID" | sed 's/:/%3A/g') + printf 'https://%s.console.aws.amazon.com/codesuite/codebuild/%s/projects/%s/build/%s/?region=%s\n' "$REGION" "$AWS_ACCOUNT_ID" "$PROJECT_NAME" "$BUILD_ID_ENCODED" "$REGION" + echo "" + done + + echo "*** waiting for builds to complete" + echo -e "${YELLOW}(ctrl+c to exit - builds will continue in background)${RESET}" + echo "" + + for BUILD_ID in $BUILD_IDS; do + while true; do + STATUS=$(aws codebuild batch-get-builds \ + --ids $BUILD_ID \ + --region $REGION \ + --query 'builds[0].buildStatus' \ + --output text) + if [[ $STATUS != "IN_PROGRESS" ]]; then + echo "$BUILD_ID: $STATUS" + break + fi + sleep 10 + printf '%s' '.' + done + done +} + +function trigger_pipeline() { + ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) + ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" + SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) + PIPELINE_NAME="mxfactorial-build-$ID_ENV" + + source scripts/zip-services.sh + + echo '*** uploading archive to s3' + aws s3 cp $SERVICES_ZIP s3://$ARTIFACTS_BUCKET/build/ --region $REGION + rm $SERVICES_ZIP + + echo "*** codepipeline $PIPELINE_NAME triggered via eventbridge" + echo "https://$REGION.console.aws.amazon.com/codesuite/codepipeline/pipelines/$PIPELINE_NAME/view/?region=$REGION" + + sleep 5 + + echo "*** waiting for codepipeline to complete" + echo -e "${YELLOW}(ctrl+c to exit - pipeline will continue in background)${RESET}" + + function get_pipeline_state() { + PIPELINE_STATE=$(aws codepipeline get-pipeline-state \ + --name $PIPELINE_NAME \ + --region $REGION \ + --query 'stageStates[?stageName==`Build`].latestExecution.status' \ + --output text) + } + + if [[ $(uname) == "Darwin" ]]; then + TIMEOUT_MAX=$(date -u -v+20M "+%s") + else + TIMEOUT_MAX=$(date -u -d "$(date -u +'%Y-%m-%dT%H:%M:%S') 20 minutes" +'%s') + fi + + PIPELINE_STATE='InProgress' + + while [[ $PIPELINE_STATE == 'InProgress' && $(date +%s) -lt $TIMEOUT_MAX ]]; do + sleep 10 + get_pipeline_state + printf '%s' '.' + done + + echo "" + + if [[ $PIPELINE_STATE == 'Succeeded' ]]; then + echo "*** build succeeded" + else + echo "*** build status: $PIPELINE_STATE" + exit 1 + fi +} + +# handle pull separately +if [[ $PULL == true ]]; then + echo "*** logging into ecr" + source scripts/auth-ecr.sh + + echo "*** pulling images from ecr" + for SVC in $(get_services); do + echo "pulling $SVC..." + docker pull $ECR_URI/$SVC:latest || echo "skipping $SVC (not found)" + docker tag $ECR_URI/$SVC:latest $SVC:latest 2>/dev/null || true + done + + echo "*** done" + exit 0 +fi + +# handle integ tests +if [[ $INTEG == true ]]; then + ARTIFACTS_BUCKET_PREFIX=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.ARTIFACTS_BUCKET_PREFIX.default' $PROJECT_CONF) + ARTIFACTS_BUCKET="$ARTIFACTS_BUCKET_PREFIX-$ID_ENV" + SERVICES_ZIP=$(yq '.scripts.env_var.set.SERVICES_ZIP.default' $PROJECT_CONF) + INTEG_TEST_OBJECT_KEY_PATH=$(yq '.infra.terraform.aws.modules["project-storage"].env_var.set.INTEG_TEST_OBJECT_KEY_PATH.default' $PROJECT_CONF) + + source scripts/zip-services.sh + + echo '*** uploading archive to s3' + aws s3 cp $SERVICES_ZIP s3://$ARTIFACTS_BUCKET/$INTEG_TEST_OBJECT_KEY_PATH/ --region $REGION + rm $SERVICES_ZIP + + PROJECT_NAME="mxfactorial-integ-$ID_ENV" + echo -e "${YELLOW}*** starting integration tests ($PROJECT_NAME)${RESET}" + echo "" + + BUILD_ID=$(aws codebuild start-build \ + --project-name $PROJECT_NAME \ + --region $REGION \ + --query 'build.id' \ + --output text) + + # URL encode the build ID (replace : with %3A) + BUILD_ID_ENCODED=$(printf '%s' "$BUILD_ID" | sed 's/:/%3A/g') + printf 'https://%s.console.aws.amazon.com/codesuite/codebuild/%s/projects/%s/build/%s/?region=%s\n' "$REGION" "$AWS_ACCOUNT_ID" "$PROJECT_NAME" "$BUILD_ID_ENCODED" "$REGION" + echo "" + + echo "*** waiting for integ tests to complete" + echo -e "${YELLOW}(ctrl+c to exit - build will continue in background)${RESET}" + echo "" + + while true; do + STATUS=$(aws codebuild batch-get-builds \ + --ids $BUILD_ID \ + --region $REGION \ + --query 'builds[0].buildStatus' \ + --output text) + if [[ $STATUS != "IN_PROGRESS" ]]; then + echo "$BUILD_ID: $STATUS" + break + fi + sleep 10 + printf '%s' '.' + done + + if [[ $STATUS == "SUCCEEDED" ]]; then + echo "*** integration tests passed" + else + echo "*** integration tests failed" + exit 1 + fi + + exit 0 +fi + +# handle build +if [[ $BUILD == true ]]; then + RUN_TESTS=$TEST + PUSH_IMAGE=$PUSH + DO_DEPLOY=$DEPLOY + + # warn if deploying without pushing + if [[ $DEPLOY == true && $PUSH == false ]]; then + echo -e "${RED}warning: deploying without pushing (will use existing ecr images)${RESET}" + echo "" + fi + + # use pipeline for full builds (all services + push + no deploy), otherwise direct codebuild + if [[ -z "$SERVICE" && $PUSH == true && $TEST == true && $DEPLOY == false ]]; then + trigger_pipeline + else + start_builds "$RUN_TESTS" "$PUSH_IMAGE" "$DO_DEPLOY" + fi +fi diff --git a/scripts/go-migrate-rds.sh b/scripts/go-migrate-rds.sh index 038f0328d..5e763afec 100644 --- a/scripts/go-migrate-rds.sh +++ b/scripts/go-migrate-rds.sh @@ -26,14 +26,26 @@ PROJECT_CONF=project.yaml ENV_ID=$(source scripts/print-env-id.sh) SSM_VERSION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.SSM_VERSION.default' $PROJECT_CONF) REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) +SSM_PREFIX="$ENV_ID/$SSM_VERSION/$ENV" -PASSPHRASE=$(aws ssm get-parameter \ - --name "/$ENV_ID/$SSM_VERSION/$ENV/tool/lambda/go_migrate/passphrase" \ +echo "*** fetching go-migrate url and passphrase from ssm" + +GO_MIGRATE_URL=$(aws ssm get-parameter \ + --name "/$SSM_PREFIX/service/lambda/go_migrate/url" \ + --with-decryption \ --query 'Parameter.Value' \ - --region $REGION \ + --output text \ + --region $REGION) + +PASSPHRASE=$(aws ssm get-parameter \ + --name "/$SSM_PREFIX/tool/lambda/go_migrate/passphrase" \ --with-decryption \ - --output text) + --query 'Parameter.Value' \ + --output text \ + --region $REGION) -PAYLOAD="{\"db_type\":\"test\",\"cmd\":\"$CMD\",\"passphrase\":\"$PASSPHRASE\"}" +echo "*** invoking go-migrate lambda" -bash scripts/invoke-function-url.sh --app-name go-migrate --payload "$PAYLOAD" --env $ENV +curl -s -X POST "$GO_MIGRATE_URL" \ + -H 'Content-Type: application/json' \ + -d "{\"db_type\":\"test\",\"cmd\":\"$CMD\",\"passphrase\":\"$PASSPHRASE\"}" | yq -o=json diff --git a/scripts/manage-rds.sh b/scripts/manage-rds.sh index 0cd3d1ee6..9db368f17 100644 --- a/scripts/manage-rds.sh +++ b/scripts/manage-rds.sh @@ -19,8 +19,8 @@ PROJECT_CONF=project.yaml REGION=$(yq '.infra.terraform.aws.modules.environment.env_var.set.REGION.default' $PROJECT_CONF) ENV_ID=$(source scripts/print-env-id.sh) -RDS_INSTANCE_NAME_PREFIX=$(yq '.infra.terraform.aws.modules.environment.env_var.set.RDS_INSTANCE_NAME_PREFIX.default' $PROJECT_CONF) -RDS_INSTANCE_NAME="$RDS_INSTANCE_NAME_PREFIX-$ENV_ID-$ENV" +NAME_PREFIX=$(yq '.infra.terraform.aws.modules.environment.env_var.set.NAME_PREFIX.default' $PROJECT_CONF) +RDS_INSTANCE_NAME="$NAME_PREFIX-$ENV_ID-$ENV" if [[ "$START" -eq 1 ]]; then aws rds start-db-instance \ diff --git a/scripts/pull-all-images.sh b/scripts/pull-all-images.sh deleted file mode 100644 index 34af58320..000000000 --- a/scripts/pull-all-images.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -e - -PROJECT_CONF=project.yaml -GITHUB_REGISTRY=$(yq '.[".github"].workflows.env_var.set.GITHUB_REGISTRY.default' $PROJECT_CONF) -GITHUB_ORG=$(yq '.[".github"].env_var.set.GITHUB_ORG.default' $PROJECT_CONF) -GITHUB_REPO_NAME=$(yq '.[".github"].env_var.set.GITHUB_REPO_NAME.default' $PROJECT_CONF) -LOCAL_TAG_VERSION=$(yq '.docker.env_var.set.LOCAL_TAG_VERSION.default' $PROJECT_CONF) - -NAMESPACE=$GITHUB_ORG/$GITHUB_REPO_NAME -REGISTRY_URI=$GITHUB_REGISTRY/$NAMESPACE - -SERVICES=($(bash scripts/list-deployments.sh | xargs basename -a)) - -for SERVICE in "${SERVICES[@]}"; do - IMAGE_NAME=$SERVICE:$LOCAL_TAG_VERSION - docker pull $REGISTRY_URI/$IMAGE_NAME - docker tag $REGISTRY_URI/$IMAGE_NAME $IMAGE_NAME -done - -echo "" -echo '*** "make compose-up" to start services in docker' \ No newline at end of file diff --git a/scripts/start-local.sh b/scripts/start-local.sh index dec176902..f6204bb3e 100644 --- a/scripts/start-local.sh +++ b/scripts/start-local.sh @@ -44,24 +44,24 @@ for d in "${APP_DIRS[@]}"; do RUNTIME=$(yq "$CONF_PATH.runtime" $PROJECT_CONF) BUILD_SRC_PATH=$(yq "$CONF_PATH.build_src_path" $PROJECT_CONF) - # compile rust before starting (skip in CI - uses pre-built binaries) - if [[ -z "$CI" ]] && [[ "$RUNTIME" == "$RUST_RUNTIME" ]]; then + # compile rust before starting (skip when using pre-built binaries from services workflow artifacts) + if [[ -z "$SERVICES_WORKFLOW" ]] && [[ "$RUNTIME" == "$RUST_RUNTIME" ]]; then echo -e -n "\n${GREEN}*** compiling $d${RESET}\n" make --no-print-directory -C "$d" compile fi - # skip starting client in workflows - if [[ "$CI" ]] && [[ "$d" == 'client' ]]; then + # skip starting client in services workflows + if [[ "$SERVICES_WORKFLOW" ]] && [[ "$d" == 'client' ]]; then continue fi echo -e -n "\n${GREEN}*** starting $d${RESET}\n" - if [[ "$CI" ]]; then + if [[ "$SERVICES_WORKFLOW" ]]; then make --no-print-directory -C "$d" get-secrets ENV=local > /dev/null # &; \ disown fails in make so backgrounding kept in bash if [[ "$RUNTIME" == "$RUST_RUNTIME" ]]; then - # run pre-built binary from CI matrix compile job + # run pre-built binary from services workflow compile job artifacts # skips cargo fingerprint check which would recompile BINARY_PATH="$(pwd)/target/debug/$(basename "$d")" (cd "$d"; eval $(cat $ENV_FILE_NAME) $BINARY_PATH > /dev/null 2>&1 & disown $!) diff --git a/scripts/zip-services.sh b/scripts/zip-services.sh index b6947a79a..d3445c137 100644 --- a/scripts/zip-services.sh +++ b/scripts/zip-services.sh @@ -25,14 +25,9 @@ zip -r $SERVICES_ZIP \ services \ crates \ tests \ - migrations/go-migrate \ - migrations/warm-cache \ - migrations/schema \ - migrations/seed \ - migrations/testseed \ + migrations \ --exclude='*/.env' \ --exclude='*/README.md' \ - --exclude='tests/testdata/*' \ --exclude='tests/thunder-tests/*' \ --exclude='services/graphql/postman/*' \ --exclude='client/node_modules/*' \ diff --git a/tests/makefile b/tests/makefile index cd9a237ba..dab3927c9 100644 --- a/tests/makefile +++ b/tests/makefile @@ -17,7 +17,7 @@ test-docker: test-cloud: @$(MAKE) -s test-env-arg - @$(MAKE) --no-print-directory -C ../migrations resetrds ENV=$(ENV) DB=test + @$(MAKE) --no-print-directory -C ../migrations/go-migrate resetrds ENV=$(ENV) DB=test @cd $(RELATIVE_PROJECT_ROOT_PATH); bash scripts/invoke-warm-cache.sh --env $(ENV) @$(MAKE) --no-print-directory -S get-secrets ENV=$(ENV) cargo test --features integration_tests -- --test-threads=1