diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 8b19b335..dc223927 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -7,7 +7,7 @@ on: branches: - master env: - VERSION: 10.0.2 + VERSION: 10.0.3 jobs: build-and-deploy: runs-on: ubuntu-latest diff --git a/Api.Auth.External.Custom/README.md b/Api.Auth.External.Custom/README.md index 884122b6..82cdea13 100644 --- a/Api.Auth.External.Custom/README.md +++ b/Api.Auth.External.Custom/README.md @@ -133,8 +133,8 @@ The secrets defined in GitHub must also be mapped for the `Staging` and `Product ```yaml env: - AUTH_JWT_PUBLIC_KEY: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_AUTH_JWT_PUBLIC_KEY || secrets.STAGING_AUTH_JWT_PUBLIC_KEY }} - AUTH_JWT_PRIVATE_KEY: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_AUTH_JWT_PRIVATE_KEY || secrets.STAGING_AUTH_JWT_PRIVATE_KEY }} + AUTH_JWT_PUBLIC_KEY: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_AUTH_JWT_PUBLIC_KEY || secrets.STAGING_AUTH_JWT_PUBLIC_KEY }} + AUTH_JWT_PRIVATE_KEY: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_AUTH_JWT_PRIVATE_KEY || secrets.STAGING_AUTH_JWT_PRIVATE_KEY }} ``` ...and created during the Kubernetes deploy step. diff --git a/Api.Auth.RootLogin/README.md b/Api.Auth.RootLogin/README.md index 7f951ac6..2ab1b9cd 100644 --- a/Api.Auth.RootLogin/README.md +++ b/Api.Auth.RootLogin/README.md @@ -133,8 +133,8 @@ The secrets defined in GitHub must also be mapped for the `Staging` and `Product ```yaml env: - AUTH_JWT_PUBLIC_KEY: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_AUTH_JWT_PUBLIC_KEY || secrets.STAGING_AUTH_JWT_PUBLIC_KEY }} - AUTH_JWT_PRIVATE_KEY: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_AUTH_JWT_PRIVATE_KEY || secrets.STAGING_AUTH_JWT_PRIVATE_KEY }} + AUTH_JWT_PUBLIC_KEY: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_AUTH_JWT_PUBLIC_KEY || secrets.STAGING_AUTH_JWT_PUBLIC_KEY }} + AUTH_JWT_PRIVATE_KEY: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_AUTH_JWT_PRIVATE_KEY || secrets.STAGING_AUTH_JWT_PRIVATE_KEY }} ``` ...and created during the Kubernetes deploy step. diff --git a/Api.Data.Identity.Auth.ApiKey/README.md b/Api.Data.Identity.Auth.ApiKey/README.md index 1d3c1a37..f7af76b9 100644 --- a/Api.Data.Identity.Auth.ApiKey/README.md +++ b/Api.Data.Identity.Auth.ApiKey/README.md @@ -80,7 +80,7 @@ The secrets defined in GitHub must also be mapped for the `Staging` and `Product ```yaml env: - AUTH_API_KEY_SECRET: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_AUTH_API_KEY_SECRET || secrets.STAGING_AUTH_API_KEY_SECRET }} + AUTH_API_KEY_SECRET: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_AUTH_API_KEY_SECRET || secrets.STAGING_AUTH_API_KEY_SECRET }} ``` ...and created during the Kubernetes deploy step. diff --git a/Api.Data.Identity.Auth.Jwt/README.md b/Api.Data.Identity.Auth.Jwt/README.md index f8660aeb..7261ced8 100644 --- a/Api.Data.Identity.Auth.Jwt/README.md +++ b/Api.Data.Identity.Auth.Jwt/README.md @@ -126,8 +126,8 @@ The secrets defined in GitHub must also be inbcluded in environmental variables ```yaml env: - AUTH_JWT_PUBLIC_KEY: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_AUTH_JWT_PUBLIC_KEY || secrets.STAGING_AUTH_JWT_PUBLIC_KEY }} - AUTH_JWT_PRIVATE_KEY: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_AUTH_JWT_PRIVATE_KEY || secrets.STAGING_AUTH_JWT_PRIVATE_KEY }} + AUTH_JWT_PUBLIC_KEY: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_AUTH_JWT_PUBLIC_KEY || secrets.STAGING_AUTH_JWT_PUBLIC_KEY }} + AUTH_JWT_PRIVATE_KEY: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_AUTH_JWT_PRIVATE_KEY || secrets.STAGING_AUTH_JWT_PRIVATE_KEY }} ``` ...and created during the Kubernetes deploy step. diff --git a/Api.Data.MySql/README.md b/Api.Data.MySql/README.md index 5573f9a4..3ef411a2 100644 --- a/Api.Data.MySql/README.md +++ b/Api.Data.MySql/README.md @@ -149,8 +149,8 @@ env: AZURE_GROUP_DATABASE : ${{ vars.AZURE_RESOURCE_GROUP_DATABASE }} SQL_NAME: nanoDb SQL_USER: api-data-mysql-user - SQL_PASSWORD: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} - SQL_ADMIN_PASSWORD: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_SQL_ADMIN_PASSWORD || secrets.STAGING_SQL_ADMIN_PASSWORD }} + SQL_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} + SQL_ADMIN_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_ADMIN_PASSWORD || secrets.STAGING_SQL_ADMIN_PASSWORD }} ``` Additionally, this step has been added to ensure database migrations are applied, and the application database user has been created before the application is deployed. diff --git a/Api.Data.PostgreSQL/README.md b/Api.Data.PostgreSQL/README.md index 04ebc449..e32673af 100644 --- a/Api.Data.PostgreSQL/README.md +++ b/Api.Data.PostgreSQL/README.md @@ -147,8 +147,8 @@ env: AZURE_GROUP_DATABASE : ${{ vars.AZURE_RESOURCE_GROUP_DATABASE }} SQL_NAME: nanoDb SQL_USER: api-data-postgres-user - SQL_PASSWORD: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} - SQL_ADMIN_PASSWORD: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_SQL_ADMIN_PASSWORD || secrets.STAGING_SQL_ADMIN_PASSWORD }} + SQL_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} + SQL_ADMIN_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_ADMIN_PASSWORD || secrets.STAGING_SQL_ADMIN_PASSWORD }} ``` Additionally, this step has been added to ensure database migrations are applied, and the application database user has been created before the application is deployed. diff --git a/Api.Data.SqlServer.Spatial/.azure/.diagnostic-settings/logs.json b/Api.Data.SqlServer.Spatial/.azure/.diagnostic-settings/logs.json new file mode 100644 index 00000000..10c3505a --- /dev/null +++ b/Api.Data.SqlServer.Spatial/.azure/.diagnostic-settings/logs.json @@ -0,0 +1,10 @@ +[ + { + "category": "QueryStoreRuntimeStatistics", + "enabled": true + }, + { + "category": "SQLSecurityAuditEvents", + "enabled": true + } +] \ No newline at end of file diff --git a/Api.Data.SqlServer.Spatial/.azure/.diagnostic-settings/metrics.json b/Api.Data.SqlServer.Spatial/.azure/.diagnostic-settings/metrics.json new file mode 100644 index 00000000..551d2f13 --- /dev/null +++ b/Api.Data.SqlServer.Spatial/.azure/.diagnostic-settings/metrics.json @@ -0,0 +1,7 @@ +[ + { + "category": "AllMetrics", + "enabled": true, + "timeGrain": "PT1M" + } +] \ No newline at end of file diff --git a/Api.Data.SqlServer.Spatial/.github/workflows/build-and-deploy.yml b/Api.Data.SqlServer.Spatial/.github/workflows/build-and-deploy.yml index 8922d769..40473d5f 100644 --- a/Api.Data.SqlServer.Spatial/.github/workflows/build-and-deploy.yml +++ b/Api.Data.SqlServer.Spatial/.github/workflows/build-and-deploy.yml @@ -18,6 +18,7 @@ env: AZURE_GROUP_KUBERNETES: ${{ vars.AZURE_RESOURCE_GROUP_KUBERNETES }} AZURE_GROUP_DELIVERY: ${{ vars.AZURE_RESOURCE_GROUP_DELIVERY }} AZURE_GROUP_DATABASE : ${{ vars.AZURE_RESOURCE_GROUP_DATABASE }} + AZURE_GROUP_LOGS: ${{ vars.AZURE_RESOURCE_GROUP_LOGS }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} @@ -37,6 +38,10 @@ env: KUBERNETES_CPU_LIMIT: 600m KUBERNETES_CPU_SCALING: 180 ASPNETCORE_ENVIRONMENT: ${{ github.ref == 'refs/heads/main' && 'Production' || 'Staging' }} + SQL_SERVICE_OBJECTIVE: GP_Gen5_2 + SQL_EDITION: GeneralPurpose + SQL_MAX_SIZE: 64GB + SQL_BACKUP_RETENTION: 35 SQL_NAME: nanoDb SQL_USER: api-data-sqlserver-user SQL_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} @@ -124,6 +129,112 @@ jobs: echo "CONTAINER_REGISTRY_HOST=$env:CONTAINER_REGISTRY_HOST" >> $env:GITHUB_ENV; + - name: Create Database + shell: pwsh + run: | + $env:SQL_SERVER_NAME = az sql server list -g $env:AZURE_GROUP_DATABASE --query "[0].name" -o tsv; + $env:SQL_DB_EXISTS = az sql db show -g $env:AZURE_GROUP_DATABASE -s $env:SQL_SERVER_NAME -n $env:SQL_NAME --query name -o tsv 2>$null; + + if (-not $env:SQL_DB_EXISTS) + { + az sql db create ` + -n $env:SQL_NAME ` + -s $env:SQL_SERVER_NAME ` + -g $env:AZURE_GROUP_DATABASE ` + --edition $env:SQL_EDITION ` + --service-objective $env:SQL_SERVICE_OBJECTIVE ` + --max-size $env:SQL_MAX_SIZE ` + --backup-storage-redundancy Geo ` + --zone-redundant true; + + $env:MAINTENANCE_CONFIG_ID = az maintenance public-configuration list --query "[?name=='SQL_Default_1'].id" -o tsv; + + az sql db update ` + -n $env:SQL_NAME ` + -s $env:SQL_SERVER_NAME ` + -g $env:AZURE_GROUP_DATABASE ` + --maint-config-id $env:MAINTENANCE_CONFIG_ID; + + $env:DIAGNOSTIC_SETTINGS_NAME = "diagnostics-" + $env:SQL_NAME; + $env:WORKSPACE_ID = az monitor log-analytics workspace list -g $env:AZURE_GROUP_LOGS --query [0].[id] -o tsv; + $env:SQLDB_ID = az sql db show -g $env:AZURE_GROUP_DATABASE -s $env:SQL_SERVER_NAME -n $env:SQL_NAME --query id -o tsv; + + az monitor diagnostic-settings create ` + --name $env:DIAGNOSTIC_SETTINGS_NAME ` + --resource $env:SQLDB_ID ` + --workspace $env:WORKSPACE_ID ` + --logs '@.azure/.diagnostic-settings/logs.json' ` + --metrics '@.azure/.diagnostic-settings/metrics.json'; + + $env:ACTION_GROUP = az monitor action-group list -g $env:AZURE_GROUP_LOGS --query [0].[id] -o tsv; + + az monitor metrics alert create ` + --name "High CPU Usage" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg cpu_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when CPU usage is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Memory/Worker Usage" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg workers_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when worker/session usage is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Number Of Connections" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "total connection_successful > 100" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when the number of successful connections exceeds 100 in 5 minutes."; + + az monitor metrics alert create ` + --name "High Storage IO" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg io_consumption_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when Storage IO consumption is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Storage Percent" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg storage_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when Storage usage exceeds 80% for 5 minutes."; + + az sql db str-policy set ` + -g $env:AZURE_GROUP_DATABASE ` + -s $env:SQL_SERVER_NAME ` + -n $env:SQL_NAME ` + --retention-days $env:SQL_BACKUP_RETENTION; + + if ($LastExitCode -ne 0) + { + throw "error"; + }; + }; + - name: Database Migration & User shell: pwsh run: | diff --git a/Api.Data.SqlServer/.azure/.diagnostic-settings/logs.json b/Api.Data.SqlServer/.azure/.diagnostic-settings/logs.json new file mode 100644 index 00000000..10c3505a --- /dev/null +++ b/Api.Data.SqlServer/.azure/.diagnostic-settings/logs.json @@ -0,0 +1,10 @@ +[ + { + "category": "QueryStoreRuntimeStatistics", + "enabled": true + }, + { + "category": "SQLSecurityAuditEvents", + "enabled": true + } +] \ No newline at end of file diff --git a/Api.Data.SqlServer/.azure/.diagnostic-settings/metrics.json b/Api.Data.SqlServer/.azure/.diagnostic-settings/metrics.json new file mode 100644 index 00000000..551d2f13 --- /dev/null +++ b/Api.Data.SqlServer/.azure/.diagnostic-settings/metrics.json @@ -0,0 +1,7 @@ +[ + { + "category": "AllMetrics", + "enabled": true, + "timeGrain": "PT1M" + } +] \ No newline at end of file diff --git a/Api.Data.SqlServer/.github/workflows/build-and-deploy.yml b/Api.Data.SqlServer/.github/workflows/build-and-deploy.yml index 20874458..1fde61c9 100644 --- a/Api.Data.SqlServer/.github/workflows/build-and-deploy.yml +++ b/Api.Data.SqlServer/.github/workflows/build-and-deploy.yml @@ -18,6 +18,7 @@ env: AZURE_GROUP_KUBERNETES: ${{ vars.AZURE_RESOURCE_GROUP_KUBERNETES }} AZURE_GROUP_DELIVERY: ${{ vars.AZURE_RESOURCE_GROUP_DELIVERY }} AZURE_GROUP_DATABASE : ${{ vars.AZURE_RESOURCE_GROUP_DATABASE }} + AZURE_GROUP_LOGS: ${{ vars.AZURE_RESOURCE_GROUP_LOGS }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} @@ -37,6 +38,10 @@ env: KUBERNETES_CPU_LIMIT: 600m KUBERNETES_CPU_SCALING: 180 ASPNETCORE_ENVIRONMENT: ${{ github.ref == 'refs/heads/main' && 'Production' || 'Staging' }} + SQL_SERVICE_OBJECTIVE: GP_Gen5_2 + SQL_EDITION: GeneralPurpose + SQL_MAX_SIZE: 64GB + SQL_BACKUP_RETENTION: 35 SQL_NAME: nanoDb SQL_USER: api-data-sqlserver-user SQL_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} @@ -124,6 +129,112 @@ jobs: echo "CONTAINER_REGISTRY_HOST=$env:CONTAINER_REGISTRY_HOST" >> $env:GITHUB_ENV; + - name: Create Database + shell: pwsh + run: | + $env:SQL_SERVER_NAME = az sql server list -g $env:AZURE_GROUP_DATABASE --query "[0].name" -o tsv; + $env:SQL_DB_EXISTS = az sql db show -g $env:AZURE_GROUP_DATABASE -s $env:SQL_SERVER_NAME -n $env:SQL_NAME --query name -o tsv 2>$null; + + if (-not $env:SQL_DB_EXISTS) + { + az sql db create ` + -n $env:SQL_NAME ` + -s $env:SQL_SERVER_NAME ` + -g $env:AZURE_GROUP_DATABASE ` + --edition $env:SQL_EDITION ` + --service-objective $env:SQL_SERVICE_OBJECTIVE ` + --max-size $env:SQL_MAX_SIZE ` + --backup-storage-redundancy Geo ` + --zone-redundant true; + + $env:MAINTENANCE_CONFIG_ID = az maintenance public-configuration list --query "[?name=='SQL_Default_1'].id" -o tsv; + + az sql db update ` + -n $env:SQL_NAME ` + -s $env:SQL_SERVER_NAME ` + -g $env:AZURE_GROUP_DATABASE ` + --maint-config-id $env:MAINTENANCE_CONFIG_ID; + + $env:DIAGNOSTIC_SETTINGS_NAME = "diagnostics-" + $env:SQL_NAME; + $env:WORKSPACE_ID = az monitor log-analytics workspace list -g $env:AZURE_GROUP_LOGS --query [0].[id] -o tsv; + $env:SQLDB_ID = az sql db show -g $env:AZURE_GROUP_DATABASE -s $env:SQL_SERVER_NAME -n $env:SQL_NAME --query id -o tsv; + + az monitor diagnostic-settings create ` + --name $env:DIAGNOSTIC_SETTINGS_NAME ` + --resource $env:SQLDB_ID ` + --workspace $env:WORKSPACE_ID ` + --logs '@.azure/.diagnostic-settings/logs.json' ` + --metrics '@.azure/.diagnostic-settings/metrics.json'; + + $env:ACTION_GROUP = az monitor action-group list -g $env:AZURE_GROUP_LOGS --query [0].[id] -o tsv; + + az monitor metrics alert create ` + --name "High CPU Usage" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg cpu_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when CPU usage is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Memory/Worker Usage" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg workers_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when worker/session usage is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Number Of Connections" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "total connection_successful > 100" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when the number of successful connections exceeds 100 in 5 minutes."; + + az monitor metrics alert create ` + --name "High Storage IO" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg io_consumption_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when Storage IO consumption is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Storage Percent" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg storage_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when Storage usage exceeds 80% for 5 minutes."; + + az sql db str-policy set ` + -g $env:AZURE_GROUP_DATABASE ` + -s $env:SQL_SERVER_NAME ` + -n $env:SQL_NAME ` + --retention-days $env:SQL_BACKUP_RETENTION; + + if ($LastExitCode -ne 0) + { + throw "error"; + }; + }; + - name: Database Migration & User shell: pwsh run: | diff --git a/Api.Data.SqlServer/README.md b/Api.Data.SqlServer/README.md index c84aff04..115a225f 100644 --- a/Api.Data.SqlServer/README.md +++ b/Api.Data.SqlServer/README.md @@ -145,13 +145,127 @@ Add the following environment variables to the `buid-and-deply.yml`. env: DOTNET_EF_TOOLS_VERSION: "10.0" AZURE_GROUP_DATABASE : ${{ vars.AZURE_RESOURCE_GROUP_DATABASE }} + AZURE_GROUP_LOGS: ${{ vars.AZURE_RESOURCE_GROUP_LOGS }} + SQL_EDITION: GeneralPurpose + SQL_SERVICE_OBJECTIVE: GP_Gen5_2 + SQL_MAX_SIZE: 64GB + SQL_BACKUP_RETENTION: 35 SQL_NAME: nanoDb SQL_USER: api-data-sqlserver-user - SQL_PASSWORD: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} - SQL_ADMIN_PASSWORD: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_SQL_ADMIN_PASSWORD || secrets.STAGING_SQL_ADMIN_PASSWORD }} + SQL_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} + SQL_ADMIN_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_ADMIN_PASSWORD || secrets.STAGING_SQL_ADMIN_PASSWORD }} ``` -Additionally, this step has been added to ensure database migrations are applied, and the application database user has been created before the application is deployed. +Additionally, two steps have been added: one to create the application's database if it doesn't already exist, and one to ensure database migrations are applied and the application +database user has been created before the application is deployed. + +```yaml +- name: Create Database + shell: pwsh + run: | + $env:SQL_SERVER_NAME = az sql server list -g $env:AZURE_GROUP_DATABASE --query "[0].name" -o tsv; + $env:SQL_DB_EXISTS = az sql db show -g $env:AZURE_GROUP_DATABASE -s $env:SQL_SERVER_NAME -n $env:SQL_NAME --query name -o tsv 2>$null; + + if (-not $env:SQL_DB_EXISTS) + { + az sql db create ` + -n $env:SQL_NAME ` + -s $env:SQL_SERVER_NAME ` + -g $env:AZURE_GROUP_DATABASE ` + --edition $env:SQL_EDITION ` + --service-objective $env:SQL_SERVICE_OBJECTIVE ` + --max-size $env:SQL_MAX_SIZE ` + --backup-storage-redundancy Geo ` + --zone-redundant true; + + $env:MAINTENANCE_CONFIG_ID = az maintenance public-configuration list --query "[?name=='SQL_Default_1'].id" -o tsv; + + az sql db update ` + -n $env:SQL_NAME ` + -s $env:SQL_SERVER_NAME ` + -g $env:AZURE_GROUP_DATABASE ` + --maint-config-id $env:MAINTENANCE_CONFIG_ID; + + $env:DIAGNOSTIC_SETTINGS_NAME = "diagnostics-" + $env:SQL_NAME; + $env:WORKSPACE_ID = az monitor log-analytics workspace list -g $env:AZURE_GROUP_LOGS --query [0].[id] -o tsv; + $env:SQLDB_ID = az sql db show -g $env:AZURE_GROUP_DATABASE -s $env:SQL_SERVER_NAME -n $env:SQL_NAME --query id -o tsv; + + az monitor diagnostic-settings create ` + --name $env:DIAGNOSTIC_SETTINGS_NAME ` + --resource $env:SQLDB_ID ` + --workspace $env:WORKSPACE_ID ` + --logs '@.azure/.diagnostic-settings/logs.json' ` + --metrics '@.azure/.diagnostic-settings/metrics.json'; + + $env:ACTION_GROUP = az monitor action-group list -g $env:AZURE_GROUP_LOGS --query [0].[id] -o tsv; + + az monitor metrics alert create ` + --name "High CPU Usage" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg cpu_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when CPU usage is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Memory/Worker Usage" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg workers_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when worker/session usage is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Number Of Connections" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "total connection_successful > 100" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when the number of successful connections exceeds 100 in 5 minutes."; + + az monitor metrics alert create ` + --name "High Storage IO" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg io_consumption_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when Storage IO consumption is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Storage Percent" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg storage_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when Storage usage exceeds 80% for 5 minutes."; + + az sql db str-policy set ` + -g $env:AZURE_GROUP_DATABASE ` + -s $env:SQL_SERVER_NAME ` + -n $env:SQL_NAME ` + --retention-days $env:SQL_BACKUP_RETENTION; + + if ($LastExitCode -ne 0) + { + throw "error"; + }; + }; +``` ```yaml - name: Database Migration & User diff --git a/Console.Data.MySql/README.md b/Console.Data.MySql/README.md index 6f1b01ad..4429a667 100644 --- a/Console.Data.MySql/README.md +++ b/Console.Data.MySql/README.md @@ -133,8 +133,8 @@ env: AZURE_GROUP_DATABASE : ${{ vars.AZURE_RESOURCE_GROUP_DATABASE }} SQL_NAME: nanoDb SQL_USER: api-data-mysql-user - SQL_PASSWORD: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} - SQL_ADMIN_PASSWORD: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_SQL_ADMIN_PASSWORD || secrets.STAGING_SQL_ADMIN_PASSWORD }} + SQL_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} + SQL_ADMIN_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_ADMIN_PASSWORD || secrets.STAGING_SQL_ADMIN_PASSWORD }} ``` Additionally, this step has been added to ensure database migrations are applied, and the application database user has been created before the application is deployed. diff --git a/Console.Data.PostgreSQL/README.md b/Console.Data.PostgreSQL/README.md index 3bca1759..85abd08a 100644 --- a/Console.Data.PostgreSQL/README.md +++ b/Console.Data.PostgreSQL/README.md @@ -129,8 +129,8 @@ env: AZURE_GROUP_DATABASE : ${{ vars.AZURE_RESOURCE_GROUP_DATABASE }} SQL_NAME: nanoDb SQL_USER: api-data-postgres-user - SQL_PASSWORD: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} - SQL_ADMIN_PASSWORD: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_SQL_ADMIN_PASSWORD || secrets.STAGING_SQL_ADMIN_PASSWORD }} + SQL_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} + SQL_ADMIN_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_ADMIN_PASSWORD || secrets.STAGING_SQL_ADMIN_PASSWORD }} ``` Additionally, this step has been added to ensure database migrations are applied, and the application database user has been created before the application is deployed. diff --git a/Console.Data.SqlServer/.azure/.diagnostic-settings/logs.json b/Console.Data.SqlServer/.azure/.diagnostic-settings/logs.json new file mode 100644 index 00000000..10c3505a --- /dev/null +++ b/Console.Data.SqlServer/.azure/.diagnostic-settings/logs.json @@ -0,0 +1,10 @@ +[ + { + "category": "QueryStoreRuntimeStatistics", + "enabled": true + }, + { + "category": "SQLSecurityAuditEvents", + "enabled": true + } +] \ No newline at end of file diff --git a/Console.Data.SqlServer/.azure/.diagnostic-settings/metrics.json b/Console.Data.SqlServer/.azure/.diagnostic-settings/metrics.json new file mode 100644 index 00000000..551d2f13 --- /dev/null +++ b/Console.Data.SqlServer/.azure/.diagnostic-settings/metrics.json @@ -0,0 +1,7 @@ +[ + { + "category": "AllMetrics", + "enabled": true, + "timeGrain": "PT1M" + } +] \ No newline at end of file diff --git a/Console.Data.SqlServer/.github/workflows/build-and-deploy.yml b/Console.Data.SqlServer/.github/workflows/build-and-deploy.yml index fa8a17ac..46a6ad5e 100644 --- a/Console.Data.SqlServer/.github/workflows/build-and-deploy.yml +++ b/Console.Data.SqlServer/.github/workflows/build-and-deploy.yml @@ -18,6 +18,7 @@ env: AZURE_GROUP_KUBERNETES: ${{ vars.AZURE_RESOURCE_GROUP_KUBERNETES }} AZURE_GROUP_DELIVERY: ${{ vars.AZURE_RESOURCE_GROUP_DELIVERY }} AZURE_GROUP_DATABASE : ${{ vars.AZURE_RESOURCE_GROUP_DATABASE }} + AZURE_GROUP_LOGS: ${{ vars.AZURE_RESOURCE_GROUP_LOGS }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} @@ -33,6 +34,10 @@ env: KUBERNETES_CPU_REQUEST: 50m KUBERNETES_CPU_LIMIT: 150m KUBERNETES_CRONJOB_SCHEDULE: "0 * * * *" + SQL_SERVICE_OBJECTIVE: GP_Gen5_2 + SQL_EDITION: GeneralPurpose + SQL_MAX_SIZE: 64GB + SQL_BACKUP_RETENTION: 35 SQL_NAME: nanoDb SQL_USER: api-data-sqlserver-user SQL_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} @@ -109,6 +114,112 @@ jobs: echo "CONTAINER_REGISTRY_HOST=$env:CONTAINER_REGISTRY_HOST" >> $env:GITHUB_ENV; + - name: Create Database + shell: pwsh + run: | + $env:SQL_SERVER_NAME = az sql server list -g $env:AZURE_GROUP_DATABASE --query "[0].name" -o tsv; + $env:SQL_DB_EXISTS = az sql db show -g $env:AZURE_GROUP_DATABASE -s $env:SQL_SERVER_NAME -n $env:SQL_NAME --query name -o tsv 2>$null; + + if (-not $env:SQL_DB_EXISTS) + { + az sql db create ` + -n $env:SQL_NAME ` + -s $env:SQL_SERVER_NAME ` + -g $env:AZURE_GROUP_DATABASE ` + --edition $env:SQL_EDITION ` + --service-objective $env:SQL_SERVICE_OBJECTIVE ` + --max-size $env:SQL_MAX_SIZE ` + --backup-storage-redundancy Geo ` + --zone-redundant true; + + $env:MAINTENANCE_CONFIG_ID = az maintenance public-configuration list --query "[?name=='SQL_Default_1'].id" -o tsv; + + az sql db update ` + -n $env:SQL_NAME ` + -s $env:SQL_SERVER_NAME ` + -g $env:AZURE_GROUP_DATABASE ` + --maint-config-id $env:MAINTENANCE_CONFIG_ID; + + $env:DIAGNOSTIC_SETTINGS_NAME = "diagnostics-" + $env:SQL_NAME; + $env:WORKSPACE_ID = az monitor log-analytics workspace list -g $env:AZURE_GROUP_LOGS --query [0].[id] -o tsv; + $env:SQLDB_ID = az sql db show -g $env:AZURE_GROUP_DATABASE -s $env:SQL_SERVER_NAME -n $env:SQL_NAME --query id -o tsv; + + az monitor diagnostic-settings create ` + --name $env:DIAGNOSTIC_SETTINGS_NAME ` + --resource $env:SQLDB_ID ` + --workspace $env:WORKSPACE_ID ` + --logs '@.azure/.diagnostic-settings/logs.json' ` + --metrics '@.azure/.diagnostic-settings/metrics.json'; + + $env:ACTION_GROUP = az monitor action-group list -g $env:AZURE_GROUP_LOGS --query [0].[id] -o tsv; + + az monitor metrics alert create ` + --name "High CPU Usage" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg cpu_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when CPU usage is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Memory/Worker Usage" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg workers_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when worker/session usage is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Number Of Connections" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "total connection_successful > 100" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when the number of successful connections exceeds 100 in 5 minutes."; + + az monitor metrics alert create ` + --name "High Storage IO" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg io_consumption_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when Storage IO consumption is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Storage Percent" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg storage_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when Storage usage exceeds 80% for 5 minutes."; + + az sql db str-policy set ` + -g $env:AZURE_GROUP_DATABASE ` + -s $env:SQL_SERVER_NAME ` + -n $env:SQL_NAME ` + --retention-days $env:SQL_BACKUP_RETENTION; + + if ($LastExitCode -ne 0) + { + throw "error"; + }; + }; + - name: Database Migration & User shell: pwsh run: | diff --git a/Console.Data.SqlServer/README.md b/Console.Data.SqlServer/README.md index d5b391da..055ff8c7 100644 --- a/Console.Data.SqlServer/README.md +++ b/Console.Data.SqlServer/README.md @@ -128,13 +128,127 @@ Add the following environment variables to the `buid-and-deply.yml`. env: DOTNET_EF_TOOLS_VERSION: "10.0" AZURE_GROUP_DATABASE : ${{ vars.AZURE_RESOURCE_GROUP_DATABASE }} + AZURE_GROUP_LOGS: ${{ vars.AZURE_RESOURCE_GROUP_LOGS }} + SQL_EDITION: GeneralPurpose + SQL_SERVICE_OBJECTIVE: GP_Gen5_2 + SQL_MAX_SIZE: 64GB + SQL_BACKUP_RETENTION: 35 SQL_NAME: nanoDb SQL_USER: api-data-sqlserver-user - SQL_PASSWORD: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} - SQL_ADMIN_PASSWORD: ${{ github.ref == 'refs/heads/master' && secrets.PRODUCTION_SQL_ADMIN_PASSWORD || secrets.STAGING_SQL_ADMIN_PASSWORD }} + SQL_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_NANO_DB_PASSWORD || secrets.STAGING_SQL_NANO_DB_PASSWORD }} + SQL_ADMIN_PASSWORD: ${{ github.ref == 'refs/heads/main' && secrets.PRODUCTION_SQL_ADMIN_PASSWORD || secrets.STAGING_SQL_ADMIN_PASSWORD }} ``` -Additionally, this step has been added to ensure database migrations are applied, and the application database user has been created before the application is deployed. +Additionally, two steps have been added: one to create the application's database if it doesn't already exist, and one to ensure database migrations are applied and the application +database user has been created before the application is deployed. + +```yaml +- name: Create Database + shell: pwsh + run: | + $env:SQL_SERVER_NAME = az sql server list -g $env:AZURE_GROUP_DATABASE --query "[0].name" -o tsv; + $env:SQL_DB_EXISTS = az sql db show -g $env:AZURE_GROUP_DATABASE -s $env:SQL_SERVER_NAME -n $env:SQL_NAME --query name -o tsv 2>$null; + + if (-not $env:SQL_DB_EXISTS) + { + az sql db create ` + -n $env:SQL_NAME ` + -s $env:SQL_SERVER_NAME ` + -g $env:AZURE_GROUP_DATABASE ` + --edition $env:SQL_EDITION ` + --service-objective $env:SQL_SERVICE_OBJECTIVE ` + --max-size $env:SQL_MAX_SIZE ` + --backup-storage-redundancy Geo ` + --zone-redundant true; + + $env:MAINTENANCE_CONFIG_ID = az maintenance public-configuration list --query "[?name=='SQL_Default_1'].id" -o tsv; + + az sql db update ` + -n $env:SQL_NAME ` + -s $env:SQL_SERVER_NAME ` + -g $env:AZURE_GROUP_DATABASE ` + --maint-config-id $env:MAINTENANCE_CONFIG_ID; + + $env:DIAGNOSTIC_SETTINGS_NAME = "diagnostics-" + $env:SQL_NAME; + $env:WORKSPACE_ID = az monitor log-analytics workspace list -g $env:AZURE_GROUP_LOGS --query [0].[id] -o tsv; + $env:SQLDB_ID = az sql db show -g $env:AZURE_GROUP_DATABASE -s $env:SQL_SERVER_NAME -n $env:SQL_NAME --query id -o tsv; + + az monitor diagnostic-settings create ` + --name $env:DIAGNOSTIC_SETTINGS_NAME ` + --resource $env:SQLDB_ID ` + --workspace $env:WORKSPACE_ID ` + --logs '@.azure/.diagnostic-settings/logs.json' ` + --metrics '@.azure/.diagnostic-settings/metrics.json'; + + $env:ACTION_GROUP = az monitor action-group list -g $env:AZURE_GROUP_LOGS --query [0].[id] -o tsv; + + az monitor metrics alert create ` + --name "High CPU Usage" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg cpu_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when CPU usage is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Memory/Worker Usage" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg workers_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when worker/session usage is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Number Of Connections" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "total connection_successful > 100" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when the number of successful connections exceeds 100 in 5 minutes."; + + az monitor metrics alert create ` + --name "High Storage IO" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg io_consumption_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when Storage IO consumption is above 80% for 5 minutes."; + + az monitor metrics alert create ` + --name "High Storage Percent" ` + --resource-group $env:AZURE_GROUP_DATABASE ` + --scopes $env:SQLDB_ID ` + --condition "avg storage_percent > 80" ` + --window-size PT5M ` + --evaluation-frequency PT1M ` + --action $env:ACTION_GROUP ` + --severity 2 ` + --description "Alert when Storage usage exceeds 80% for 5 minutes."; + + az sql db str-policy set ` + -g $env:AZURE_GROUP_DATABASE ` + -s $env:SQL_SERVER_NAME ` + -n $env:SQL_NAME ` + --retention-days $env:SQL_BACKUP_RETENTION; + + if ($LastExitCode -ne 0) + { + throw "error"; + }; + }; +``` ```yaml - name: Database Migration & User