diff --git a/Actions/Invoke-AlGoAction.ps1 b/Actions/Invoke-AlGoAction.ps1 index 854cf9021..be36f030a 100644 --- a/Actions/Invoke-AlGoAction.ps1 +++ b/Actions/Invoke-AlGoAction.ps1 @@ -22,12 +22,14 @@ try { if (-not $SkipTelemetry) { $AdditionalData["ActionDuration"] = (((Get-Date) - $startTime).TotalSeconds).ToString() + $AdditionalData["ActionConclusion"] = "Success" Trace-Information -ActionName $ActionName -AdditionalData $AdditionalData } } catch { if (-not $SkipTelemetry) { $AdditionalData["ActionDuration"] = (((Get-Date) - $startTime).TotalSeconds).ToString() + $AdditionalData["ActionConclusion"] = "Failure" Trace-Exception -ActionName $ActionName -ErrorRecord $_ -AdditionalData $AdditionalData } diff --git a/Actions/TelemetryHelper.psm1 b/Actions/TelemetryHelper.psm1 index c79cbc66c..96d9ef49b 100644 --- a/Actions/TelemetryHelper.psm1 +++ b/Actions/TelemetryHelper.psm1 @@ -45,6 +45,56 @@ function Get-ApplicationInsightsTelemetryClient($TelemetryConnectionString) } #endregion +<# + .SYNOPSIS + Gets the GitHub hosting platform for telemetry. +#> +function Get-GitHubHostingType { + $hostingType = 'Unknown' + $serverUri = $null + if ($ENV:GITHUB_SERVER_URL -and + [Uri]::TryCreate($ENV:GITHUB_SERVER_URL, [UriKind]::Absolute, [ref]$serverUri) -and + $serverUri.Scheme -in @('https', 'http') -and $serverUri.Host) { + if ($serverUri.Host -eq 'github.com') { + $hostingType = 'GitHub.com' + } + elseif ($serverUri.Host.EndsWith('.ghe.com', [StringComparison]::OrdinalIgnoreCase)) { + $hostingType = 'GHEC' + } + else { + $hostingType = 'GHES' + } + } + else { + Write-Host "::Warning::Unable to determine GitHub hosting type from GITHUB_SERVER_URL." + } + return $hostingType +} + +<# + .SYNOPSIS + Gets the workflow repository's fork status for telemetry. +#> +function Get-RepositoryIsFork { + $repositoryIsFork = 'Unknown' + try { + if (-not $ENV:GITHUB_EVENT_PATH) { + throw 'GITHUB_EVENT_PATH is unavailable.' + } + $githubEvent = Get-Content -LiteralPath $ENV:GITHUB_EVENT_PATH -Raw -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -recurse + if ($githubEvent -isnot [System.Collections.IDictionary] -or + $githubEvent.repository -isnot [System.Collections.IDictionary] -or + $githubEvent.repository.fork -isnot [bool]) { + throw 'The event does not contain a boolean repository.fork.' + } + $repositoryIsFork = $githubEvent.repository.fork.ToString().ToLowerInvariant() + } + catch { + Write-Host "::Warning::Unable to determine repository fork status from GITHUB_EVENT_PATH. RepositoryIsFork will be Unknown." + } + return $repositoryIsFork +} + function AddTelemetryEvent() { param( @@ -77,6 +127,8 @@ function AddTelemetryEvent() ### Add GitHub Repository information Add-TelemetryProperty -Hashtable $Data -Key 'Repository' -Value $ENV:GITHUB_REPOSITORY_ID Add-TelemetryProperty -Hashtable $Data -Key 'RepositoryOwnerID' -Value $ENV:GITHUB_REPOSITORY_OWNER_ID + Add-TelemetryProperty -Hashtable $Data -Key 'GitHubHostingType' -Value (Get-GitHubHostingType) + Add-TelemetryProperty -Hashtable $Data -Key 'RepositoryIsFork' -Value (Get-RepositoryIsFork) $repoSettings = ReadSettings if ($repoSettings.microsoftTelemetryConnectionString -ne '') { @@ -135,6 +187,7 @@ function Trace-Information() { $Message = "AL-Go action ran: $ActionName" } + Add-TelemetryProperty -Hashtable $AdditionalData -Key 'ActionName' -Value $ActionName AddTelemetryEvent -Message $Message -Severity 'Information' -Data $AdditionalData } @@ -206,6 +259,7 @@ function Trace-Exception() { if (-not $Message) { $Message = "AL-Go action failed: $ActionName" } + Add-TelemetryProperty -Hashtable $AdditionalData -Key 'ActionName' -Value $ActionName AddTelemetryEvent -Message $Message -Severity 'Error' -Data $AdditionalData } diff --git a/RELEASENOTES.md b/RELEASENOTES.md index c4417fda5..9afd7be37 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,3 +1,9 @@ +### AL-Go Telemetry Enhancements + +AL-Go now includes `RepositoryIsFork` and `GitHubHostingType` in Microsoft and partner telemetry. These identify whether the workflow repository is a fork and whether it runs on GitHub.com, GitHub Enterprise Cloud with data residency (`*.ghe.com`), or GitHub Enterprise Server. GitHub.com includes Enterprise Cloud customers hosted there; this classification does not identify subscription plans or send server hostnames. Unavailable metadata is reported as `Unknown`. + +Action telemetry now includes structured `ActionName` and `ActionConclusion` dimensions. The action wrapper reports `Success` or `Failure`, allowing consumers to group action outcomes without parsing message text. Handled error events do not independently mark the action as failed. + ### Allow pre-release packages as NuGet dependencies AL-Go now supports including pre-release versions of Business Central app packages from NuGet feeds. To enable this, append `-allowPrerelease` to the `nuGetFeedSelectMode` setting in your project configuration. For example, `LatestMatching-allowPrerelease` will select the latest matching version of the package, including pre-release versions. diff --git a/Scenarios/EnablingTelemetry.md b/Scenarios/EnablingTelemetry.md index 74e460003..72881ef22 100644 --- a/Scenarios/EnablingTelemetry.md +++ b/Scenarios/EnablingTelemetry.md @@ -133,6 +133,12 @@ AL-Go logs four different types of telemetry events: AL-Go action ran/failed and | RunNumber | The Run Number | | RunAttempt | The attempt number | | Repository | The repository Id | +| RepositoryIsFork | `true` or `false`, from the workflow event's `repository.fork`; `Unknown` if unavailable | +| GitHubHostingType | `GitHub.com`, `GHEC` (Enterprise Cloud with data residency on `*.ghe.com`), or `GHES` (other GitHub server hosts), based on `GITHUB_SERVER_URL`; `Unknown` if missing or invalid | + +`RepositoryIsFork` describes the repository running the workflow, not the source repository of an incoming pull request. A repository created from an AL-Go template is not a fork unless GitHub marks it as one. + +These two dimensions are included in both Microsoft and partner telemetry, subject to the existing telemetry settings. The hosting classification does not send the server hostname or URL. `GitHub.com` includes Enterprise Cloud customers hosted on github.com; the hostname cannot distinguish their subscription plan. Missing or invalid metadata produces a warning and an `Unknown` value without preventing the event from being logged. ### AL-Go action ran @@ -144,6 +150,8 @@ Additional Dimensions: | Dimension | Description | |-----------|-------------| | ActionDuration | The duration of the action | +| ActionName | The name of the AL-Go action | +| ActionConclusion | `Success` when the action wrapper completes successfully | ### AL-Go action failed @@ -156,6 +164,10 @@ Additional Dimensions: | Dimension | Description | |-----------|-------------| | ErrorMessage | The error message thrown | +| ActionName | The name of the AL-Go action | +| ActionConclusion | `Failure` when the action wrapper catches a terminating error | + +`ActionName` is included when telemetry is logged with an action name. `ActionConclusion` is emitted by the action wrapper, not by individual exception logging calls: a handled error can produce an error event without failing the action. To count failed action executions, filter on `ActionConclusion == "Failure"`. Message-only events, such as workflow summaries and test results, do not automatically receive these action dimensions. Both dimensions are included in Microsoft and partner telemetry, subject to the existing telemetry settings. ### AL-Go workflow ran diff --git a/Tests/TelemetryHelper.Test.ps1 b/Tests/TelemetryHelper.Test.ps1 new file mode 100644 index 000000000..d8c16cdb2 --- /dev/null +++ b/Tests/TelemetryHelper.Test.ps1 @@ -0,0 +1,242 @@ +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', '', Justification = 'Test case parameters are used in Pester mock parameter filters')] +param() + +$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 + +Import-Module (Join-Path $PSScriptRoot '../Actions/TelemetryHelper.psm1') -Force -DisableNameChecking + +Describe 'Structured action telemetry' { + InModuleScope TelemetryHelper { + BeforeAll { + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'wrapperPath', Justification = 'Used by tests in separate Pester scriptblocks')] + $wrapperPath = Join-Path $PSScriptRoot '../Actions/Invoke-AlGoAction.ps1' -Resolve + } + + BeforeEach { + $additionalData = [System.Collections.Generic.Dictionary[string, string]]::new() + $additionalData.Add('CustomProperty', 'preserved') + Mock AddTelemetryEvent {} + Mock Write-Host {} + } + + It 'Adds the action name without inferring a conclusion from ' -TestCases @( + @{ command = 'Trace-Information'; expectedMessage = 'AL-Go action ran: Test'; expectedSeverity = 'Information' } + @{ command = 'Trace-Exception'; expectedMessage = 'AL-Go action failed: Test'; expectedSeverity = 'Error' } + ) { + param($command, $expectedMessage, $expectedSeverity) + & $command -ActionName 'Test' -AdditionalData $additionalData + + Assert-MockCalled AddTelemetryEvent -Times 1 -Exactly -ParameterFilter { + $Data['ActionName'] -eq 'Test' -and + -not $Data.ContainsKey('ActionConclusion') -and + $Data['CustomProperty'] -eq 'preserved' -and + $Message -eq $expectedMessage -and $Severity -eq $expectedSeverity + } + } + + It 'Does not add action dimensions to message-only events' -TestCases @( + @{ command = 'Trace-Information' } + @{ command = 'Trace-Exception' } + @{ command = 'Trace-Warning' } + ) { + param($command) + & $command -Message 'Custom message' + + Assert-MockCalled AddTelemetryEvent -Times 1 -Exactly -ParameterFilter { + -not $Data.ContainsKey('ActionName') -and + -not $Data.ContainsKey('ActionConclusion') -and + $Message -eq 'Custom message' + } + } + + It 'Reports successful completion with the existing duration and additional data' { + & $wrapperPath -ActionName 'Test' -Action {} -AdditionalData $additionalData + + Assert-MockCalled AddTelemetryEvent -Times 1 -Exactly -ParameterFilter { + $Data['ActionName'] -eq 'Test' -and + $Data['ActionConclusion'] -eq 'Success' -and + $Data.ContainsKey('ActionDuration') -and [double]$Data['ActionDuration'] -ge 0 -and + $Data['CustomProperty'] -eq 'preserved' + } + } + + It 'Reports a terminating failure and preserves the error and exit code' { + & $wrapperPath -ActionName 'Test' -Action { throw 'Action failed' } + + $LASTEXITCODE | Should -Be 1 + Assert-MockCalled AddTelemetryEvent -Times 1 -Exactly -ParameterFilter { + $Data['ActionName'] -eq 'Test' -and + $Data['ActionConclusion'] -eq 'Failure' -and + $Data['ErrorMessage'] -eq 'Action failed' -and + $Data.ContainsKey('ActionDuration') -and [double]$Data['ActionDuration'] -ge 0 + } + } + + It 'Does not count a handled error as a failed action execution' { + & $wrapperPath -ActionName 'Test' -Action { + try { + throw 'Handled error' + } + catch { + Trace-Exception -ActionName 'Test' -ErrorRecord $_ + } + } + + Assert-MockCalled AddTelemetryEvent -Times 2 -Exactly + Assert-MockCalled AddTelemetryEvent -Times 1 -Exactly -ParameterFilter { + $Data['ActionName'] -eq 'Test' -and + $Data['ErrorMessage'] -eq 'Handled error' -and + -not $Data.ContainsKey('ActionConclusion') + } + Assert-MockCalled AddTelemetryEvent -Times 1 -Exactly -ParameterFilter { + $Data['ActionConclusion'] -eq 'Success' + } + } + + It 'Honors SkipTelemetry when the action fails: ' -TestCases @( + @{ fail = $false } + @{ fail = $true } + ) { + param($fail) + & $wrapperPath -ActionName 'Test' -SkipTelemetry -Action { + if ($fail) { throw 'Action failed' } + } + + Assert-MockCalled AddTelemetryEvent -Times 0 -Exactly + if ($fail) { + $LASTEXITCODE | Should -Be 1 + } + } + } +} + +Describe 'Repository environment telemetry' { + InModuleScope TelemetryHelper { + BeforeEach { + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'savedServerUrl', Justification = 'Restored in AfterEach')] + $savedServerUrl = $env:GITHUB_SERVER_URL + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'savedEventPath', Justification = 'Restored in AfterEach')] + $savedEventPath = $env:GITHUB_EVENT_PATH + $env:GITHUB_SERVER_URL = 'https://github.com' + $env:GITHUB_EVENT_PATH = Join-Path $TestDrive 'event.json' + '{"repository":{"fork":false}}' | Set-Content -LiteralPath $env:GITHUB_EVENT_PATH -Encoding UTF8 + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'data', Justification = 'Used by tests in separate Pester scriptblocks')] + $data = [System.Collections.Generic.Dictionary[string, string]]::new() + Mock Write-Host {} + } + + AfterEach { + $env:GITHUB_SERVER_URL = $savedServerUrl + $env:GITHUB_EVENT_PATH = $savedEventPath + } + + It 'Classifies as ' -TestCases @( + @{ url = 'https://github.com'; expected = 'GitHub.com' } + @{ url = 'https://GITHUB.COM/'; expected = 'GitHub.com' } + @{ url = 'https://contoso.ghe.com'; expected = 'GHEC' } + @{ url = 'https://CONTOSO.GHE.COM/'; expected = 'GHEC' } + @{ url = 'https://github.contoso.com'; expected = 'GHES' } + @{ url = 'https://github.contoso.com:8443/'; expected = 'GHES' } + @{ url = 'https://notghe.com'; expected = 'GHES' } + @{ url = 'https://contoso.ghe.com.example.org'; expected = 'GHES' } + @{ url = 'https://github.com.example.org'; expected = 'GHES' } + ) { + param($url, $expected) + $env:GITHUB_SERVER_URL = $url + + Get-GitHubHostingType | Should -Be $expected + Assert-MockCalled Write-Host -Times 0 + } + + It 'Reports Unknown for invalid server URL ' -TestCases @( + @{ url = '' } + @{ url = 'not a URL' } + @{ url = 'github.com' } + @{ url = 'file:///tmp/github.com' } + ) { + param($url) + $env:GITHUB_SERVER_URL = $url + + Get-GitHubHostingType | Should -Be 'Unknown' + Assert-MockCalled Write-Host -Times 1 -ParameterFilter { $Object -like '::Warning::*GITHUB_SERVER_URL*' } + } + + It 'Uses the workflow repository fork flag, not the pull request source' -TestCases @( + @{ repositoryFork = $true; headFork = $false; expected = 'true' } + @{ repositoryFork = $false; headFork = $true; expected = 'false' } + ) { + param($repositoryFork, $headFork, $expected) + @{ + repository = @{ fork = $repositoryFork } + pull_request = @{ head = @{ repo = @{ fork = $headFork } } } + } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $env:GITHUB_EVENT_PATH -Encoding UTF8 + + Get-RepositoryIsFork | Should -Be $expected + Assert-MockCalled Write-Host -Times 0 + } + + It 'Reports Unknown for unavailable or invalid fork metadata: ' -TestCases @( + @{ json = '{}' } + @{ json = '{"repository":{}}' } + @{ json = '{"repository":{"fork":null}}' } + @{ json = '{"repository":{"fork":"false"}}' } + @{ json = '{"repository":{"fork":0}}' } + @{ json = '{"repository":null}' } + @{ json = 'null' } + @{ json = '{invalid json' } + ) { + param($json) + $json | Set-Content -LiteralPath $env:GITHUB_EVENT_PATH -Encoding UTF8 + + Get-RepositoryIsFork | Should -Be 'Unknown' + Assert-MockCalled Write-Host -Times 1 -ParameterFilter { $Object -like '::Warning::*GITHUB_EVENT_PATH*' } + } + + It 'Reports Unknown when the event file is unavailable' -TestCases @( + @{ missingPath = $true } + @{ missingPath = $false } + ) { + param($missingPath) + if ($missingPath) { + $env:GITHUB_EVENT_PATH = $null + } + else { + $env:GITHUB_EVENT_PATH = Join-Path $TestDrive 'missing.json' + } + + Get-RepositoryIsFork | Should -Be 'Unknown' + Assert-MockCalled Write-Host -Times 1 -ParameterFilter { $Object -like '::Warning::*GITHUB_EVENT_PATH*' } + } + + It 'Adds repository dimensions to events without replacing existing data' { + Mock ReadSettings { return @{ microsoftTelemetryConnectionString = ''; partnerTelemetryConnectionString = '' } } + Mock Get-Module { return $null } -ParameterFilter { $Name -eq 'BcContainerHelper' } + Mock Get-ApplicationInsightsTelemetryClient { throw 'Telemetry must not be sent when disabled.' } + $data.Add('ActionDuration', '42') + + AddTelemetryEvent -Message 'AL-Go action ran: Test' -Data $data + + $data['RepositoryIsFork'] | Should -Be 'false' + $data['GitHubHostingType'] | Should -Be 'GitHub.com' + $data['PowerShellVersion'] | Should -Be $PSVersionTable.PSVersion.ToString() + $data['ActionDuration'] | Should -Be '42' + Assert-MockCalled ReadSettings -Times 1 -Exactly + Assert-MockCalled Get-ApplicationInsightsTelemetryClient -Times 0 + } + + It 'Continues logging existing telemetry when new metadata is invalid' { + Mock ReadSettings { return @{ microsoftTelemetryConnectionString = ''; partnerTelemetryConnectionString = '' } } + Mock Get-Module { return $null } -ParameterFilter { $Name -eq 'BcContainerHelper' } + $env:GITHUB_SERVER_URL = 'invalid' + '{invalid json' | Set-Content -LiteralPath $env:GITHUB_EVENT_PATH -Encoding UTF8 + + AddTelemetryEvent -Message 'AL-Go workflow ran' -Data $data + + $data['RepositoryIsFork'] | Should -Be 'Unknown' + $data['GitHubHostingType'] | Should -Be 'Unknown' + $data['PowerShellVersion'] | Should -Be $PSVersionTable.PSVersion.ToString() + Assert-MockCalled ReadSettings -Times 1 -Exactly + Assert-MockCalled Write-Host -Times 0 -ParameterFilter { $Object -like 'Failed to log telemetry event:*' } + } + } +}