diff --git a/.github/workflows/test-compile-ios.yml b/.github/workflows/test-compile-ios.yml index 84a6d347b..c10318f1f 100644 --- a/.github/workflows/test-compile-ios.yml +++ b/.github/workflows/test-compile-ios.yml @@ -82,6 +82,8 @@ jobs: - name: iOS test run: ./scripts/compile-xcode-project.ps1 timeout-minutes: 20 + env: + SENTRY_HTTP_MAX_RETRIES: 20 - name: Assert symbols and sources were uploaded run: ./test/Scripts.Integration.Test/assert-symbol-upload.ps1 -LogPath samples/IntegrationTest/Build/sentry-symbols-upload.log diff --git a/scripts/unity-utils.ps1 b/scripts/unity-utils.ps1 index 0573b0525..1ee54531a 100644 --- a/scripts/unity-utils.ps1 +++ b/scripts/unity-utils.ps1 @@ -25,6 +25,12 @@ function GetUnityRetrySignature([string] $unityPath, [string[]] $arguments, [int return 'killed' } + # Unity gives the LicensingClient it just launched 5s to open its IPC channel. The next launch finds it running. + if ($exitCode -eq 199 -and ($stdout -match "IPC channel to LicensingClient doesn't exist; aborting")) + { + return 'licensing-ipc' + } + # A crash while running tests may be an SDK bug. if ($arguments -contains '-runTests') { diff --git a/test/IntegrationTest/Integration.Tests.ps1 b/test/IntegrationTest/Integration.Tests.ps1 index dafec356d..6f2e815d3 100644 --- a/test/IntegrationTest/Integration.Tests.ps1 +++ b/test/IntegrationTest/Integration.Tests.ps1 @@ -280,21 +280,38 @@ BeforeAll { # Save result to JSON file $runResult | ConvertTo-Json -Depth 5 | Out-File -FilePath (Get-OutputFilePath "${Action}-result.json") - # Launch app again to ensure crash report is sent - if ($Action -eq "crash-capture") { - Write-Host "Running crash-send to ensure crash report is sent..." + # A shutdown flush that times out leaves the event in the offline cache until the next launch. + # Relaunch once per job; a second timeout points at the SDK rather than a slow first request. + $flushTimedOut = $Action -ne "crash-capture" -and [bool]($runResult.Output -match "Timeout when trying to flush queue") + if ($flushTimedOut -and $script:FlushTimeoutRelaunched) { + Write-Host "::warning::$Action quit before the SDK flushed its queue again (signature: shutdown-flush-timeout). Not relaunching a second time, its event may stay in the offline cache." + $flushTimedOut = $false + } + elseif ($flushTimedOut) { + $script:FlushTimeoutRelaunched = $true + Write-Host "::warning::$Action quit before the SDK flushed its queue (signature: shutdown-flush-timeout). Relaunching to send the cached envelopes." + } + # crash-send starts the SDK, which sends the cached crash report and envelopes, and flushes before quitting + if ($Action -eq "crash-capture" -or $flushTimedOut) { + $sendName = if ($flushTimedOut) { "crash-send-after-$Action" } else { "crash-send" } + Write-Host "Running crash-send to ensure cached envelopes are sent..." + + $androidLaunches = $script:AndroidUninitialisedLaunches $sendResult = Invoke-App -Action "crash-send" + if ($flushTimedOut) { + $script:AndroidUninitialisedLaunches = $androidLaunches + } if ($script:Platform -eq "Xbox") { $sendResult = Get-XboxLogOutput -RunResult $sendResult -Action "crash-send" } # Save crash-send result to JSON for debugging - $sendResult | ConvertTo-Json -Depth 5 | Out-File -FilePath (Get-OutputFilePath "crash-send-result.json") + $sendResult | ConvertTo-Json -Depth 5 | Out-File -FilePath (Get-OutputFilePath "$sendName-result.json") # Print crash-send output - Write-Host "::group::App output (crash-send)" + Write-Host "::group::App output ($sendName)" $sendResult.Output | ForEach-Object { Write-Host $_ } Write-Host "::endgroup::" @@ -311,6 +328,7 @@ BeforeAll { } $script:AndroidUninitialisedLaunches = 0 + $script:FlushTimeoutRelaunched = $false $script:Platform = $env:SENTRY_TEST_PLATFORM if ([string]::IsNullOrEmpty($script:Platform)) { diff --git a/test/Scripts.Integration.Test/Scripts/CliConfiguration.cs b/test/Scripts.Integration.Test/Scripts/CliConfiguration.cs index 89b72cd1f..d92b29d9a 100644 --- a/test/Scripts.Integration.Test/Scripts/CliConfiguration.cs +++ b/test/Scripts.Integration.Test/Scripts/CliConfiguration.cs @@ -13,6 +13,12 @@ public override void Configure(SentryCliOptions cliOptions) cliOptions.UploadSources = cliOptions.UploadSymbols; cliOptions.Auth = authToken; + // sentry-cli's default 5 retries give up within ~30s of sentry.io answering 503/504. Reaches the uploads the Editor spawns; iOS sets it on xcodebuild. + if (Environment.GetEnvironmentVariable("SENTRY_HTTP_MAX_RETRIES") is null) + { + Environment.SetEnvironmentVariable("SENTRY_HTTP_MAX_RETRIES", "20"); + } + cliOptions.Organization = "sentry-sdks"; cliOptions.Project = "sentry-unity-integration-tests"; diff --git a/test/Scripts.Integration.Test/assert-symbol-upload.ps1 b/test/Scripts.Integration.Test/assert-symbol-upload.ps1 index e0f6b8f42..ae5e1682f 100644 --- a/test/Scripts.Integration.Test/assert-symbol-upload.ps1 +++ b/test/Scripts.Integration.Test/assert-symbol-upload.ps1 @@ -69,4 +69,11 @@ if (-not $uploadSucceeded) "Expected 'Uploaded N missing debug information files' or 'Nothing to upload'." } +$retries = [regex]::Matches($log, "retry number \d+ failed due to request failed with retryable status code (\d+)") +if ($retries.Count -gt 0) +{ + $codes = ($retries | ForEach-Object { $_.Groups[1].Value } | Sort-Object -Unique) -join ', ' + Write-Host "::warning::sentry-cli retried $($retries.Count) upload request(s) after HTTP $codes (signature: sentry-cli-http-retry)." +} + Write-Host "Symbol-upload assertion passed: sentry-cli uploaded debug symbols and sources ('$LogPath')." -ForegroundColor Green