ci: recover from LicensingClient IPC timeouts, sentry-cli 5xx and flush timeouts

.github/workflows/test-compile-ios.yml

@@ -82,6 +82,8 @@ jobs:
       - name: iOS test
         run: ./scripts/compile-xcode-project.ps1
         timeout-minutes: 20
+        env:
+          SENTRY_HTTP_MAX_RETRIES: 20
 
       - name: Assert symbols and sources were uploaded
         run: ./test/Scripts.Integration.Test/assert-symbol-upload.ps1 -LogPath samples/IntegrationTest/Build/sentry-symbols-upload.log

scripts/unity-utils.ps1

@@ -25,6 +25,12 @@ function GetUnityRetrySignature([string] $unityPath, [string[]] $arguments, [int
         return 'killed'
     }
 
+    # Unity gives the LicensingClient it just launched 5s to open its IPC channel. The next launch finds it running.
+    if ($exitCode -eq 199 -and ($stdout -match "IPC channel to LicensingClient doesn't exist; aborting"))
+    {
+        return 'licensing-ipc'
+    }
+
     # A crash while running tests may be an SDK bug.
     if ($arguments -contains '-runTests')
     {

test/IntegrationTest/Integration.Tests.ps1

@@ -280,21 +280,38 @@ BeforeAll {
         # Save result to JSON file
         $runResult | ConvertTo-Json -Depth 5 | Out-File -FilePath (Get-OutputFilePath "${Action}-result.json")
 
-        # Launch app again to ensure crash report is sent
-        if ($Action -eq "crash-capture") {
-            Write-Host "Running crash-send to ensure crash report is sent..."
+        # A shutdown flush that times out leaves the event in the offline cache until the next launch.
+        # Relaunch once per job; a second timeout points at the SDK rather than a slow first request.
+        $flushTimedOut = $Action -ne "crash-capture" -and [bool]($runResult.Output -match "Timeout when trying to flush queue")
+        if ($flushTimedOut -and $script:FlushTimeoutRelaunched) {
+            Write-Host "::warning::$Action quit before the SDK flushed its queue again (signature: shutdown-flush-timeout). Not relaunching a second time, its event may stay in the offline cache."
+            $flushTimedOut = $false
+        }
+        elseif ($flushTimedOut) {
+            $script:FlushTimeoutRelaunched = $true
+            Write-Host "::warning::$Action quit before the SDK flushed its queue (signature: shutdown-flush-timeout). Relaunching to send the cached envelopes."
+        }
 
+        # crash-send starts the SDK, which sends the cached crash report and envelopes, and flushes before quitting
+        if ($Action -eq "crash-capture" -or $flushTimedOut) {
+            $sendName = if ($flushTimedOut) { "crash-send-after-$Action" } else { "crash-send" }
+            Write-Host "Running crash-send to ensure cached envelopes are sent..."
+
+            $androidLaunches = $script:AndroidUninitialisedLaunches
             $sendResult = Invoke-App -Action "crash-send"
+            if ($flushTimedOut) {
+                $script:AndroidUninitialisedLaunches = $androidLaunches
+            }
 
             if ($script:Platform -eq "Xbox") {
                 $sendResult = Get-XboxLogOutput -RunResult $sendResult -Action "crash-send"
             }
 
             # Save crash-send result to JSON for debugging
-            $sendResult | ConvertTo-Json -Depth 5 | Out-File -FilePath (Get-OutputFilePath "crash-send-result.json")
+            $sendResult | ConvertTo-Json -Depth 5 | Out-File -FilePath (Get-OutputFilePath "$sendName-result.json")
 
             # Print crash-send output
-            Write-Host "::group::App output (crash-send)"
+            Write-Host "::group::App output ($sendName)"
             $sendResult.Output | ForEach-Object { Write-Host $_ }
             Write-Host "::endgroup::"
 
@@ -311,6 +328,7 @@ BeforeAll {
     }
 
     $script:AndroidUninitialisedLaunches = 0
+    $script:FlushTimeoutRelaunched = $false
 
     $script:Platform = $env:SENTRY_TEST_PLATFORM
     if ([string]::IsNullOrEmpty($script:Platform)) {

test/Scripts.Integration.Test/Scripts/CliConfiguration.cs

@@ -13,6 +13,12 @@ public override void Configure(SentryCliOptions cliOptions)
         cliOptions.UploadSources = cliOptions.UploadSymbols;
         cliOptions.Auth = authToken;
 
+        // sentry-cli's default 5 retries give up within ~30s of sentry.io answering 503/504. Reaches the uploads the Editor spawns; iOS sets it on xcodebuild.
+        if (Environment.GetEnvironmentVariable("SENTRY_HTTP_MAX_RETRIES") is null)
+        {
+            Environment.SetEnvironmentVariable("SENTRY_HTTP_MAX_RETRIES", "20");
+        }
+
         cliOptions.Organization = "sentry-sdks";
         cliOptions.Project = "sentry-unity-integration-tests";
 

test/Scripts.Integration.Test/assert-symbol-upload.ps1

@@ -69,4 +69,11 @@ if (-not $uploadSucceeded)
         "Expected 'Uploaded N missing debug information files' or 'Nothing to upload'."
 }
 
+$retries = [regex]::Matches($log, "retry number \d+ failed due to request failed with retryable status code (\d+)")
+if ($retries.Count -gt 0)
+{
+    $codes = ($retries | ForEach-Object { $_.Groups[1].Value } | Sort-Object -Unique) -join ', '
+    Write-Host "::warning::sentry-cli retried $($retries.Count) upload request(s) after HTTP $codes (signature: sentry-cli-http-retry)."
+}
+
 Write-Host "Symbol-upload assertion passed: sentry-cli uploaded debug symbols and sources ('$LogPath')." -ForegroundColor Green