fix(launch): stop launch --wait duplicate runs and support API 1.230 options [COMP-2645]

USAGE.md

@@ -772,12 +772,15 @@ tw lineage upstream -w my-org/my-workspace -i <lid>
   - AWS Cloud: `--log-group`. Google Cloud: `--project-id`.
   - HPC platforms: `--propagate-head-job-options`.
   - Seqera Compute: `--data-retention-policy`.
-- `tw launch`, `tw pipelines add|update`, `tw runs relaunch`, and `tw actions add|update`: `--tower-config`. List the Nextflow and Fusion versions you can launch with `tw info versions` (`-c fusion` for Fusion).
+  - Azure Cloud: `--key-vault-url`. Google Cloud: `--billing-export-table`.
+- `tw launch`, `tw pipelines add|update`, `tw runs relaunch`, and `tw actions add|update`: `--fusion-version` and `--tower-config`. List the Nextflow and Fusion versions you can launch with `tw info versions` (`-c fusion` for Fusion).
 - `tw runs star|unstar`, `tw runs view log [--next <cursor>]`, and `tw runs delete -i <id>,<id>`.
 - `tw pipelines schema`: show a pipeline's parameter schema.
 - `tw studios star|unstar|extend|logs|rename-checkpoint`. `--mount-data-uris` also accepts a folder inside a data link.
 - `tw data-links delete-content|download-script`.
 - `tw datasets add --url`, `tw datasets update --url`, and `tw datasets disable-version`.
+- `tw credentials federation-setup -p aws|google`: show the values to configure in your cloud provider before adding workload identity credentials.
+- `tw launch --wait` keeps polling through transient errors after a successful submit. If the run's status can't be checked, it exits with code 3 instead of 1.
 
 [compute-envs]: https://docs.seqera.io/platform/latest/compute-envs/overview
 [credentials]: https://docs.seqera.io/platform/latest/credentials/overview

conf/reflect-config.json

@@ -912,6 +912,13 @@
   "allDeclaredMethods":true,
   "methods":[{"name":"<init>","parameterTypes":[] }]
 },
+{
+  "name":"io.seqera.tower.cli.commands.credentials.FederationSetupCmd",
+  "queryAllDeclaredMethods":true,
+  "queryAllDeclaredConstructors":true,
+  "fields":[{"name":"provider"}, {"name":"workspace"}],
+  "methods":[{"name":"<init>","parameterTypes":[] }]
+},
 {
   "name":"io.seqera.tower.cli.commands.credentials.ListCmd",
   "allDeclaredFields":true,
@@ -2550,6 +2557,12 @@
   "queryAllDeclaredMethods":true,
   "queryAllDeclaredConstructors":true
 },
+{
+  "name":"io.seqera.tower.cli.responses.CredentialsFederationSetup",
+  "allDeclaredFields":true,
+  "queryAllDeclaredMethods":true,
+  "queryAllDeclaredConstructors":true
+},
 {
   "name":"io.seqera.tower.cli.responses.CredentialsList",
   "allDeclaredFields":true,

src/main/java/io/seqera/tower/cli/commands/AbstractApiCmd.java

@@ -59,6 +59,7 @@
 import io.seqera.tower.cli.exceptions.NoComputeEnvironmentException;
 import io.seqera.tower.cli.exceptions.OrganizationNotFoundException;
 import io.seqera.tower.cli.exceptions.ShowUsageException;
+import io.seqera.tower.cli.exceptions.StatusCheckFailedException;
 import io.seqera.tower.cli.exceptions.TowerException;
 import io.seqera.tower.cli.exceptions.WorkspaceNotFoundException;
 import io.seqera.tower.cli.responses.Response;
@@ -700,8 +701,8 @@ public Integer call() {
             return onBeforeExit(exitCode, response);
         } catch (Exception e) {
             errorMessage(app(), e);
+            return e instanceof StatusCheckFailedException ? StatusCheckFailedException.EXIT_CODE : CommandLine.ExitCode.SOFTWARE;
         }
-        return CommandLine.ExitCode.SOFTWARE;
     }
 
     protected Integer onBeforeExit(int exitCode, Response response) throws ApiException {

src/main/java/io/seqera/tower/cli/commands/CredentialsCmd.java

@@ -18,6 +18,7 @@
 
 import io.seqera.tower.cli.commands.credentials.AddCmd;
 import io.seqera.tower.cli.commands.credentials.DeleteCmd;
+import io.seqera.tower.cli.commands.credentials.FederationSetupCmd;
 import io.seqera.tower.cli.commands.credentials.ListCmd;
 import io.seqera.tower.cli.commands.credentials.UpdateCmd;
 import io.seqera.tower.cli.commands.credentials.ValidateCmd;
@@ -32,7 +33,8 @@
                 UpdateCmd.class,
                 DeleteCmd.class,
                 ListCmd.class,
-                ValidateCmd.class
+                ValidateCmd.class,
+                FederationSetupCmd.class
         }
 )
 public class CredentialsCmd extends AbstractRootCmd {

src/main/java/io/seqera/tower/cli/commands/LaunchCmd.java

@@ -23,6 +23,7 @@
 import io.seqera.tower.cli.commands.pipelines.NullableLaunchOptions;
 import io.seqera.tower.cli.commands.pipelines.versions.VersionRefOptions;
 import io.seqera.tower.cli.exceptions.InvalidResponseException;
+import io.seqera.tower.cli.exceptions.StatusCheckFailedException;
 import io.seqera.tower.cli.responses.Response;
 import io.seqera.tower.cli.responses.runs.RunSubmited;
 import io.seqera.tower.model.ComputeEnvResponseDto;
@@ -89,7 +90,7 @@ public class LaunchCmd extends AbstractRootCmd {
     @ArgGroup(multiplicity = "0..1")
     VersionRefOptions.VersionRef versionRef;
 
-    @Option(names = {"--wait"}, description = "Wait until workflow reaches specified status: ${COMPLETION-CANDIDATES}")
+    @Option(names = {"--wait"}, description = "Wait until workflow reaches specified status: ${COMPLETION-CANDIDATES}. Exits with code 3 if the run was submitted but its status could not be checked.")
     public WorkflowStatus wait;
 
     @Option(names = {"-l", "--labels"}, split = ",", description = "Labels to assign to each pipeline run. Provide comma-separated label values (use key=value format for resource labels). Labels will be created if they don't exist", converter = Label.LabelConverter.class)
@@ -250,18 +251,22 @@ protected Integer onBeforeExit(int exitCode, Response response) {
                     () -> checkWorkflowStatus(submitted.workflowId, submitted.workspaceId),
                     WorkflowStatus.CANCELLED, WorkflowStatus.FAILED, WorkflowStatus.SUCCEEDED
             );
+        } catch (StatusCheckFailedException e) {
+            String workspaceOption = submitted.workspaceId == null ? "" : " -w " + submitted.workspaceId;
+            throw new StatusCheckFailedException(String.format("Run %s was submitted but its status could not be checked (%s). It may still be running, check it with: tw runs view -i %s%s", submitted.workflowId, e.getReason(), submitted.workflowId, workspaceOption), e);
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
             return exitCode;
         }
     }
 
-    private WorkflowStatus checkWorkflowStatus(String workflowId, Long workspaceId) {
-        try {
-            return workflowsApi().describeWorkflow(workflowId, workspaceId, NO_WORKFLOW_ATTRIBUTES).getWorkflow().getStatus();
-        } catch (ApiException | NullPointerException e) {
-            return null;
+    private WorkflowStatus checkWorkflowStatus(String workflowId, Long workspaceId) throws ApiException {
+        WorkflowStatus status = workflowsApi().describeWorkflow(workflowId, workspaceId, NO_WORKFLOW_ATTRIBUTES).getWorkflow().getStatus();
+        // Platform lost contact with the run, which may still be running: keep polling as for a failed check
+        if (status == WorkflowStatus.UNKNOWN) {
+            throw new IllegalStateException("run status is UNKNOWN");
         }
+        return status;
     }
 
     private List<Long> obtainLabelIDs(@Nullable Long workspaceId) throws ApiException {

src/main/java/io/seqera/tower/cli/commands/computeenvs/DeleteCmd.java

@@ -86,11 +86,7 @@ protected Integer onBeforeExit(int exitCode, Response response) {
         }
     }
 
-    private ComputeEnvStatus checkComputeEnvStatus(String computeEnvId, Long workspaceId) {
-        try {
-            return computeEnvsApi().describeComputeEnv(computeEnvId, workspaceId, Collections.emptyList()).getComputeEnv().getStatus();
-        } catch (ApiException | NullPointerException e) {
-            return null;
-        }
+    private ComputeEnvStatus checkComputeEnvStatus(String computeEnvId, Long workspaceId) throws ApiException {
+        return computeEnvsApi().describeComputeEnv(computeEnvId, workspaceId, Collections.emptyList()).getComputeEnv().getStatus();
     }
 }

src/main/java/io/seqera/tower/cli/commands/computeenvs/add/AbstractAddCmd.java

@@ -94,12 +94,8 @@ protected Integer onBeforeExit(int exitCode, Response response) {
         }
     }
 
-    private ComputeEnvStatus checkComputeEnvStatus(String computeEnvId, Long workspaceId) {
-        try {
-            return computeEnvsApi().describeComputeEnv(computeEnvId, workspaceId, Collections.emptyList()).getComputeEnv().getStatus();
-        } catch (ApiException | NullPointerException e) {
-            return null;
-        }
+    private ComputeEnvStatus checkComputeEnvStatus(String computeEnvId, Long workspaceId) throws ApiException {
+        return computeEnvsApi().describeComputeEnv(computeEnvId, workspaceId, Collections.emptyList()).getComputeEnv().getStatus();
     }
 
     protected ComputeEnvAdded addComputeEnv(PlatformEnum platform, ComputeConfig config) throws ApiException {

src/main/java/io/seqera/tower/cli/commands/computeenvs/platforms/AzCloudPlatform.java

@@ -92,6 +92,11 @@ public AzCloudConfig computeConfig() throws ApiException, IOException {
                     .logTableName(adv.logTableName)
                     .dataCollectionEndpoint(adv.dataCollectionEndpoint)
                     .dataCollectionRuleId(adv.dataCollectionRuleId);
+
+            // keyVaultUrl is JsonNullable: setting null would send an explicit null.
+            if (adv.keyVaultUrl != null) {
+                config.keyVaultUrl(adv.keyVaultUrl);
+            }
         }
 
         // Common
@@ -125,6 +130,9 @@ public static class AdvancedOptions {
         @Option(names = {"--instance-type"}, description = "Azure virtual machine size (e.g., Standard_D2s_v3, Standard_E4s_v3). If omitted, a default VM size is used.")
         public String instanceType;
 
+        @Option(names = {"--key-vault-url"}, description = "Azure Key Vault that stores pipeline secrets, in the form https://<vault-name>.vault.azure.net. Only used with --sched-enabled. If absent, the default Key Vault is used, if one is configured.")
+        public String keyVaultUrl;
+
         @Option(names = {"--log-table-name"}, description = "Custom table name in Log Analytics workspace for storing compute environment logs. Enables organized log management.")
         public String logTableName;
 

src/main/java/io/seqera/tower/cli/commands/computeenvs/platforms/GoogleCloudPlatform.java

@@ -82,6 +82,10 @@ public GoogleCloudConfig computeConfig() throws ApiException, IOException {
             if (sched.machineTypes != null) {
                 schedConfig.machineTypes(sched.machineTypes);
             }
+            // billingExportTable is JsonNullable: setting null would send an explicit null.
+            if (sched.billingExportTable != null) {
+                schedConfig.billingExportTable(sched.billingExportTable);
+            }
             config.schedConfig(schedConfig);
         }
 
@@ -148,6 +152,9 @@ public static class SchedOptions {
 
         @Option(names = {"--sched-machine-types"}, description = "Compute Engine machine types for compute nodes managed by the Seqera scheduler. Comma-separated list (e.g., n2-standard-4,c2-standard-8). Leave empty to let the scheduler select the most cost-effective types.", split = ",")
         public List<String> machineTypes;
+
+        @Option(names = {"--billing-export-table"}, description = "Fully-qualified BigQuery table holding the Cloud Billing export, as project.dataset.table. Enables cost reporting for this compute environment. Costs are unavailable for runs that predate the export.")
+        public String billingExportTable;
     }
 
     public static class AdvancedOptions {

src/main/java/io/seqera/tower/cli/commands/credentials/FederationSetupCmd.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2021-2026, Seqera.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package io.seqera.tower.cli.commands.credentials;
+
+import io.seqera.tower.ApiException;
+import io.seqera.tower.cli.commands.global.WorkspaceOptionalOptions;
+import io.seqera.tower.cli.responses.CredentialsFederationSetup;
+import io.seqera.tower.cli.responses.Response;
+import io.seqera.tower.model.CredentialsSetupValuesResponse;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Mixin;
+import picocli.CommandLine.Option;
+
+@Command(
+        name = "federation-setup",
+        description = "Show the values to configure in your cloud provider before adding workload identity credentials"
+)
+public class FederationSetupCmd extends AbstractCredentialsCmd {
+
+    // Platform returns no values, rather than an error, for any other provider
+    public enum Provider { aws, google }
+
+    @Option(names = {"-p", "--provider"}, description = "Credentials provider: ${COMPLETION-CANDIDATES}.", required = true)
+    public Provider provider;
+
+    @Mixin
+    public WorkspaceOptionalOptions workspace;
+
+    @Override
+    protected Response exec() throws ApiException {
+        Long wspId = workspaceId(workspace.workspace);
+        CredentialsSetupValuesResponse response = credentialsApi().describeCredentialsSetup(provider.name(), wspId);
+        return new CredentialsFederationSetup(provider.name(), workspaceRef(wspId), response.getSetupValues());
+    }
+}

src/main/java/io/seqera/tower/cli/commands/pipelines/NullableLaunchOptions.java

@@ -43,6 +43,9 @@ public class NullableLaunchOptions {
     @Option(names = {"--nextflow-version"}, description = "Nextflow version to run the workflow with. Must exist in the Platform version catalog and meet the minimum required by the compute environment (see 'tw info versions'). Takes precedence over the value stored in the launch configuration.")
     public String nextflowVersion;
 
+    @Option(names = {"--fusion-version"}, description = "Fusion version to run the workflow with. Must exist in the Platform version catalog and only applies when the compute environment enables Fusion v2 (see 'tw info versions -c fusion'). Takes precedence over the value stored in the launch configuration.")
+    public String fusionVersion;
+
     @Option(names = {"--output-dir"}, description = "Per-run output directory, passed to Nextflow as '-output-dir'. Requires Nextflow 24.10.0 or later and the workflow outputs syntax. Takes precedence over the value stored in the launch configuration.")
     public String outputDir;
 
@@ -53,6 +56,9 @@ public void applyNullableOptions(WorkflowLaunchRequest request) {
         if (nextflowVersion != null) {
             request.nextflowVersion(nextflowVersion);
         }
+        if (fusionVersion != null) {
+            request.fusionVersion(fusionVersion);
+        }
         if (outputDir != null) {
             request.outputDir(outputDir);
         }

src/main/java/io/seqera/tower/cli/commands/studios/AbstractStudiosCmd.java

@@ -152,13 +152,9 @@ protected Integer onBeforeExit(int exitCode, String sessionId, Long workspaceId,
         }
     }
 
-    protected DataStudioStatus checkStudioStatus(String sessionId, Long workspaceId) {
-        try {
-            DataStudioStatusInfo statusInfo = studiosApi().describeDataStudio(sessionId, workspaceId).getStatusInfo();
-            return statusInfo == null ? null : statusInfo.getStatus();
-        } catch (ApiException e) {
-            return null;
-        }
+    protected DataStudioStatus checkStudioStatus(String sessionId, Long workspaceId) throws ApiException {
+        DataStudioStatusInfo statusInfo = studiosApi().describeDataStudio(sessionId, workspaceId).getStatusInfo();
+        return statusInfo == null ? null : statusInfo.getStatus();
     }
 
     protected DataStudioConfiguration studioConfigurationFrom(Long wspId, StudioConfigurationOptions configurationOptions, String condaEnvOverride) throws ApiException {

src/main/java/io/seqera/tower/cli/exceptions/StatusCheckFailedException.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2021-2026, Seqera.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.seqera.tower.cli.exceptions;
+
+/**
+ * The resource was created or submitted, but polling its status kept failing, so its outcome is unknown.
+ */
+public class StatusCheckFailedException extends TowerRuntimeException {
+
+    // Distinct from 1 so automation does not mistake "status unknown" for "failed" and retry, creating duplicates
+    public static final int EXIT_CODE = 3;
+
+    private final String reason;
+
+    public StatusCheckFailedException(String reason, Throwable cause) {
+        super(String.format("Status could not be checked (%s)", reason), cause);
+        this.reason = reason;
+    }
+
+    public StatusCheckFailedException(String message, StatusCheckFailedException failure) {
+        super(message, failure.getCause());
+        this.reason = failure.reason;
+    }
+
+    public String getReason() {
+        return reason;
+    }
+
+}

src/main/java/io/seqera/tower/cli/responses/CredentialsFederationSetup.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2021-2026, Seqera.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package io.seqera.tower.cli.responses;
+
+import io.seqera.tower.cli.utils.TableList;
+import io.seqera.tower.model.CredentialsSetupValue;
+
+import java.io.PrintWriter;
+import java.util.List;
+
+public class CredentialsFederationSetup extends Response {
+
+    public final String provider;
+    public final String workspaceRef;
+    public final List<CredentialsSetupValue> setupValues;
+
+    public CredentialsFederationSetup(String provider, String workspaceRef, List<CredentialsSetupValue> setupValues) {
+        this.provider = provider;
+        this.workspaceRef = workspaceRef;
+        this.setupValues = setupValues;
+    }
+
+    @Override
+    public void toString(PrintWriter out) {
+        out.println(ansi(String.format("%n  @|bold Workload identity setup values for '%s' credentials at %s workspace:|@%n", provider, workspaceRef)));
+        if (setupValues == null || setupValues.isEmpty()) {
+            out.println(ansi("    @|yellow No setup values for this provider|@"));
+            return;
+        }
+
+        TableList table = new TableList(out, 2);
+        table.setPrefix("    ");
+        setupValues.forEach(value -> table.addRow(value.getLabel(), value.getValue()));
+        table.print();
+        out.println("");
+    }
+}

src/main/java/io/seqera/tower/cli/utils/ModelHelper.java

@@ -53,7 +53,7 @@ public static WorkflowLaunchRequest createLaunchRequest(WorkflowLaunchResponse l
      * API gains in a future SDK release are carried over with no change here.
      * <p>
      * Going through JSON is what makes this safe for the nullable fields ({@code syntaxParser},
-     * {@code nextflowVersion}, {@code outputDir}): an undefined value stays undefined instead of becoming an
+     * {@code nextflowVersion}, {@code fusionVersion}, {@code outputDir}): an undefined value stays undefined instead of becoming an
      * explicit {@code null}, which the fluent setters cannot express because they wrap their argument in
      * {@code JsonNullable.of()} unconditionally.
      */