diff --git a/conductor-client-metrics/src/main/java/com/netflix/conductor/client/metrics/prometheus/PrometheusMetricsCollector.java b/conductor-client-metrics/src/main/java/com/netflix/conductor/client/metrics/prometheus/PrometheusMetricsCollector.java index 1d99e6df1..81e008d36 100644 --- a/conductor-client-metrics/src/main/java/com/netflix/conductor/client/metrics/prometheus/PrometheusMetricsCollector.java +++ b/conductor-client-metrics/src/main/java/com/netflix/conductor/client/metrics/prometheus/PrometheusMetricsCollector.java @@ -25,6 +25,7 @@ import com.netflix.conductor.client.events.task.TaskPayloadUsedEvent; import com.netflix.conductor.client.events.task.TaskResultPayloadSizeEvent; import com.netflix.conductor.client.events.taskrunner.ActiveWorkersChanged; +import com.netflix.conductor.client.events.taskrunner.AdaptiveConcurrencyChanged; import com.netflix.conductor.client.events.taskrunner.PollCompleted; import com.netflix.conductor.client.events.taskrunner.PollFailure; import com.netflix.conductor.client.events.taskrunner.PollStarted; @@ -88,6 +89,7 @@ public class PrometheusMetricsCollector implements MetricsCollector { private final PrometheusMeterRegistry registry; private final PrometheusApiClientMetrics apiClientMetrics; private final ConcurrentHashMap activeWorkerGauges = new ConcurrentHashMap<>(); + private final ConcurrentHashMap adaptiveGauges = new ConcurrentHashMap<>(); public PrometheusMetricsCollector() { this(SHARED_REGISTRY); @@ -196,6 +198,31 @@ public void consume(ActiveWorkersChanged e) { .set(Math.max(0, e.getCount())); } + // ----- Adaptive concurrency ----- + + @Override + public void consume(AdaptiveConcurrencyChanged e) { + String taskType = nullToEmpty(e.getTaskType()); + adaptiveGauge("adaptive_worker_concurrency_target", "Adaptive concurrency share of the worker", + taskType).set(e.getTargetConcurrency()); + adaptiveGauge("adaptive_worker_held_slots", "Slots held by the worker's polls and tasks", + taskType).set(e.getHeldSlots()); + adaptiveGauge("adaptive_worker_task_slot_age_seconds", + "Seconds since the worker last released a task slot, while any are held; a wedged worker grows this", + taskType).set((int) Math.min(Integer.MAX_VALUE, e.getTaskSlotAgeSeconds())); + adaptiveGauge("adaptive_queue_depth", "Last sampled queue depth of the worker's task queue", + taskType).set(e.getQueueDepth()); + adaptiveGauge("adaptive_process_concurrency_target", "Adaptive concurrency target of the process", + null).set(e.getProcessTargetConcurrency()); + adaptiveGauge("adaptive_process_held_slots", "Slots held by all adaptive workers of the process", + null).set(e.getProcessHeldSlots()); + adaptiveGauge("adaptive_admission_paused", "1 while task acquisition is paused by heap pressure", + null).set(e.isAdmissionPaused() ? 1 : 0); + counter("adaptive_concurrency_changes_total", "Adaptive concurrency observations by reason", + "taskType", taskType, + "reason", nullToEmpty(e.getReason())).increment(); + } + // ----- Task update ----- @Override @@ -291,6 +318,19 @@ public void consume(WorkflowStartedEvent event) { // ----- Helpers ----- + private AtomicInteger adaptiveGauge(String name, String description, String taskType) { + return adaptiveGauges.computeIfAbsent(taskType == null ? name : name + "|" + taskType, k -> { + AtomicInteger val = new AtomicInteger(0); + Gauge.Builder builder = Gauge.builder(name, val, AtomicInteger::doubleValue) + .description(description); + if (taskType != null) { + builder.tag("taskType", taskType); + } + builder.register(registry); + return val; + }); + } + private Counter counter(String name, String description, String... tagKv) { return Counter.builder(name) .description(description) diff --git a/conductor-client-metrics/src/test/java/com/netflix/conductor/client/metrics/prometheus/PrometheusMetricsCollectorTest.java b/conductor-client-metrics/src/test/java/com/netflix/conductor/client/metrics/prometheus/PrometheusMetricsCollectorTest.java index c39c7c268..847e79589 100644 --- a/conductor-client-metrics/src/test/java/com/netflix/conductor/client/metrics/prometheus/PrometheusMetricsCollectorTest.java +++ b/conductor-client-metrics/src/test/java/com/netflix/conductor/client/metrics/prometheus/PrometheusMetricsCollectorTest.java @@ -23,6 +23,7 @@ import com.netflix.conductor.client.events.task.TaskPayloadUsedEvent; import com.netflix.conductor.client.events.task.TaskResultPayloadSizeEvent; import com.netflix.conductor.client.events.taskrunner.ActiveWorkersChanged; +import com.netflix.conductor.client.events.taskrunner.AdaptiveConcurrencyChanged; import com.netflix.conductor.client.events.taskrunner.PollCompleted; import com.netflix.conductor.client.events.taskrunner.PollFailure; import com.netflix.conductor.client.events.taskrunner.PollStarted; @@ -134,6 +135,21 @@ void activeWorkersChangedSetsGauge() { assertEquals(7.0, val); } + @Test + void adaptiveConcurrencyChangedUpdatesGauges() { + collector.consume(new AdaptiveConcurrencyChanged("SIMPLE", null, 29, 12, 3, 48, 32, 30, false, "DEPTH_SAMPLE")); + collector.consume(new AdaptiveConcurrencyChanged("SIMPLE", null, 24, 12, 7, 48, 24, 30, true, "HEAP_PAUSE")); + + assertEquals(24.0, registry.get("adaptive_worker_concurrency_target").tag("taskType", "SIMPLE").gauge().value()); + assertEquals(12.0, registry.get("adaptive_worker_held_slots").tag("taskType", "SIMPLE").gauge().value()); + assertEquals(7.0, registry.get("adaptive_worker_task_slot_age_seconds").tag("taskType", "SIMPLE").gauge().value()); + assertEquals(48.0, registry.get("adaptive_queue_depth").tag("taskType", "SIMPLE").gauge().value()); + assertEquals(24.0, registry.get("adaptive_process_concurrency_target").gauge().value()); + assertEquals(1.0, registry.get("adaptive_admission_paused").gauge().value()); + assertEquals(1.0, registry.get("adaptive_concurrency_changes_total") + .tag("taskType", "SIMPLE").tag("reason", "HEAP_PAUSE").counter().count()); + } + @Test void activeWorkersChangedClampsNegativeToZero() { collector.consume(new ActiveWorkersChanged("SIMPLE", -1)); diff --git a/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/ClientProperties.java b/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/ClientProperties.java index 5e1b5f554..cf3d7cd6a 100644 --- a/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/ClientProperties.java +++ b/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/ClientProperties.java @@ -18,6 +18,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties; +import com.netflix.conductor.client.automator.AdaptiveConcurrencyConfig; + @ConfigurationProperties("conductor.client") public class ClientProperties { @@ -51,6 +53,229 @@ public void setWrite(int write) { } } + public static class AdaptiveConcurrency { + + // Enables adaptive concurrency for workers started by this client + private boolean enabled = false; + + // Aggregate admission ceiling for all adaptive workers + private int maxProcessConcurrency = 256; + + // Initial process target; 0 uses min(logical queues, available processors) + private int initialProcessConcurrency = 0; + + // Lowest process target an idle process decays to; CPU and heap pressure may go below it + private int minProcessConcurrency = 1; + + // Per-worker ceiling for workers without their own + private int defaultMaxWorkerConcurrency = 64; + + // Ceiling for the number of tasks requested by one poll + private int maxPollBatch = 16; + + private Duration resourceSampleInterval = Duration.ofSeconds(1); + + private Duration metricFreshness = Duration.ofSeconds(5); + + private Duration depthSampleInterval = Duration.ofSeconds(1); + + private Duration depthFreshness = Duration.ofSeconds(5); + + private Duration heartbeatInterval = Duration.ofSeconds(5); + + private Duration standbyPollTimeout = Duration.ofSeconds(1); + + private Duration controlInterval = Duration.ofSeconds(5); + + private double cpuSmoothingAlpha = 0.2; + + private double growthThreshold = 0.75; + + private double cpuPressureThreshold = 0.90; + + private double heapPauseThreshold = 0.90; + + private double heapResumeThreshold = 0.80; + + // Allow running without CPU and heap metrics + private boolean demandOnlyEnabled = false; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public int getMaxProcessConcurrency() { + return maxProcessConcurrency; + } + + public void setMaxProcessConcurrency(int maxProcessConcurrency) { + this.maxProcessConcurrency = maxProcessConcurrency; + } + + public int getInitialProcessConcurrency() { + return initialProcessConcurrency; + } + + public void setInitialProcessConcurrency(int initialProcessConcurrency) { + this.initialProcessConcurrency = initialProcessConcurrency; + } + + public int getMinProcessConcurrency() { + return minProcessConcurrency; + } + + public void setMinProcessConcurrency(int minProcessConcurrency) { + this.minProcessConcurrency = minProcessConcurrency; + } + + public int getDefaultMaxWorkerConcurrency() { + return defaultMaxWorkerConcurrency; + } + + public void setDefaultMaxWorkerConcurrency(int defaultMaxWorkerConcurrency) { + this.defaultMaxWorkerConcurrency = defaultMaxWorkerConcurrency; + } + + public int getMaxPollBatch() { + return maxPollBatch; + } + + public void setMaxPollBatch(int maxPollBatch) { + this.maxPollBatch = maxPollBatch; + } + + public Duration getResourceSampleInterval() { + return resourceSampleInterval; + } + + public void setResourceSampleInterval(Duration resourceSampleInterval) { + this.resourceSampleInterval = resourceSampleInterval; + } + + public Duration getMetricFreshness() { + return metricFreshness; + } + + public void setMetricFreshness(Duration metricFreshness) { + this.metricFreshness = metricFreshness; + } + + public Duration getDepthSampleInterval() { + return depthSampleInterval; + } + + public void setDepthSampleInterval(Duration depthSampleInterval) { + this.depthSampleInterval = depthSampleInterval; + } + + public Duration getDepthFreshness() { + return depthFreshness; + } + + public void setDepthFreshness(Duration depthFreshness) { + this.depthFreshness = depthFreshness; + } + + public Duration getHeartbeatInterval() { + return heartbeatInterval; + } + + public void setHeartbeatInterval(Duration heartbeatInterval) { + this.heartbeatInterval = heartbeatInterval; + } + + public Duration getStandbyPollTimeout() { + return standbyPollTimeout; + } + + public void setStandbyPollTimeout(Duration standbyPollTimeout) { + this.standbyPollTimeout = standbyPollTimeout; + } + + public Duration getControlInterval() { + return controlInterval; + } + + public void setControlInterval(Duration controlInterval) { + this.controlInterval = controlInterval; + } + + public double getCpuSmoothingAlpha() { + return cpuSmoothingAlpha; + } + + public void setCpuSmoothingAlpha(double cpuSmoothingAlpha) { + this.cpuSmoothingAlpha = cpuSmoothingAlpha; + } + + public double getGrowthThreshold() { + return growthThreshold; + } + + public void setGrowthThreshold(double growthThreshold) { + this.growthThreshold = growthThreshold; + } + + public double getCpuPressureThreshold() { + return cpuPressureThreshold; + } + + public void setCpuPressureThreshold(double cpuPressureThreshold) { + this.cpuPressureThreshold = cpuPressureThreshold; + } + + public double getHeapPauseThreshold() { + return heapPauseThreshold; + } + + public void setHeapPauseThreshold(double heapPauseThreshold) { + this.heapPauseThreshold = heapPauseThreshold; + } + + public double getHeapResumeThreshold() { + return heapResumeThreshold; + } + + public void setHeapResumeThreshold(double heapResumeThreshold) { + this.heapResumeThreshold = heapResumeThreshold; + } + + public boolean isDemandOnlyEnabled() { + return demandOnlyEnabled; + } + + public void setDemandOnlyEnabled(boolean demandOnlyEnabled) { + this.demandOnlyEnabled = demandOnlyEnabled; + } + + public AdaptiveConcurrencyConfig toConfig() { + return new AdaptiveConcurrencyConfig.Builder() + .withMaxProcessConcurrency(maxProcessConcurrency) + .withInitialProcessConcurrency(initialProcessConcurrency) + .withMinProcessConcurrency(minProcessConcurrency) + .withDefaultMaxWorkerConcurrency(defaultMaxWorkerConcurrency) + .withMaxPollBatch(maxPollBatch) + .withResourceSampleInterval(resourceSampleInterval) + .withMetricFreshness(metricFreshness) + .withDepthSampleInterval(depthSampleInterval) + .withDepthFreshness(depthFreshness) + .withHeartbeatInterval(heartbeatInterval) + .withStandbyPollTimeout(standbyPollTimeout) + .withControlInterval(controlInterval) + .withCpuSmoothingAlpha(cpuSmoothingAlpha) + .withGrowthThreshold(growthThreshold) + .withCpuPressureThreshold(cpuPressureThreshold) + .withHeapPauseThreshold(heapPauseThreshold) + .withHeapResumeThreshold(heapResumeThreshold) + .withDemandOnlyEnabled(demandOnlyEnabled) + .build(); + } + } + private String rootUri; private String basePath; @@ -75,6 +300,36 @@ public void setWrite(int write) { private boolean verifyingSsl = true; + private boolean useVirtualThreads = false; + + private boolean updateV2 = false; + + private AdaptiveConcurrency adaptiveConcurrency = new AdaptiveConcurrency(); + + public boolean isUseVirtualThreads() { + return useVirtualThreads; + } + + public void setUseVirtualThreads(boolean useVirtualThreads) { + this.useVirtualThreads = useVirtualThreads; + } + + public boolean isUpdateV2() { + return updateV2; + } + + public void setUpdateV2(boolean updateV2) { + this.updateV2 = updateV2; + } + + public AdaptiveConcurrency getAdaptiveConcurrency() { + return adaptiveConcurrency; + } + + public void setAdaptiveConcurrency(AdaptiveConcurrency adaptiveConcurrency) { + this.adaptiveConcurrency = adaptiveConcurrency; + } + public void setVerifyingSsl(boolean verifyingSsl) { this.verifyingSsl = verifyingSsl; } diff --git a/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/ConductorClientAutoConfiguration.java b/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/ConductorClientAutoConfiguration.java index 0baf28177..ab5f7ca8d 100644 --- a/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/ConductorClientAutoConfiguration.java +++ b/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/ConductorClientAutoConfiguration.java @@ -23,11 +23,13 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.core.Ordered; import org.springframework.core.env.Environment; +import com.netflix.conductor.client.automator.AdaptiveConcurrencyController; import com.netflix.conductor.client.automator.TaskRunnerConfigurer; import com.netflix.conductor.client.http.ConductorClient; import com.netflix.conductor.client.http.TaskClient; @@ -75,6 +77,13 @@ public AnnotatedWorkerExecutor annotatedWorkerExecutor(TaskClient taskClient) { return new AnnotatedWorkerExecutor(taskClient); } + @Bean + @ConditionalOnProperty(prefix = "conductor.client.adaptive-concurrency", name = "enabled", havingValue = "true") + @ConditionalOnMissingBean + public AdaptiveConcurrencyController adaptiveConcurrencyController(ClientProperties clientProperties) { + return new AdaptiveConcurrencyController(clientProperties.getAdaptiveConcurrency().toConfig()); + } + @Bean(initMethod = "init", destroyMethod = "shutdown") @ConditionalOnBean(ConductorClient.class) @ConditionalOnMissingBean @@ -82,7 +91,8 @@ public TaskRunnerConfigurer taskRunnerConfigurer(Environment env, TaskClient taskClient, ClientProperties clientProperties, List workers, - Optional metricsCollector) { + Optional metricsCollector, + Optional adaptiveConcurrencyController) { Map taskThreadCount = new HashMap<>(); for (Worker worker : workers) { String key = "conductor.worker." + worker.getTaskDefName() + ".threadCount"; @@ -104,8 +114,22 @@ public TaskRunnerConfigurer taskRunnerConfigurer(Environment env, .withUpdateRetryCount(clientProperties.getUpdateRetryCount()) .withTaskToDomain(clientProperties.getTaskToDomain()) .withShutdownGracePeriodSeconds(clientProperties.getShutdownGracePeriodSeconds()) - .withTaskPollTimeout(clientProperties.getTaskPollTimeout()); + .withTaskPollTimeout(clientProperties.getTaskPollTimeout()) + .withUseVirtualThreads(clientProperties.isUseVirtualThreads()) + .withUpdateV2(clientProperties.isUpdateV2()); metricsCollector.ifPresent(builder::withMetricsCollector); + adaptiveConcurrencyController.ifPresent(controller -> { + var workerConfiguration = new SpringWorkerConfiguration(env); + Map taskAdaptiveMaxConcurrency = new HashMap<>(); + for (Worker worker : workers) { + int maxConcurrency = workerConfiguration.getAdaptiveMaxConcurrency(worker.getTaskDefName()); + if (maxConcurrency > 0) { + taskAdaptiveMaxConcurrency.put(worker.getTaskDefName(), maxConcurrency); + } + } + builder.withAdaptiveConcurrency(controller) + .withTaskAdaptiveMaxConcurrency(taskAdaptiveMaxConcurrency); + }); return builder.build(); } diff --git a/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/ConductorWorkerAutoConfiguration.java b/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/ConductorWorkerAutoConfiguration.java index 399708e8d..9947488dc 100644 --- a/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/ConductorWorkerAutoConfiguration.java +++ b/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/ConductorWorkerAutoConfiguration.java @@ -24,6 +24,7 @@ import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; +import com.netflix.conductor.client.automator.AdaptiveConcurrencyController; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.client.metrics.MetricsCollector; import com.netflix.conductor.sdk.workflow.executor.task.AnnotatedWorkerExecutor; @@ -51,6 +52,13 @@ public void onApplicationEvent(ContextRefreshedEvent refreshedEvent) { if (beanNames.length > 0) { annotatedWorkerExecutor.setMetricsCollector(applicationContext.getBean(MetricsCollector.class)); } + ClientProperties clientProperties = applicationContext.getBeanProvider(ClientProperties.class).getIfAvailable(); + if (clientProperties != null) { + annotatedWorkerExecutor.setUseVirtualThreads(clientProperties.isUseVirtualThreads()); + annotatedWorkerExecutor.setUpdateV2(clientProperties.isUpdateV2()); + } + applicationContext.getBeanProvider(AdaptiveConcurrencyController.class) + .ifAvailable(annotatedWorkerExecutor::setAdaptiveConcurrencyController); Map beans = applicationContext.getBeansWithAnnotation(Component.class); beans.values().forEach(annotatedWorkerExecutor::addBean); annotatedWorkerExecutor.startPolling(); diff --git a/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/SpringWorkerConfiguration.java b/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/SpringWorkerConfiguration.java index 60de682fb..a196b6490 100644 --- a/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/SpringWorkerConfiguration.java +++ b/conductor-client-spring-boot4/src/main/java/com/netflix/conductor/client/spring/SpringWorkerConfiguration.java @@ -57,6 +57,15 @@ public int getThreadCount(String taskName) { return getProperty(taskName, "threadCount", Integer.class, 0); } + @Override + public int getAdaptiveMaxConcurrency(String taskName) { + Integer maxConcurrency = getProperty(taskName, "adaptiveMaxConcurrency", Integer.class, 0); + if (maxConcurrency == 0) { + maxConcurrency = getProperty(taskName, Worker.PROP_ADAPTIVE_MAX_CONCURRENCY, Integer.class, 0); + } + return maxConcurrency; + } + @Override public String getDomain(String taskName) { return getProperty(taskName, Worker.PROP_DOMAIN, String.class, null); diff --git a/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ClientProperties.java b/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ClientProperties.java index 5e1b5f554..cf3d7cd6a 100644 --- a/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ClientProperties.java +++ b/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ClientProperties.java @@ -18,6 +18,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties; +import com.netflix.conductor.client.automator.AdaptiveConcurrencyConfig; + @ConfigurationProperties("conductor.client") public class ClientProperties { @@ -51,6 +53,229 @@ public void setWrite(int write) { } } + public static class AdaptiveConcurrency { + + // Enables adaptive concurrency for workers started by this client + private boolean enabled = false; + + // Aggregate admission ceiling for all adaptive workers + private int maxProcessConcurrency = 256; + + // Initial process target; 0 uses min(logical queues, available processors) + private int initialProcessConcurrency = 0; + + // Lowest process target an idle process decays to; CPU and heap pressure may go below it + private int minProcessConcurrency = 1; + + // Per-worker ceiling for workers without their own + private int defaultMaxWorkerConcurrency = 64; + + // Ceiling for the number of tasks requested by one poll + private int maxPollBatch = 16; + + private Duration resourceSampleInterval = Duration.ofSeconds(1); + + private Duration metricFreshness = Duration.ofSeconds(5); + + private Duration depthSampleInterval = Duration.ofSeconds(1); + + private Duration depthFreshness = Duration.ofSeconds(5); + + private Duration heartbeatInterval = Duration.ofSeconds(5); + + private Duration standbyPollTimeout = Duration.ofSeconds(1); + + private Duration controlInterval = Duration.ofSeconds(5); + + private double cpuSmoothingAlpha = 0.2; + + private double growthThreshold = 0.75; + + private double cpuPressureThreshold = 0.90; + + private double heapPauseThreshold = 0.90; + + private double heapResumeThreshold = 0.80; + + // Allow running without CPU and heap metrics + private boolean demandOnlyEnabled = false; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public int getMaxProcessConcurrency() { + return maxProcessConcurrency; + } + + public void setMaxProcessConcurrency(int maxProcessConcurrency) { + this.maxProcessConcurrency = maxProcessConcurrency; + } + + public int getInitialProcessConcurrency() { + return initialProcessConcurrency; + } + + public void setInitialProcessConcurrency(int initialProcessConcurrency) { + this.initialProcessConcurrency = initialProcessConcurrency; + } + + public int getMinProcessConcurrency() { + return minProcessConcurrency; + } + + public void setMinProcessConcurrency(int minProcessConcurrency) { + this.minProcessConcurrency = minProcessConcurrency; + } + + public int getDefaultMaxWorkerConcurrency() { + return defaultMaxWorkerConcurrency; + } + + public void setDefaultMaxWorkerConcurrency(int defaultMaxWorkerConcurrency) { + this.defaultMaxWorkerConcurrency = defaultMaxWorkerConcurrency; + } + + public int getMaxPollBatch() { + return maxPollBatch; + } + + public void setMaxPollBatch(int maxPollBatch) { + this.maxPollBatch = maxPollBatch; + } + + public Duration getResourceSampleInterval() { + return resourceSampleInterval; + } + + public void setResourceSampleInterval(Duration resourceSampleInterval) { + this.resourceSampleInterval = resourceSampleInterval; + } + + public Duration getMetricFreshness() { + return metricFreshness; + } + + public void setMetricFreshness(Duration metricFreshness) { + this.metricFreshness = metricFreshness; + } + + public Duration getDepthSampleInterval() { + return depthSampleInterval; + } + + public void setDepthSampleInterval(Duration depthSampleInterval) { + this.depthSampleInterval = depthSampleInterval; + } + + public Duration getDepthFreshness() { + return depthFreshness; + } + + public void setDepthFreshness(Duration depthFreshness) { + this.depthFreshness = depthFreshness; + } + + public Duration getHeartbeatInterval() { + return heartbeatInterval; + } + + public void setHeartbeatInterval(Duration heartbeatInterval) { + this.heartbeatInterval = heartbeatInterval; + } + + public Duration getStandbyPollTimeout() { + return standbyPollTimeout; + } + + public void setStandbyPollTimeout(Duration standbyPollTimeout) { + this.standbyPollTimeout = standbyPollTimeout; + } + + public Duration getControlInterval() { + return controlInterval; + } + + public void setControlInterval(Duration controlInterval) { + this.controlInterval = controlInterval; + } + + public double getCpuSmoothingAlpha() { + return cpuSmoothingAlpha; + } + + public void setCpuSmoothingAlpha(double cpuSmoothingAlpha) { + this.cpuSmoothingAlpha = cpuSmoothingAlpha; + } + + public double getGrowthThreshold() { + return growthThreshold; + } + + public void setGrowthThreshold(double growthThreshold) { + this.growthThreshold = growthThreshold; + } + + public double getCpuPressureThreshold() { + return cpuPressureThreshold; + } + + public void setCpuPressureThreshold(double cpuPressureThreshold) { + this.cpuPressureThreshold = cpuPressureThreshold; + } + + public double getHeapPauseThreshold() { + return heapPauseThreshold; + } + + public void setHeapPauseThreshold(double heapPauseThreshold) { + this.heapPauseThreshold = heapPauseThreshold; + } + + public double getHeapResumeThreshold() { + return heapResumeThreshold; + } + + public void setHeapResumeThreshold(double heapResumeThreshold) { + this.heapResumeThreshold = heapResumeThreshold; + } + + public boolean isDemandOnlyEnabled() { + return demandOnlyEnabled; + } + + public void setDemandOnlyEnabled(boolean demandOnlyEnabled) { + this.demandOnlyEnabled = demandOnlyEnabled; + } + + public AdaptiveConcurrencyConfig toConfig() { + return new AdaptiveConcurrencyConfig.Builder() + .withMaxProcessConcurrency(maxProcessConcurrency) + .withInitialProcessConcurrency(initialProcessConcurrency) + .withMinProcessConcurrency(minProcessConcurrency) + .withDefaultMaxWorkerConcurrency(defaultMaxWorkerConcurrency) + .withMaxPollBatch(maxPollBatch) + .withResourceSampleInterval(resourceSampleInterval) + .withMetricFreshness(metricFreshness) + .withDepthSampleInterval(depthSampleInterval) + .withDepthFreshness(depthFreshness) + .withHeartbeatInterval(heartbeatInterval) + .withStandbyPollTimeout(standbyPollTimeout) + .withControlInterval(controlInterval) + .withCpuSmoothingAlpha(cpuSmoothingAlpha) + .withGrowthThreshold(growthThreshold) + .withCpuPressureThreshold(cpuPressureThreshold) + .withHeapPauseThreshold(heapPauseThreshold) + .withHeapResumeThreshold(heapResumeThreshold) + .withDemandOnlyEnabled(demandOnlyEnabled) + .build(); + } + } + private String rootUri; private String basePath; @@ -75,6 +300,36 @@ public void setWrite(int write) { private boolean verifyingSsl = true; + private boolean useVirtualThreads = false; + + private boolean updateV2 = false; + + private AdaptiveConcurrency adaptiveConcurrency = new AdaptiveConcurrency(); + + public boolean isUseVirtualThreads() { + return useVirtualThreads; + } + + public void setUseVirtualThreads(boolean useVirtualThreads) { + this.useVirtualThreads = useVirtualThreads; + } + + public boolean isUpdateV2() { + return updateV2; + } + + public void setUpdateV2(boolean updateV2) { + this.updateV2 = updateV2; + } + + public AdaptiveConcurrency getAdaptiveConcurrency() { + return adaptiveConcurrency; + } + + public void setAdaptiveConcurrency(AdaptiveConcurrency adaptiveConcurrency) { + this.adaptiveConcurrency = adaptiveConcurrency; + } + public void setVerifyingSsl(boolean verifyingSsl) { this.verifyingSsl = verifyingSsl; } diff --git a/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ConductorClientAutoConfiguration.java b/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ConductorClientAutoConfiguration.java index 0dc00aa29..b947bdeeb 100644 --- a/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ConductorClientAutoConfiguration.java +++ b/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ConductorClientAutoConfiguration.java @@ -24,12 +24,14 @@ import org.springframework.boot.autoconfigure.AutoConfigureOrder; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.core.Ordered; import org.springframework.core.env.Environment; +import com.netflix.conductor.client.automator.AdaptiveConcurrencyController; import com.netflix.conductor.client.automator.TaskRunnerConfigurer; import com.netflix.conductor.client.http.ConductorClient; import com.netflix.conductor.client.http.TaskClient; @@ -78,6 +80,13 @@ public AnnotatedWorkerExecutor annotatedWorkerExecutor(TaskClient taskClient) { return new AnnotatedWorkerExecutor(taskClient); } + @Bean + @ConditionalOnProperty(prefix = "conductor.client.adaptive-concurrency", name = "enabled", havingValue = "true") + @ConditionalOnMissingBean + public AdaptiveConcurrencyController adaptiveConcurrencyController(ClientProperties clientProperties) { + return new AdaptiveConcurrencyController(clientProperties.getAdaptiveConcurrency().toConfig()); + } + @Bean(initMethod = "init", destroyMethod = "shutdown") @ConditionalOnBean(ConductorClient.class) @ConditionalOnMissingBean @@ -85,7 +94,8 @@ public TaskRunnerConfigurer taskRunnerConfigurer(Environment env, TaskClient taskClient, ClientProperties clientProperties, List workers, - Optional metricsCollector) { + Optional metricsCollector, + Optional adaptiveConcurrencyController) { Map taskThreadCount = new HashMap<>(); for (Worker worker : workers) { String key = "conductor.worker." + worker.getTaskDefName() + ".threadCount"; @@ -107,8 +117,22 @@ public TaskRunnerConfigurer taskRunnerConfigurer(Environment env, .withUpdateRetryCount(clientProperties.getUpdateRetryCount()) .withTaskToDomain(clientProperties.getTaskToDomain()) .withShutdownGracePeriodSeconds(clientProperties.getShutdownGracePeriodSeconds()) - .withTaskPollTimeout(clientProperties.getTaskPollTimeout()); + .withTaskPollTimeout(clientProperties.getTaskPollTimeout()) + .withUseVirtualThreads(clientProperties.isUseVirtualThreads()) + .withUpdateV2(clientProperties.isUpdateV2()); metricsCollector.ifPresent(builder::withMetricsCollector); + adaptiveConcurrencyController.ifPresent(controller -> { + var workerConfiguration = new SpringWorkerConfiguration(env); + Map taskAdaptiveMaxConcurrency = new HashMap<>(); + for (Worker worker : workers) { + int maxConcurrency = workerConfiguration.getAdaptiveMaxConcurrency(worker.getTaskDefName()); + if (maxConcurrency > 0) { + taskAdaptiveMaxConcurrency.put(worker.getTaskDefName(), maxConcurrency); + } + } + builder.withAdaptiveConcurrency(controller) + .withTaskAdaptiveMaxConcurrency(taskAdaptiveMaxConcurrency); + }); return builder.build(); } diff --git a/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ConductorWorkerAutoConfiguration.java b/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ConductorWorkerAutoConfiguration.java index 399708e8d..9947488dc 100644 --- a/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ConductorWorkerAutoConfiguration.java +++ b/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/ConductorWorkerAutoConfiguration.java @@ -24,6 +24,7 @@ import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; +import com.netflix.conductor.client.automator.AdaptiveConcurrencyController; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.client.metrics.MetricsCollector; import com.netflix.conductor.sdk.workflow.executor.task.AnnotatedWorkerExecutor; @@ -51,6 +52,13 @@ public void onApplicationEvent(ContextRefreshedEvent refreshedEvent) { if (beanNames.length > 0) { annotatedWorkerExecutor.setMetricsCollector(applicationContext.getBean(MetricsCollector.class)); } + ClientProperties clientProperties = applicationContext.getBeanProvider(ClientProperties.class).getIfAvailable(); + if (clientProperties != null) { + annotatedWorkerExecutor.setUseVirtualThreads(clientProperties.isUseVirtualThreads()); + annotatedWorkerExecutor.setUpdateV2(clientProperties.isUpdateV2()); + } + applicationContext.getBeanProvider(AdaptiveConcurrencyController.class) + .ifAvailable(annotatedWorkerExecutor::setAdaptiveConcurrencyController); Map beans = applicationContext.getBeansWithAnnotation(Component.class); beans.values().forEach(annotatedWorkerExecutor::addBean); annotatedWorkerExecutor.startPolling(); diff --git a/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/SpringWorkerConfiguration.java b/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/SpringWorkerConfiguration.java index 60de682fb..a196b6490 100644 --- a/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/SpringWorkerConfiguration.java +++ b/conductor-client-spring/src/main/java/com/netflix/conductor/client/spring/SpringWorkerConfiguration.java @@ -57,6 +57,15 @@ public int getThreadCount(String taskName) { return getProperty(taskName, "threadCount", Integer.class, 0); } + @Override + public int getAdaptiveMaxConcurrency(String taskName) { + Integer maxConcurrency = getProperty(taskName, "adaptiveMaxConcurrency", Integer.class, 0); + if (maxConcurrency == 0) { + maxConcurrency = getProperty(taskName, Worker.PROP_ADAPTIVE_MAX_CONCURRENCY, Integer.class, 0); + } + return maxConcurrency; + } + @Override public String getDomain(String taskName) { return getProperty(taskName, Worker.PROP_DOMAIN, String.class, null); diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/automator/AdaptiveConcurrencyConfig.java b/conductor-client/src/main/java/com/netflix/conductor/client/automator/AdaptiveConcurrencyConfig.java new file mode 100644 index 000000000..c7815e47b --- /dev/null +++ b/conductor-client/src/main/java/com/netflix/conductor/client/automator/AdaptiveConcurrencyConfig.java @@ -0,0 +1,377 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.client.automator; + +import java.time.Duration; + +/** + * Settings for {@link AdaptiveConcurrencyController}. Defaults are starting points for load testing; see + * {@code design/adaptive-worker-concurrency.md}. + * + *

Create instances with {@link AdaptiveConcurrencyConfig.Builder}. + */ +public class AdaptiveConcurrencyConfig { + + private final int maxProcessConcurrency; + private final int initialProcessConcurrency; + private final int minProcessConcurrency; + private final int defaultMaxWorkerConcurrency; + private final int maxPollBatch; + private final Duration resourceSampleInterval; + private final Duration metricFreshness; + private final Duration depthSampleInterval; + private final Duration depthFreshness; + private final Duration heartbeatInterval; + private final Duration standbyPollTimeout; + private final Duration controlInterval; + private final double cpuSmoothingAlpha; + private final double growthThreshold; + private final double cpuPressureThreshold; + private final double heapPauseThreshold; + private final double heapResumeThreshold; + private final boolean demandOnlyEnabled; + + private AdaptiveConcurrencyConfig(Builder builder) { + this.maxProcessConcurrency = builder.maxProcessConcurrency; + this.initialProcessConcurrency = builder.initialProcessConcurrency; + this.minProcessConcurrency = builder.minProcessConcurrency; + this.defaultMaxWorkerConcurrency = builder.defaultMaxWorkerConcurrency; + this.maxPollBatch = builder.maxPollBatch; + this.resourceSampleInterval = builder.resourceSampleInterval; + this.metricFreshness = builder.metricFreshness; + this.depthSampleInterval = builder.depthSampleInterval; + this.depthFreshness = builder.depthFreshness; + this.heartbeatInterval = builder.heartbeatInterval; + this.standbyPollTimeout = builder.standbyPollTimeout; + this.controlInterval = builder.controlInterval; + this.cpuSmoothingAlpha = builder.cpuSmoothingAlpha; + this.growthThreshold = builder.growthThreshold; + this.cpuPressureThreshold = builder.cpuPressureThreshold; + this.heapPauseThreshold = builder.heapPauseThreshold; + this.heapResumeThreshold = builder.heapResumeThreshold; + this.demandOnlyEnabled = builder.demandOnlyEnabled; + } + + /** + * @return aggregate admission ceiling for all workers sharing the controller + */ + public int getMaxProcessConcurrency() { + return maxProcessConcurrency; + } + + /** + * @return configured initial process target, or 0 to use min(logical queues, available processors); never below + * {@link #getMinProcessConcurrency()} + */ + public int getInitialProcessConcurrency() { + return initialProcessConcurrency; + } + + /** + * @return lowest process target that demand decay settles at; CPU and heap pressure may still go below it + */ + public int getMinProcessConcurrency() { + return minProcessConcurrency; + } + + /** + * @return per-queue ceiling used when a worker does not configure its own + */ + public int getDefaultMaxWorkerConcurrency() { + return defaultMaxWorkerConcurrency; + } + + /** + * @return ceiling for the number of tasks requested by a single poll + */ + public int getMaxPollBatch() { + return maxPollBatch; + } + + public Duration getResourceSampleInterval() { + return resourceSampleInterval; + } + + public Duration getMetricFreshness() { + return metricFreshness; + } + + public Duration getDepthSampleInterval() { + return depthSampleInterval; + } + + public Duration getDepthFreshness() { + return depthFreshness; + } + + public Duration getHeartbeatInterval() { + return heartbeatInterval; + } + + /** + * @return server wait of a standby poll; zero when standby polls are disabled + */ + public Duration getStandbyPollTimeout() { + return standbyPollTimeout; + } + + public Duration getControlInterval() { + return controlInterval; + } + + public double getCpuSmoothingAlpha() { + return cpuSmoothingAlpha; + } + + public double getGrowthThreshold() { + return growthThreshold; + } + + public double getCpuPressureThreshold() { + return cpuPressureThreshold; + } + + public double getHeapPauseThreshold() { + return heapPauseThreshold; + } + + public double getHeapResumeThreshold() { + return heapResumeThreshold; + } + + /** + * @return true if the controller may run without any supported resource metric + */ + public boolean isDemandOnlyEnabled() { + return demandOnlyEnabled; + } + + /** + * Builder used to create instances of {@link AdaptiveConcurrencyConfig}. + */ + public static class Builder { + + private int maxProcessConcurrency = 256; + private int initialProcessConcurrency = 0; + private int minProcessConcurrency = 1; + private int defaultMaxWorkerConcurrency = 64; + private int maxPollBatch = 16; + private Duration resourceSampleInterval = Duration.ofSeconds(1); + private Duration metricFreshness = Duration.ofSeconds(5); + private Duration depthSampleInterval = Duration.ofSeconds(1); + private Duration depthFreshness = Duration.ofSeconds(5); + private Duration heartbeatInterval = Duration.ofSeconds(5); + private Duration standbyPollTimeout = Duration.ofSeconds(1); + private Duration controlInterval = Duration.ofSeconds(5); + private double cpuSmoothingAlpha = 0.2; + private double growthThreshold = 0.75; + private double cpuPressureThreshold = 0.90; + private double heapPauseThreshold = 0.90; + private double heapResumeThreshold = 0.80; + private boolean demandOnlyEnabled; + + /** + * @param maxProcessConcurrency aggregate admission ceiling, defaults to 256 + * @return Builder instance + */ + public Builder withMaxProcessConcurrency(int maxProcessConcurrency) { + this.maxProcessConcurrency = maxProcessConcurrency; + return this; + } + + /** + * @param initialProcessConcurrency initial process target; 0 uses min(logical queues, available processors). + * Raised to the minimum process concurrency when below it + * @return Builder instance + */ + public Builder withInitialProcessConcurrency(int initialProcessConcurrency) { + this.initialProcessConcurrency = initialProcessConcurrency; + return this; + } + + /** + * Keeps the process ready for bursts: an idle process decays to this target instead of one, so the next burst + * does not ramp up from a single slot. Idle queues still hold no threads. CPU and heap pressure may reduce + * the target below it. + * + * @param minProcessConcurrency lowest target demand decay settles at, defaults to 1 + * @return Builder instance + */ + public Builder withMinProcessConcurrency(int minProcessConcurrency) { + this.minProcessConcurrency = minProcessConcurrency; + return this; + } + + /** + * @param defaultMaxWorkerConcurrency per-queue ceiling for workers without their own, defaults to 64 + * @return Builder instance + */ + public Builder withDefaultMaxWorkerConcurrency(int defaultMaxWorkerConcurrency) { + this.defaultMaxWorkerConcurrency = defaultMaxWorkerConcurrency; + return this; + } + + /** + * @param maxPollBatch ceiling for the number of tasks requested by one poll, defaults to 16 + * @return Builder instance + */ + public Builder withMaxPollBatch(int maxPollBatch) { + this.maxPollBatch = maxPollBatch; + return this; + } + + public Builder withResourceSampleInterval(Duration resourceSampleInterval) { + this.resourceSampleInterval = resourceSampleInterval; + return this; + } + + public Builder withMetricFreshness(Duration metricFreshness) { + this.metricFreshness = metricFreshness; + return this; + } + + public Builder withDepthSampleInterval(Duration depthSampleInterval) { + this.depthSampleInterval = depthSampleInterval; + return this; + } + + public Builder withDepthFreshness(Duration depthFreshness) { + this.depthFreshness = depthFreshness; + return this; + } + + public Builder withHeartbeatInterval(Duration heartbeatInterval) { + this.heartbeatInterval = heartbeatInterval; + return this; + } + + /** + * A queue whose share is fully held keeps one single-task long poll open on a slot no share claims, so a task + * that arrives is picked up as it lands instead of at the next depth sample. An idle queue sends about one + * request per timeout. + * + * @param standbyPollTimeout server wait of a standby poll, at most 5 seconds; zero disables standby polls, + * defaults to 1 second + * @return Builder instance + */ + public Builder withStandbyPollTimeout(Duration standbyPollTimeout) { + this.standbyPollTimeout = standbyPollTimeout; + return this; + } + + public Builder withControlInterval(Duration controlInterval) { + this.controlInterval = controlInterval; + return this; + } + + public Builder withCpuSmoothingAlpha(double cpuSmoothingAlpha) { + this.cpuSmoothingAlpha = cpuSmoothingAlpha; + return this; + } + + public Builder withGrowthThreshold(double growthThreshold) { + this.growthThreshold = growthThreshold; + return this; + } + + public Builder withCpuPressureThreshold(double cpuPressureThreshold) { + this.cpuPressureThreshold = cpuPressureThreshold; + return this; + } + + public Builder withHeapPauseThreshold(double heapPauseThreshold) { + this.heapPauseThreshold = heapPauseThreshold; + return this; + } + + public Builder withHeapResumeThreshold(double heapResumeThreshold) { + this.heapResumeThreshold = heapResumeThreshold; + return this; + } + + /** + * @param demandOnlyEnabled allow running when neither CPU nor heap can be measured + * @return Builder instance + */ + public Builder withDemandOnlyEnabled(boolean demandOnlyEnabled) { + this.demandOnlyEnabled = demandOnlyEnabled; + return this; + } + + /** + * @return a validated configuration + * @throws IllegalArgumentException if a value is out of range + */ + public AdaptiveConcurrencyConfig build() { + requirePositive(maxProcessConcurrency, "maxProcessConcurrency"); + requirePositive(defaultMaxWorkerConcurrency, "defaultMaxWorkerConcurrency"); + requirePositive(maxPollBatch, "maxPollBatch"); + if (minProcessConcurrency < 1 || minProcessConcurrency > maxProcessConcurrency) { + throw new IllegalArgumentException("minProcessConcurrency must be between 1 and maxProcessConcurrency"); + } + if (initialProcessConcurrency < 0 || initialProcessConcurrency > maxProcessConcurrency) { + throw new IllegalArgumentException( + "initialProcessConcurrency must be between 0 and maxProcessConcurrency"); + } + requirePositive(resourceSampleInterval, "resourceSampleInterval"); + requirePositive(metricFreshness, "metricFreshness"); + requirePositive(depthSampleInterval, "depthSampleInterval"); + requirePositive(depthFreshness, "depthFreshness"); + requirePositive(heartbeatInterval, "heartbeatInterval"); + requirePositive(controlInterval, "controlInterval"); + if (standbyPollTimeout == null || standbyPollTimeout.isNegative() + || standbyPollTimeout.toMillis() > AdaptiveConcurrencyController.MAX_POLL_TIMEOUT_MILLIS) { + throw new IllegalArgumentException("standbyPollTimeout must be between 0 and 5 seconds"); + } + if (metricFreshness.compareTo(resourceSampleInterval) <= 0) { + throw new IllegalArgumentException("metricFreshness must exceed resourceSampleInterval"); + } + if (depthFreshness.compareTo(depthSampleInterval) <= 0) { + throw new IllegalArgumentException("depthFreshness must exceed depthSampleInterval"); + } + if (!(cpuSmoothingAlpha > 0 && cpuSmoothingAlpha <= 1)) { + throw new IllegalArgumentException("cpuSmoothingAlpha must be in (0, 1]"); + } + requireRatio(growthThreshold, "growthThreshold"); + requireRatio(cpuPressureThreshold, "cpuPressureThreshold"); + requireRatio(heapPauseThreshold, "heapPauseThreshold"); + requireRatio(heapResumeThreshold, "heapResumeThreshold"); + if (!(growthThreshold < heapResumeThreshold && heapResumeThreshold < heapPauseThreshold)) { + throw new IllegalArgumentException( + "Thresholds must satisfy growthThreshold < heapResumeThreshold < heapPauseThreshold"); + } + if (!(growthThreshold < cpuPressureThreshold)) { + throw new IllegalArgumentException("growthThreshold must be below cpuPressureThreshold"); + } + return new AdaptiveConcurrencyConfig(this); + } + + private static void requirePositive(int value, String name) { + if (value < 1) { + throw new IllegalArgumentException(name + " must be positive"); + } + } + + private static void requirePositive(Duration value, String name) { + if (value == null || value.isZero() || value.isNegative()) { + throw new IllegalArgumentException(name + " must be positive"); + } + } + + private static void requireRatio(double value, String name) { + if (!(value > 0 && value <= 1)) { + throw new IllegalArgumentException(name + " must be in (0, 1]"); + } + } + } +} diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/automator/AdaptiveConcurrencyController.java b/conductor-client/src/main/java/com/netflix/conductor/client/automator/AdaptiveConcurrencyController.java new file mode 100644 index 000000000..bd1a8cc37 --- /dev/null +++ b/conductor-client/src/main/java/com/netflix/conductor/client/automator/AdaptiveConcurrencyController.java @@ -0,0 +1,1351 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.client.automator; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.BooleanSupplier; +import java.util.function.LongSupplier; +import java.util.function.Supplier; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.concurrent.BasicThreadFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.conductor.client.events.dispatcher.EventDispatcher; +import com.netflix.conductor.client.events.taskrunner.AdaptiveConcurrencyChanged; +import com.netflix.conductor.client.events.taskrunner.TaskRunnerEvent; +import com.netflix.conductor.client.http.TaskClient; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Coordinates execution concurrency for the workers of a process. One process target bounds aggregate admission and + * is divided among logical worker queues in proportion to their demand, the slots they hold plus their sampled queue + * depth. The target grows under demand while JVM CPU and post-collection heap have headroom, shrinks under pressure, + * and admission pauses when post-collection heap is near the maximum. Pollers reserve capacity before every poll. + * + *

Share one instance among all participating {@link TaskRunnerConfigurer}s of a process. The controller starts + * its sampling threads when the first configurer starts and stops them when the last one shuts down. + * + *

See {@code design/adaptive-worker-concurrency.md}. + */ +public class AdaptiveConcurrencyController { + + private static final Logger LOGGER = LoggerFactory.getLogger(AdaptiveConcurrencyController.class); + + /** Unset timestamp; compares as negative infinity. */ + static final long NONE = Long.MIN_VALUE; + + /** The server rejects batch polls with a longer timeout. */ + static final int MAX_POLL_TIMEOUT_MILLIS = 5000; + + private static final double GROWTH_FRACTION = 0.20; + private static final int MAX_GROWTH_STEP = 16; + private static final double REDUCTION_FACTOR = 0.75; + private static final double JITTER_FRACTION = 0.20; + private static final long CPU_PRESSURE_DURATION = TimeUnit.SECONDS.toNanos(3); + private static final long REDUCTION_INTERVAL = TimeUnit.SECONDS.toNanos(15); + private static final long DECAY_INTERVAL = TimeUnit.SECONDS.toNanos(30); + private static final long DRAINED_RESUME_DELAY = TimeUnit.SECONDS.toNanos(30); + private static final long HEAP_CEILING_HOLD = TimeUnit.SECONDS.toNanos(120); + private static final long HEAP_GROWTH_WINDOW = TimeUnit.SECONDS.toNanos(30); + private static final long MAX_EMPTY_POLL_DELAY = TimeUnit.SECONDS.toNanos(5); + private static final long MAX_RESERVE_WAIT = TimeUnit.SECONDS.toNanos(1); + + enum AdjustmentReason { + STARTUP, + DEPTH_SAMPLE, + EMPTY_POLL, + DEMAND_GROWTH, + DEMAND_DECAY, + CPU_PRESSURE, + HEAP_PRESSURE, + HEAP_PAUSE, + HEAP_RESUME, + DRAINED_RESUME, + REGISTRATION + } + + private enum Wait { + WORKER, + PROCESS, + HEARTBEAT_SLOT + } + + private static final class CpuState { + boolean supported; + double raw; + double smoothed; + long observedAt = NONE; + long pressureSince = NONE; + } + + private static final class HeapState { + boolean supported; + double utilization; + long collectionAt = NONE; + long observedAt = NONE; + long lastHighCollectionAt = NONE; + /** Lowest post-collection utilization seen: the heap the process needs without tasks, or close to it. */ + double baseline = Double.NaN; + /** Collections of the last {@link #HEAP_GROWTH_WINDOW}, oldest first. */ + final ArrayDeque recent = new ArrayDeque<>(); + } + + private record HeapReading(long collectionAt, long observedAt, double utilization) {} + + private final AdaptiveConcurrencyConfig config; + private final Supplier resourceSamplerFactory; + private final LongSupplier clock; + private final boolean scheduleTimers; + private final long metricFreshness; + private final long depthFreshness; + private final long heartbeatInterval; + private final long standbyPollTimeout; + + private final ReentrantLock lock = new ReentrantLock(); + private final Map queues = new LinkedHashMap<>(); + private final CpuState cpu = new CpuState(); + private final HeapState heap = new HeapState(); + + private int registeredRunners; + private boolean running; + private ResourceSampler resourceSampler; + private ScheduledExecutorService controlExecutor; + private ScheduledExecutorService depthExecutor; + private int depthSampleFailures; + /** Due time of the next control tick; read and written only by the control timer. */ + private long nextControlTickAt = NONE; + + private int targetConcurrency = 1; + private int heldSlots; + private boolean admissionPaused; + private long lastResumeAt = NONE; + private long lastReductionAt = NONE; + /** Growth limit after heap pressure: the reduced target, raised one slot per tick after the hold. */ + private int heapGrowthCeiling = Integer.MAX_VALUE; + private long heapCeilingSince = NONE; + private long surplusSince = NONE; + private long waitSequence; + private int publishedProcessTarget = -1; + private boolean publishedAdmissionPaused; + + public AdaptiveConcurrencyController(AdaptiveConcurrencyConfig config) { + this(config, JvmResourceSampler::new, System::nanoTime, true); + } + + @VisibleForTesting + AdaptiveConcurrencyController(AdaptiveConcurrencyConfig config, + Supplier resourceSamplerFactory, + LongSupplier clock, + boolean scheduleTimers) { + this.config = config; + this.resourceSamplerFactory = resourceSamplerFactory; + this.clock = clock; + this.scheduleTimers = scheduleTimers; + this.metricFreshness = config.getMetricFreshness().toNanos(); + this.depthFreshness = config.getDepthFreshness().toNanos(); + this.heartbeatInterval = config.getHeartbeatInterval().toNanos(); + this.standbyPollTimeout = config.getStandbyPollTimeout().toNanos(); + } + + public AdaptiveConcurrencyConfig getConfig() { + return config; + } + + /** + * @return the current process target + */ + public int getTargetConcurrency() { + lock.lock(); + try { + return targetConcurrency; + } finally { + lock.unlock(); + } + } + + /** + * @return slots held by outstanding polls and accepted tasks across all workers + */ + public int getHeldSlots() { + lock.lock(); + try { + return heldSlots; + } finally { + lock.unlock(); + } + } + + /** + * @return true while admission is paused by heap pressure + */ + public boolean isAdmissionPaused() { + lock.lock(); + try { + return admissionPaused; + } finally { + lock.unlock(); + } + } + + // ----- Registration and lifecycle ----- + + /** + * Joins or creates the logical queue identified by client scope, task type and resolved domain. Every poller + * registers once; pollers of the same logical queue share its share and execution pool. + */ + WorkerQueue register(TaskClient taskClient, + String taskType, + String domain, + int maximumConcurrency, + int pollingIntervalMillis, + BooleanSupplier pollable, + ThreadFactory threadFactory, + boolean useVirtualThreads, + EventDispatcher eventDispatcher) { + lock.lock(); + try { + var key = new WorkerQueue.Key(taskClient, taskType, StringUtils.trimToNull(domain)); + WorkerQueue queue = queues.get(key); + if (queue == null) { + queue = new WorkerQueue(key, maximumConcurrency, pollingIntervalMillis, pollable, threadFactory, + useVirtualThreads, eventDispatcher, lock.newCondition(), clock.getAsLong()); + queues.put(key, queue); + if (running) { + applyTargets(clock.getAsLong(), AdjustmentReason.REGISTRATION, false); + } + } else { + // The newest registration wins, so a restarted configurer replaces the one it supersedes. + queue.updateRegistration(pollable, eventDispatcher); + } + queue.registrations++; + registeredRunners++; + return queue; + } finally { + lock.unlock(); + } + } + + /** + * Starts sampling after a batch of registrations. The initial process target is computed from the queues + * registered at this moment; later calls while running only re-divide the target. + * + * @throws IllegalStateException if neither CPU nor heap can be measured and demand-only operation is disabled + */ + void activate() { + lock.lock(); + try { + if (running || registeredRunners == 0) { + return; + } + ResourceSampler sampler = resourceSamplerFactory.get(); + cpu.supported = sampler.isCpuSupported(); + heap.supported = sampler.isHeapSupported(); + if (!cpu.supported) { + LOGGER.warn("JVM process CPU is not measurable; adaptive concurrency runs without the CPU signal"); + } + if (!heap.supported) { + LOGGER.warn("Post-collection heap is not measurable; adaptive concurrency runs without the heap signal"); + } + if (!cpu.supported && !heap.supported) { + if (!config.isDemandOnlyEnabled()) { + sampler.close(); + throw new IllegalStateException("Adaptive concurrency needs JVM CPU or heap metrics and neither is " + + "supported; enable demand-only operation to run without resource gating"); + } + LOGGER.warn("Adaptive concurrency is running in demand-only mode without resource gating"); + } + resourceSampler = sampler; + + int initial = config.getInitialProcessConcurrency(); + if (initial == 0) { + initial = Math.min(queues.size(), Runtime.getRuntime().availableProcessors()); + } + targetConcurrency = clamp(initial, config.getMinProcessConcurrency(), config.getMaxProcessConcurrency()); + admissionPaused = false; + lastResumeAt = NONE; + lastReductionAt = NONE; + surplusSince = NONE; + resetCpu(); + resetHeap(); + running = true; + LOGGER.info("Adaptive concurrency started with process target {} for {} queues, ceiling {}", + targetConcurrency, queues.size(), config.getMaxProcessConcurrency()); + applyTargets(clock.getAsLong(), AdjustmentReason.STARTUP, true); + if (scheduleTimers) { + startTimers(); + } + } finally { + lock.unlock(); + } + } + + /** + * Removes one poller registration. + * + * @return true if this was the last registration of the queue, so its execution pool may be shut down + */ + boolean deregister(WorkerQueue queue) { + lock.lock(); + try { + queue.registrations--; + registeredRunners = Math.max(0, registeredRunners - 1); + boolean last = queue.registrations <= 0; + if (last) { + queues.remove(queue.key, queue); + } + queue.slotAvailable.signalAll(); + if (registeredRunners == 0) { + stop(); + } else if (last && running) { + applyTargets(clock.getAsLong(), AdjustmentReason.REGISTRATION, false); + } + return last; + } finally { + lock.unlock(); + } + } + + private void stop() { + if (!running) { + return; + } + running = false; + if (controlExecutor != null) { + controlExecutor.shutdownNow(); + controlExecutor = null; + } + if (depthExecutor != null) { + depthExecutor.shutdownNow(); + depthExecutor = null; + } + if (resourceSampler != null) { + resourceSampler.close(); + resourceSampler = null; + } + LOGGER.info("Adaptive concurrency stopped"); + } + + private void startTimers() { + controlExecutor = Executors.newSingleThreadScheduledExecutor(daemonThreadFactory("adaptive-concurrency-control-%d")); + depthExecutor = Executors.newSingleThreadScheduledExecutor(daemonThreadFactory("adaptive-concurrency-depth-%d")); + controlExecutor.scheduleWithFixedDelay(guarded(this::sampleResources, "resource sampling"), + 0, config.getResourceSampleInterval().toNanos(), TimeUnit.NANOSECONDS); + // A periodic task survives a failed run, where a tick that reschedules itself stops for good if the + // rescheduling fails, as it did when the heap ran out. The jittered due time keeps processes apart. + long controlInterval = config.getControlInterval().toNanos(); + nextControlTickAt = clock.getAsLong() + jitter(controlInterval); + controlExecutor.scheduleWithFixedDelay(guarded(this::controlTickIfDue, "control tick"), + controlInterval / 20, controlInterval / 20, TimeUnit.NANOSECONDS); + depthExecutor.scheduleWithFixedDelay(guarded(this::sampleDepths, "queue depth sampling"), + 0, config.getDepthSampleInterval().toNanos(), TimeUnit.NANOSECONDS); + } + + private void controlTickIfDue() { + long now = clock.getAsLong(); + if (now < nextControlTickAt) { + return; + } + nextControlTickAt = now + jitter(config.getControlInterval().toNanos()); + controlTick(); + } + + // ----- Poll path ----- + + /** + * Reserves capacity for one poll, waiting up to one second for a slot. A poller whose share is held is granted a + * single-task standby poll on capacity no share claims, or else a heartbeat when one is due and a slot is open. + * + * @param active checked before every attempt; the wait ends when it returns false + * @return the reservation, or null if none was granted within the wait + */ + SlotReservation reserve(WorkerQueue queue, BooleanSupplier active) { + lock.lock(); + boolean waitingForProcess = false; + boolean waitingForHeartbeat = false; + try { + long remaining = MAX_RESERVE_WAIT; + while (active.getAsBoolean() && queue.registrations > 0) { + long now = clock.getAsLong(); + Object outcome = tryReserveLocked(queue, now); + if (outcome instanceof SlotReservation reservation) { + return reservation; + } + boolean heartbeatWait = outcome == Wait.HEARTBEAT_SLOT; + if (heartbeatWait != waitingForHeartbeat) { + queue.heartbeatWaiters += heartbeatWait ? 1 : -1; + waitingForHeartbeat = heartbeatWait; + } + boolean processWait = outcome == Wait.PROCESS; + if (processWait != waitingForProcess) { + if (processWait) { + if (queue.processWaiters++ == 0) { + queue.processWaitSequence = ++waitSequence; + } + } else { + queue.processWaiters--; + } + waitingForProcess = processWait; + } + long waitNanos = remaining; + if (outcome == Wait.WORKER && queue.nextHeartbeatAt > now) { + waitNanos = Math.min(waitNanos, queue.nextHeartbeatAt - now); + } + if (outcome == Wait.WORKER && standbyPollTimeout > 0 && queue.nextStandbyAt > now) { + waitNanos = Math.min(waitNanos, queue.nextStandbyAt - now); + } + if (waitNanos <= 0) { + return null; + } + long waitStarted = System.nanoTime(); + queue.waiters++; + try { + queue.slotAvailable.awaitNanos(waitNanos); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } finally { + queue.waiters--; + } + remaining -= System.nanoTime() - waitStarted; + } + return null; + } finally { + if (waitingForProcess) { + queue.processWaiters--; + } + if (waitingForHeartbeat) { + queue.heartbeatWaiters--; + } + lock.unlock(); + } + } + + /** + * One reservation attempt under the lock. + * + * @return a {@link SlotReservation}, or the resource the poller must wait for + */ + @VisibleForTesting + Object tryReserve(WorkerQueue queue, long now) { + lock.lock(); + try { + return tryReserveLocked(queue, now); + } finally { + lock.unlock(); + } + } + + private Object tryReserveLocked(WorkerQueue queue, long now) { + // Rule: a queue is polled only with an open slot for every task the poll may return. + if (!running || admissionPaused) { + return Wait.PROCESS; + } + int processAvailable = Math.max(0, targetConcurrency - heldSlots); + int workerAvailable = Math.max(0, queue.deficit()); + if (workerAvailable == 0) { + SlotReservation standby = tryReserveStandbyLocked(queue, now); + return standby != null ? standby : tryReserveHeartbeatLocked(queue, now, processAvailable); + } + // Due heartbeats go first: one slot each, briefly, at most once per interval per queue. + int forHeartbeats = dueHeartbeatWaiters(now, queue); + if (processAvailable <= forHeartbeats) { + WorkerQueue heartbeat = oldestDueHeartbeatWaiter(now, queue); + if (heartbeat != null) { + heartbeat.slotAvailable.signal(); + } + return Wait.PROCESS; + } + WorkerQueue needier = bestProcessWaiter(queue); + if (needier != null && outranks(needier, queue)) { + // The slot belongs to a queue further below its share. + needier.slotAvailable.signal(); + return Wait.PROCESS; + } + int count = Math.min(Math.min(workerAvailable, processAvailable - forHeartbeats), config.getMaxPollBatch()); + queue.heldSlots += count; + heldSlots += count; + queue.reservedPollSlots += count; + queue.outstandingPolls++; + queue.nextHeartbeatAt = now + heartbeatInterval; + queue.resizePool(); + if (targetConcurrency > heldSlots) { + // Capacity remains: pass the turn on so it is not left idle until a waiter's timeout. + if (queue.deficit() > 0 && queue.waiters > 0) { + queue.slotAvailable.signal(); + } + WorkerQueue next = bestProcessWaiter(queue); + if (next != null) { + next.slotAvailable.signal(); + } + } + return new SlotReservation(queue, count, SlotReservation.Kind.SHARE); + } + + /** + * Share held: keep one single-task long poll open so a task is picked up as it arrives, instead of at the next + * depth sample. It uses only a slot that no queue's share claims, after due heartbeats, so it never delays a + * queue's share by more than the standby timeout. Control counts one standby slot per queue with nothing waiting + * as demand, so the process target leaves room for them while resources allow. + */ + private SlotReservation tryReserveStandbyLocked(WorkerQueue queue, long now) { + // Like every poll it also needs an open process slot: unclaimed slots are computed over registered + // queues, which can overstate free capacity while a deregistered worker's tasks drain. + if (standbyPollTimeout == 0 + || queue.outstandingPolls > 0 + || now < queue.nextStandbyAt + || queue.heldSlots >= queue.maximumConcurrency + || heldSlots >= targetConcurrency + || unclaimedSlots() <= dueHeartbeatWaiters(now, queue)) { + return null; + } + queue.heldSlots++; + heldSlots++; + queue.reservedPollSlots++; + queue.outstandingPolls++; + queue.standbyPolls++; + // It refreshes the server's active-poller window, as a heartbeat would. + queue.nextHeartbeatAt = now + heartbeatInterval; + queue.nextStandbyAt = now + standbyPollTimeout; + queue.resizePool(); + return new SlotReservation(queue, 1, SlotReservation.Kind.STANDBY); + } + + /** + * @return process slots neither held nor promised to a queue's share + */ + private int unclaimedSlots() { + long claimed = 0; + for (WorkerQueue candidate : queues.values()) { + claimed += Math.max(candidate.targetConcurrency, candidate.heldSlots); + } + return (int) Math.max(0, targetConcurrency - claimed); + } + + /** + * No share, or the share is already held: only a heartbeat, at most once per interval. Like every poll it needs an + * open slot, and it requests one task for it without waiting, so it holds the slot only for the request. A due + * heartbeat stays due until it gets a slot and goes first when one opens, so idle queues keep the server's + * active-poller window, which domain routing depends on, whenever slots turn over. With every slot held it waits. + */ + private Object tryReserveHeartbeatLocked(WorkerQueue queue, long now, int processAvailable) { + if (queue.outstandingPolls > 0 || now < queue.nextHeartbeatAt) { + return Wait.WORKER; + } + if (processAvailable == 0 || queue.heldSlots >= queue.maximumConcurrency) { + return Wait.HEARTBEAT_SLOT; + } + WorkerQueue older = oldestDueHeartbeatWaiter(now, queue); + if (older != null && older.nextHeartbeatAt < queue.nextHeartbeatAt && processAvailable == 1) { + older.slotAvailable.signal(); + return Wait.HEARTBEAT_SLOT; + } + queue.heldSlots++; + heldSlots++; + queue.reservedPollSlots++; + queue.outstandingPolls++; + queue.nextHeartbeatAt = now + heartbeatInterval; + queue.resizePool(); + return new SlotReservation(queue, 1, SlotReservation.Kind.HEARTBEAT); + } + + private boolean isDueHeartbeatWaiter(WorkerQueue queue, long now) { + return queue.heartbeatWaiters > 0 + && queue.outstandingPolls == 0 + && queue.nextHeartbeatAt <= now + && queue.heldSlots < queue.maximumConcurrency; + } + + private int dueHeartbeatWaiters(long now, WorkerQueue exclude) { + int count = 0; + for (WorkerQueue candidate : queues.values()) { + if (candidate != exclude && isDueHeartbeatWaiter(candidate, now)) { + count++; + } + } + return count; + } + + private WorkerQueue oldestDueHeartbeatWaiter(long now, WorkerQueue exclude) { + WorkerQueue oldest = null; + for (WorkerQueue candidate : queues.values()) { + if (candidate != exclude && isDueHeartbeatWaiter(candidate, now) + && (oldest == null || candidate.nextHeartbeatAt < oldest.nextHeartbeatAt)) { + oldest = candidate; + } + } + return oldest; + } + + /** + * Settles a poll: slots not matched by returned tasks are released, the returned tasks keep theirs. + */ + void completePoll(SlotReservation reservation, int delivered, long pollStartedAt) { + settlePoll(reservation, delivered, pollStartedAt, false); + } + + /** + * Settles a failed poll. A poll error is an empty response for backoff and demand. + */ + void failPoll(SlotReservation reservation, long pollStartedAt) { + settlePoll(reservation, 0, pollStartedAt, true); + } + + private void settlePoll(SlotReservation reservation, int delivered, long pollStartedAt, boolean failed) { + lock.lock(); + try { + WorkerQueue queue = reservation.queue(); + long now = clock.getAsLong(); + boolean hadTaskSlots = queue.heldSlots - queue.reservedPollSlots > 0; + int unused = reservation.count() - delivered; + queue.heldSlots -= unused; + heldSlots -= unused; + queue.reservedPollSlots -= reservation.count(); + queue.outstandingPolls--; + if (reservation.standby()) { + queue.standbyPolls--; + } + if (!hadTaskSlots && delivered > 0) { + // The slot-age observation restarts when an idle queue accepts work again. + queue.lastTaskSlotReleaseAt = now; + } + guardCounters(queue); + // A standby wait that ran out on an empty queue is expected: nextStandbyAt paces it, not the backoff. + boolean standbyExpired = reservation.standby() && delivered == 0 && !failed; + if (delivered == 0 && !standbyExpired) { + queue.consecutiveEmptyPolls++; + queue.depthAtEmptyPoll = elapsed(now, queue.depthObservedAt) <= depthFreshness ? queue.depth : 0; + queue.nextPollAllowedAt = Math.max(now, pollStartedAt + emptyPollDelay(queue)); + if (running) { + // The share now counts at most one waiting task until a delivery. + applyTargets(now, AdjustmentReason.EMPTY_POLL, false); + } + } else if (delivered > 0) { + queue.consecutiveEmptyPolls = 0; + queue.nextPollAllowedAt = now; + queue.nextStandbyAt = now; + } + if (unused > 0) { + queue.resizePool(); + wakeAfterRelease(queue); + } + } finally { + lock.unlock(); + } + } + + /** + * Releases the slot owned by a finished or rejected task. + */ + void releaseTaskSlot(WorkerQueue queue) { + lock.lock(); + try { + queue.heldSlots--; + heldSlots--; + queue.lastTaskSlotReleaseAt = clock.getAsLong(); + guardCounters(queue); + queue.resizePool(); + wakeAfterRelease(queue); + } finally { + lock.unlock(); + } + } + + /** + * Invariant backstop: a counter driven negative by a release bug would inflate deficits and over-admit, the one + * direction the design forbids, so a breach is clamped and reported instead of compounding silently. + */ + private void guardCounters(WorkerQueue queue) { + if (queue.heldSlots < 0 || queue.outstandingPolls < 0 || queue.standbyPolls < 0 + || queue.reservedPollSlots < 0 || heldSlots < 0) { + LOGGER.error("Slot accounting breach for {}: heldSlots={}, outstandingPolls={}, standbyPolls={}, " + + "reservedPollSlots={}, processHeldSlots={}; clamping to zero", + queue.queueName, queue.heldSlots, queue.outstandingPolls, queue.standbyPolls, + queue.reservedPollSlots, heldSlots); + queue.heldSlots = Math.max(0, queue.heldSlots); + queue.outstandingPolls = Math.max(0, queue.outstandingPolls); + queue.standbyPolls = Math.max(0, queue.standbyPolls); + queue.reservedPollSlots = Math.max(0, queue.reservedPollSlots); + heldSlots = Math.max(0, heldSlots); + } + } + + /** + * @return true if a finishing task may keep its slot for a task returned by update-and-fetch + */ + boolean isReplacementAllowed(WorkerQueue queue) { + lock.lock(); + try { + if (!running || admissionPaused) { + return false; + } + if (queue.heldSlots - queue.standbyPolls > queue.targetConcurrency || heldSlots > targetConcurrency) { + return false; + } + // The freed slot would be granted back to this queue anyway. + WorkerQueue needier = bestProcessWaiter(queue); + return needier == null || queue.deficit() + 1 >= needier.deficit(); + } finally { + lock.unlock(); + } + } + + /** + * Records a task delivered by update-and-fetch; it reuses the finishing task's slot. + */ + void recordReplacement(WorkerQueue queue) { + lock.lock(); + try { + queue.consecutiveEmptyPolls = 0; + // A replacement means a task just finished and handed over its slot: release activity for the slot age. + queue.lastTaskSlotReleaseAt = clock.getAsLong(); + } finally { + lock.unlock(); + } + } + + /** + * @return nanoseconds until the queue may poll again after an empty response or poll error + */ + long pollDelayNanos(WorkerQueue queue) { + lock.lock(); + try { + return Math.max(0, queue.nextPollAllowedAt - clock.getAsLong()); + } finally { + lock.unlock(); + } + } + + long now() { + return clock.getAsLong(); + } + + // ----- Monitoring ----- + + @VisibleForTesting + void sampleResources() { + ResourceSampler sampler; + boolean cpuSupported; + boolean heapSupported; + lock.lock(); + try { + sampler = resourceSampler; + cpuSupported = cpu.supported; + heapSupported = heap.supported; + } finally { + lock.unlock(); + } + if (sampler == null) { + return; + } + // MXBean calls run outside the lock; each metric is attempted independently. + double cpuReading = cpuSupported ? sampler.sampleCpu() : Double.NaN; + ResourceSampler.HeapSample heapReading = heapSupported ? sampler.sampleHeap() : null; + + lock.lock(); + try { + if (!running) { + return; + } + long now = clock.getAsLong(); + if (cpu.supported) { + if (Double.isFinite(cpuReading) && cpuReading >= 0 && cpuReading <= 1) { + if (elapsed(now, cpu.observedAt) > metricFreshness) { + cpu.smoothed = cpuReading; + cpu.pressureSince = NONE; + } else { + double alpha = config.getCpuSmoothingAlpha(); + cpu.smoothed = alpha * cpuReading + (1 - alpha) * cpu.smoothed; + } + cpu.raw = cpuReading; + cpu.observedAt = now; + if (cpuReading >= config.getCpuPressureThreshold()) { + if (cpu.pressureSince == NONE) { + cpu.pressureSince = now; + } + } else { + cpu.pressureSince = NONE; + } + } else { + cpu.observedAt = NONE; + cpu.pressureSince = NONE; + } + } + if (heap.supported) { + if (heapReading != null) { + heap.utilization = heapReading.utilization(); + heap.collectionAt = heapReading.collectionAtNanos(); + heap.observedAt = now; + if (heap.collectionAt != NONE && heap.utilization >= config.getHeapPauseThreshold()) { + heap.lastHighCollectionAt = heap.collectionAt; + } + if (heap.collectionAt != NONE) { + heap.baseline = Double.isNaN(heap.baseline) + ? heap.utilization + : Math.min(heap.baseline, heap.utilization); + HeapReading last = heap.recent.peekLast(); + if (last == null || last.collectionAt() != heap.collectionAt) { + heap.recent.addLast(new HeapReading(heap.collectionAt, now, heap.utilization)); + } + } + } else { + heap.observedAt = NONE; + } + } + reconcileHeapPause(now); + } finally { + lock.unlock(); + } + } + + private void reconcileHeapPause(long now) { + boolean heapFresh = isFresh(heap.supported, heap.observedAt, now, metricFreshness); + if (!admissionPaused) { + if (heapFresh + && heap.collectionAt != NONE + && heap.utilization >= config.getHeapPauseThreshold() + && heap.collectionAt > lastResumeAt) { + admissionPaused = true; + limitHeapGrowth((int) Math.floor(REDUCTION_FACTOR * Math.max(targetConcurrency, heldSlots)), now); + LOGGER.warn("Post-collection heap at {}%; pausing task acquisition", + Math.round(heap.utilization * 100)); + applyTargets(now, AdjustmentReason.HEAP_PAUSE, false); + } + return; + } + + boolean drained; + if (heapFresh && heap.utilization < config.getHeapResumeThreshold()) { + drained = false; + } else if (heldSlots == 0 && elapsed(now, heap.lastHighCollectionAt) >= DRAINED_RESUME_DELAY) { + // Idle with a stale estimate: resume at one so ordinary allocation triggers a fresh collection. + drained = true; + targetConcurrency = 1; + } else { + return; + } + admissionPaused = false; + // Pause re-entry needs a collection newer than this. + lastResumeAt = drained ? now : heap.collectionAt; + LOGGER.info("Resuming task acquisition at process target {}", targetConcurrency); + applyTargets(now, drained ? AdjustmentReason.DRAINED_RESUME : AdjustmentReason.HEAP_RESUME, false); + } + + @VisibleForTesting + void controlTick() { + lock.lock(); + try { + if (!running) { + return; + } + long now = clock.getAsLong(); + reconcileHeapPause(now); + + boolean cpuFresh = isFresh(cpu.supported, cpu.observedAt, now, metricFreshness); + boolean heapFresh = isFresh(heap.supported, heap.observedAt, now, metricFreshness); + boolean cpuPressure = cpuFresh + && cpu.pressureSince != NONE + && now - cpu.pressureSince >= CPU_PRESSURE_DURATION; + boolean heapPressure = heap.supported && heap.lastHighCollectionAt > lastReductionAt; + boolean reductionIntervalElapsed = elapsed(now, lastReductionAt) >= REDUCTION_INTERVAL; + + if ((cpuPressure || heapPressure) && reductionIntervalElapsed) { + targetConcurrency = Math.max(1, (int) Math.floor(REDUCTION_FACTOR * targetConcurrency)); + lastReductionAt = now; + if (heapPressure) { + limitHeapGrowth(targetConcurrency, now); + } + if (cpu.pressureSince != NONE) { + // The next reduction needs a new run of pressure. + cpu.pressureSince = now; + } + applyTargets(now, cpuPressure ? AdjustmentReason.CPU_PRESSURE : AdjustmentReason.HEAP_PRESSURE, false); + return; + } + + long totalDemand = totalDemand(now); + AdjustmentReason reason = null; + if (totalDemand >= targetConcurrency) { + surplusSince = NONE; + } else if (surplusSince == NONE) { + surplusSince = now; + } else if (now - surplusSince >= DECAY_INTERVAL) { + long halved = (targetConcurrency + 1) / 2; + long decayed = Math.max(config.getMinProcessConcurrency(), Math.max(totalDemand, halved)); + // Decay only lowers: a target that pressure took below the minimum stays there until growth. + targetConcurrency = (int) Math.min(targetConcurrency, decayed); + surplusSince = now; + reason = AdjustmentReason.DEMAND_DECAY; + } + + double heapForGrowth = heapGrowthUtilization(now); + boolean growthAllowed = totalDemand > targetConcurrency + && !admissionPaused + && reductionIntervalElapsed + && (!cpu.supported || (cpuFresh + && cpu.smoothed < config.getGrowthThreshold() + && cpu.raw < config.getCpuPressureThreshold())) + && (!heap.supported || (heapFresh && heapForGrowth < config.getGrowthThreshold())); + relaxHeapGrowthCeiling(now); + int increment = growthAllowed ? growthIncrement(heapFresh, heapForGrowth) : 0; + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Control tick: target {}, demand {}, held {} ({} standby), cpu {}/{}, heap {} (peak {}, " + + "baseline {}, fresh {}), ceiling {}, increment {}", + targetConcurrency, totalDemand, heldSlots, standbyPolls(), cpu.raw, cpu.smoothed, + heap.utilization, heapForGrowth, heap.baseline, heapFresh, heapGrowthCeiling, increment); + } + if (increment > 0) { + long grown = Math.min(totalDemand, (long) targetConcurrency + increment); + targetConcurrency = (int) Math.min(config.getMaxProcessConcurrency(), grown); + reason = AdjustmentReason.DEMAND_GROWTH; + } + if (reason != null) { + applyTargets(now, reason, false); + } + } finally { + lock.unlock(); + } + } + + /** + * Post-collection utilization for growth decisions: the highest of the last 30 seconds. A single reading depends + * on which task payloads a collection happened to find live or promote, and end to end a low one let the target + * grow into an OutOfMemoryError seconds later. + */ + private double heapGrowthUtilization(long now) { + while (!heap.recent.isEmpty() && now - heap.recent.peekFirst().observedAt() > HEAP_GROWTH_WINDOW) { + heap.recent.removeFirst(); + } + double peak = heap.utilization; + for (HeapReading reading : heap.recent) { + peak = Math.max(peak, reading.utilization()); + } + return peak; + } + + /** + * The growth step: a fraction of the target, limited by the ceiling left by heap pressure and by the slots whose + * observed heap cost still fits under the growth threshold. + */ + private int growthIncrement(boolean heapFresh, double heapUtilization) { + int increment = clamp((int) Math.ceil(GROWTH_FRACTION * targetConcurrency), 1, MAX_GROWTH_STEP); + increment = (int) Math.min(increment, (long) heapGrowthCeiling - targetConcurrency); + // Standby slots hold no task, so they carry none of the heap. + int taskSlots = heldSlots - standbyPolls(); + if (heap.supported && heapFresh && taskSlots > 0 && heapUtilization > heap.baseline) { + double perSlot = (heapUtilization - heap.baseline) / taskSlots; + int room = (int) Math.floor((config.getGrowthThreshold() - heapUtilization) / perSlot); + increment = Math.min(increment, room); + } + return Math.max(0, increment); + } + + private int standbyPolls() { + int total = 0; + for (WorkerQueue queue : queues.values()) { + total += queue.standbyPolls; + } + return total; + } + + private void limitHeapGrowth(int ceiling, long now) { + heapGrowthCeiling = Math.max(1, Math.min(heapGrowthCeiling, ceiling)); + heapCeilingSince = now; + } + + private void relaxHeapGrowthCeiling(long now) { + if (heapGrowthCeiling == Integer.MAX_VALUE || elapsed(now, heapCeilingSince) < HEAP_CEILING_HOLD) { + return; + } + heapGrowthCeiling = heapGrowthCeiling >= config.getMaxProcessConcurrency() + ? Integer.MAX_VALUE + : heapGrowthCeiling + 1; + } + + /** + * Samples the depth of every logical queue with one {@code GET /tasks/queue/sizes} request per client scope. + * Worker pause and poll filters are evaluated here, outside the lock; a worker that would not poll has a depth + * of zero. + */ + @VisibleForTesting + void sampleDepths() { + List snapshot; + lock.lock(); + try { + if (!running) { + return; + } + snapshot = new ArrayList<>(queues.values()); + } finally { + lock.unlock(); + } + + Map pollable = new HashMap<>(); + Map> byClient = new LinkedHashMap<>(); + for (WorkerQueue queue : snapshot) { + boolean canPoll = queue.isPollable(); + pollable.put(queue, canPoll); + if (canPoll) { + byClient.computeIfAbsent(queue.key.taskClient(), c -> new ArrayList<>()).add(queue); + } + } + Map sizes = new HashMap<>(); + for (Map.Entry> entry : byClient.entrySet()) { + var names = new LinkedHashSet(); + entry.getValue().forEach(queue -> names.add(queue.queueName)); + try { + Map result = entry.getKey().getQueueSizesForTasks(new ArrayList<>(names)); + if (result == null) { + throw new IllegalStateException("Queue sizes response was empty"); + } + for (WorkerQueue queue : entry.getValue()) { + Integer size = result.get(queue.queueName); + sizes.put(queue, size == null ? 0 : size); + } + depthSampleFailures = 0; + } catch (Exception e) { + if (depthSampleFailures < 3 || depthSampleFailures % 60 == 0) { + LOGGER.warn("Unable to sample queue depth for {}; depths count as unknown", names, e); + } + depthSampleFailures++; + } + } + + lock.lock(); + try { + long now = clock.getAsLong(); + for (WorkerQueue queue : snapshot) { + queue.pollableAtSample = pollable.get(queue); + if (!pollable.get(queue)) { + queue.depth = 0; + queue.depthObservedAt = now; + } else if (sizes.containsKey(queue)) { + queue.depth = Math.max(0, sizes.get(queue)); + queue.depthObservedAt = now; + if (queue.consecutiveEmptyPolls > 0 && queue.depth > queue.depthAtEmptyPoll) { + // Tasks arrived since the last empty response: poll now rather than after the backoff. + queue.nextPollAllowedAt = Math.min(queue.nextPollAllowedAt, now); + } + } else { + queue.depthObservedAt = NONE; + } + } + if (running) { + applyTargets(now, AdjustmentReason.DEPTH_SAMPLE, true); + } + } finally { + lock.unlock(); + } + } + + // ----- Allocation ----- + + /** + * @return the queue's backlog: the slots it holds for tasks plus its sampled depth, uncapped. Standby slots do + * not count until they deliver a task. + */ + private long backlog(WorkerQueue queue, long now) { + // Unknown depth counts as one. + long depth = elapsed(now, queue.depthObservedAt) <= depthFreshness ? queue.depth : 1; + if (queue.consecutiveEmptyPolls > 0) { + // The server just delivered nothing from this depth: count one task, plus any that arrived since. + depth = Math.min(depth, 1 + Math.max(0, depth - queue.depthAtEmptyPoll)); + } + return Math.max(0, queue.heldSlots - queue.standbyPolls) + depth; + } + + /** + * @return the slots the queue could use: its backlog, capped at its maximum concurrency + */ + private long demand(WorkerQueue queue, long now) { + return Math.min(queue.maximumConcurrency, backlog(queue, now)); + } + + /** + * @return one for a pollable queue with nothing waiting beyond the slots it holds and room for one more task: the + * slot of its standby poll + */ + private int standbyDemand(WorkerQueue queue, long now) { + if (standbyPollTimeout == 0 || !queue.pollableAtSample) { + return 0; + } + long held = Math.max(0, queue.heldSlots - queue.standbyPolls); + return backlog(queue, now) == held && held < queue.maximumConcurrency ? 1 : 0; + } + + /** + * @return the slots all queues could use, including a standby slot for each queue that has nothing waiting + */ + private long totalDemand(long now) { + long total = 0; + for (WorkerQueue queue : queues.values()) { + total += demand(queue, now) + standbyDemand(queue, now); + } + return total; + } + + /** + * Divides the process target among queues in proportion to backlog, so a deeper queue gets more slots even + * beyond its maximum concurrency. A share never exceeds the queue's demand: a queue that reaches it keeps + * exactly its demand and the rest is re-divided among the others. Every queue with demand gets at least one + * slot, then largest-remainder rounding keeps the shares summing to the target. + */ + private void allocate(long now) { + WorkerQueue[] all = queues.values().toArray(new WorkerQueue[0]); + int n = all.length; + if (n == 0) { + return; + } + long[] backlogs = new long[n]; + long[] demands = new long[n]; + long totalDemand = 0; + for (int i = 0; i < n; i++) { + backlogs[i] = backlog(all[i], now); + demands[i] = Math.min(all[i].maximumConcurrency, backlogs[i]); + totalDemand += demands[i]; + } + if (totalDemand <= targetConcurrency) { + for (int i = 0; i < n; i++) { + all[i].targetConcurrency = (int) demands[i]; + } + return; + } + int[] shares = new int[n]; + double[] quotas = new double[n]; + boolean[] capped = new boolean[n]; + long remaining = targetConcurrency; + boolean cappedAny = true; + while (cappedAny) { + // Water-filling: queues whose proportional quota reaches their demand are fixed at it. + cappedAny = false; + long weight = 0; + for (int i = 0; i < n; i++) { + if (!capped[i]) { + weight += backlogs[i]; + } + } + for (int i = 0; i < n; i++) { + if (capped[i] || demands[i] == 0) { + continue; + } + quotas[i] = (double) remaining * backlogs[i] / weight; + if (quotas[i] >= demands[i]) { + capped[i] = true; + shares[i] = (int) demands[i]; + remaining -= demands[i]; + cappedAny = true; + } + } + } + long leftover = remaining; + for (int i = 0; i < n; i++) { + if (!capped[i] && demands[i] > 0) { + shares[i] = (int) Math.floor(quotas[i]); + leftover -= shares[i]; + } + } + // Floors first: every queue with demand gets a slot before remainders are rounded up. + for (int i = 0; i < n && leftover > 0; i++) { + if (!capped[i] && demands[i] > 0 && shares[i] == 0) { + shares[i] = 1; + leftover--; + } + } + Integer[] order = new Integer[n]; + Arrays.setAll(order, i -> i); + Arrays.sort(order, Comparator.comparingDouble((Integer i) -> quotas[i] - Math.floor(quotas[i])).reversed()); + for (int k = 0; k < n && leftover > 0; k++) { + int i = order[k]; + if (!capped[i] && demands[i] > 0 && shares[i] < demands[i] && shares[i] == Math.floor(quotas[i])) { + shares[i]++; + leftover--; + } + } + for (int i = 0; i < n; i++) { + all[i].targetConcurrency = demands[i] > 0 ? Math.max(1, shares[i]) : 0; + } + } + + /** + * The single exit for target and admission changes: re-divides the process target, resizes execution pools, + * publishes observations and wakes pollers that the new targets admit. + */ + private void applyTargets(long now, AdjustmentReason reason, boolean publishAll) { + allocate(now); + boolean processChanged = targetConcurrency != publishedProcessTarget + || admissionPaused != publishedAdmissionPaused; + boolean resumed = publishedAdmissionPaused && !admissionPaused; + for (WorkerQueue queue : queues.values()) { + boolean changed = queue.targetConcurrency != queue.publishedTarget; + queue.resizePool(); + if (changed || processChanged || publishAll) { + int taskSlots = queue.heldSlots - queue.reservedPollSlots; + long taskSlotAgeSeconds = taskSlots > 0 + ? TimeUnit.NANOSECONDS.toSeconds(Math.max(0, now - queue.lastTaskSlotReleaseAt)) + : 0; + queue.eventDispatcher().publish(new AdaptiveConcurrencyChanged( + queue.taskType(), + queue.domain(), + queue.targetConcurrency, + queue.heldSlots, + taskSlotAgeSeconds, + queue.depth, + targetConcurrency, + heldSlots, + admissionPaused, + reason.name())); + queue.publishedTarget = queue.targetConcurrency; + } + if (resumed) { + queue.slotAvailable.signalAll(); + } else if ((changed || processChanged) && queue.waiters > 0) { + int wakeups = Math.min(queue.waiters, Math.max(0, queue.deficit())); + for (int i = 0; i < wakeups; i++) { + queue.slotAvailable.signal(); + } + } + } + if (processChanged) { + LOGGER.debug("Adaptive concurrency process target {} -> {} ({}), paused={}", + publishedProcessTarget, targetConcurrency, reason, admissionPaused); + } + publishedProcessTarget = targetConcurrency; + publishedAdmissionPaused = admissionPaused; + } + + /** + * Wakes one poller of the queue that released a slot and the waiting queue furthest below its share. + */ + private void wakeAfterRelease(WorkerQueue queue) { + WorkerQueue heartbeat = oldestDueHeartbeatWaiter(clock.getAsLong(), null); + if (heartbeat != null) { + heartbeat.slotAvailable.signal(); + } + if (queue.waiters > 0) { + queue.slotAvailable.signal(); + } + WorkerQueue needier = bestProcessWaiter(queue); + if (needier != null) { + needier.slotAvailable.signal(); + } + } + + /** + * @return the queue waiting for a process slot with the largest positive deficit, earliest arrival first, other + * than {@code exclude}; null if none + */ + private WorkerQueue bestProcessWaiter(WorkerQueue exclude) { + WorkerQueue best = null; + for (WorkerQueue candidate : queues.values()) { + if (candidate == exclude || candidate.processWaiters == 0 || candidate.deficit() <= 0) { + continue; + } + if (best == null || outranks(candidate, best)) { + best = candidate; + } + } + return best; + } + + private static boolean outranks(WorkerQueue a, WorkerQueue b) { + if (a.deficit() != b.deficit()) { + return a.deficit() > b.deficit(); + } + return a.waitSequence() < b.waitSequence(); + } + + // ----- Helpers ----- + + private long emptyPollDelay(WorkerQueue queue) { + long base = TimeUnit.MILLISECONDS.toNanos(Math.max(1, queue.pollingIntervalMillis)); + long cap = Math.max(base, MAX_EMPTY_POLL_DELAY); + long delay = base; + for (int i = 1; i < queue.consecutiveEmptyPolls && delay < cap; i++) { + delay = Math.min(cap, delay * 2); + } + return jitter(delay); + } + + private static long jitter(long nanos) { + double factor = 1 + ThreadLocalRandom.current().nextDouble(-JITTER_FRACTION, JITTER_FRACTION); + return Math.max(1, (long) (nanos * factor)); + } + + private static long elapsed(long now, long since) { + return since == NONE ? Long.MAX_VALUE : now - since; + } + + private static boolean isFresh(boolean supported, long observedAt, long now, long freshness) { + return supported && observedAt != NONE && now - observedAt <= freshness; + } + + private static int clamp(int value, int min, int max) { + return Math.max(min, Math.min(max, value)); + } + + private void resetCpu() { + cpu.raw = 0; + cpu.smoothed = 0; + cpu.observedAt = NONE; + cpu.pressureSince = NONE; + } + + private void resetHeap() { + heap.utilization = 0; + heap.collectionAt = NONE; + heap.observedAt = NONE; + heap.lastHighCollectionAt = NONE; + heap.baseline = Double.NaN; + heap.recent.clear(); + heapGrowthCeiling = Integer.MAX_VALUE; + heapCeilingSince = NONE; + } + + private static ThreadFactory daemonThreadFactory(String namingPattern) { + return new BasicThreadFactory.Builder() + .namingPattern(namingPattern) + .daemon(true) + .build(); + } + + private static Runnable guarded(Runnable runnable, String name) { + return () -> { + try { + runnable.run(); + } catch (Throwable t) { + // A periodic task that throws never runs again, so a failure to log is dropped too. + try { + LOGGER.error("Adaptive concurrency {} failed", name, t); + } catch (Throwable ignored) { + // Nothing left to report it with. + } + } + }; + } +} diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/automator/JvmResourceSampler.java b/conductor-client/src/main/java/com/netflix/conductor/client/automator/JvmResourceSampler.java new file mode 100644 index 000000000..297b436b5 --- /dev/null +++ b/conductor-client/src/main/java/com/netflix/conductor/client/automator/JvmResourceSampler.java @@ -0,0 +1,366 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.client.automator; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryType; +import java.lang.management.MemoryUsage; +import java.lang.management.RuntimeMXBean; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import javax.management.Notification; +import javax.management.NotificationEmitter; +import javax.management.NotificationListener; +import javax.management.openmbean.CompositeData; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; +import com.sun.management.GarbageCollectionNotificationInfo; + +/** + * Reads JVM process CPU from the growth of {@code OperatingSystemMXBean.getProcessCpuTime()} between samples, relative + * to the CPUs the process may use, and post-collection heap from GC notifications, falling back to + * {@link MemoryPoolMXBean#getCollectionUsage()} when notifications are unavailable. + *

+ * Post-collection heap is the larger of the whole heap's utilization and the fullest long-lived pool's utilization of + * its own capacity. With a fixed-size old generation (Serial, Parallel) long-lived data can only live there, so an old + * generation that is full after collection is pressure even while an empty eden keeps the whole-heap ratio near 70%. + * Long-lived pools are those that support usage thresholds, which excludes eden and survivor spaces under every + * collector; a pool's reading is taken only from collections that collect it, since after a young collection the old + * generation still holds its uncollected garbage. + *

+ * {@code getProcessCpuLoad()} is not used: it measures the interval since the previous call by any caller in the JVM, + * so a metrics scrape reading it just before this sampler leaves a window of milliseconds and a reading of zero on a + * saturated core. + */ +class JvmResourceSampler implements ResourceSampler { + + private static final Logger LOGGER = LoggerFactory.getLogger(JvmResourceSampler.class); + + private static final Path CGROUP_V2_CPU_MAX = Path.of("/sys/fs/cgroup/cpu.max"); + private static final Path CGROUP_V1_CPU_QUOTA = Path.of("/sys/fs/cgroup/cpu/cpu.cfs_quota_us"); + private static final Path CGROUP_V1_CPU_PERIOD = Path.of("/sys/fs/cgroup/cpu/cpu.cfs_period_us"); + + private final com.sun.management.OperatingSystemMXBean operatingSystemBean; + private final MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean(); + private final RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean(); + private final List collectorBeans = ManagementFactory.getGarbageCollectorMXBeans(); + private final List heapPools; + private final Set heapPoolNames; + private final Set longLivedPoolNames; + /** Collector name to the long-lived pools it collects. */ + private final Map> collectedLongLivedPools = new HashMap<>(); + /** Long-lived pool utilization after the latest collection of that pool; written by the GC notification thread. */ + private final Map longLivedUtilization = new ConcurrentHashMap<>(); + private final List emitters = new ArrayList<>(); + private final NotificationListener gcListener = this::onCollection; + private final boolean heapSupported; + + // Written by the GC notification thread, read by the sampling thread; null until the first notification. + private volatile CollectionResult lastNotified; + + // CPU state, sampling thread only. + private long lastCpuTime = -1; + private long lastCpuSampledAt; + + // Fallback path state, sampling thread only. + private long lastCollectionCount = -1; + private long lastCollectionAt = AdaptiveConcurrencyController.NONE; + + /** Collections tolerated without a notification before the notification pipeline counts as stalled. */ + private static final int NOTIFICATION_STALL_COLLECTIONS = 4; + // Stall detection, sampling thread only: re-reading one notification forever while collections continue would + // freeze the heap estimate at a stale value that still looks fresh. + private CollectionResult lastSeenNotification; + private long collectionCountAtNotification; + private boolean notificationStallLogged; + + JvmResourceSampler() { + var osBean = ManagementFactory.getOperatingSystemMXBean(); + this.operatingSystemBean = osBean instanceof com.sun.management.OperatingSystemMXBean sunBean ? sunBean : null; + this.heapPools = ManagementFactory.getMemoryPoolMXBeans().stream() + .filter(pool -> pool.getType() == MemoryType.HEAP) + .collect(Collectors.toList()); + this.heapPoolNames = heapPools.stream().map(MemoryPoolMXBean::getName).collect(Collectors.toSet()); + this.longLivedPoolNames = heapPools.stream() + .filter(MemoryPoolMXBean::isUsageThresholdSupported) + .map(MemoryPoolMXBean::getName) + .collect(Collectors.toSet()); + for (GarbageCollectorMXBean collector : collectorBeans) { + Set collected = new HashSet<>(List.of(collector.getMemoryPoolNames())); + collected.retainAll(longLivedPoolNames); + collectedLongLivedPools.put(collector.getName(), collected); + } + + for (GarbageCollectorMXBean collector : collectorBeans) { + if (collector instanceof NotificationEmitter emitter) { + try { + emitter.addNotificationListener(gcListener, null, null); + emitters.add(emitter); + } catch (RuntimeException e) { + LOGGER.warn("Unable to register GC notification listener on {}", collector.getName(), e); + } + } + } + boolean collectionUsageAvailable = heapPools.stream().anyMatch(pool -> pool.getCollectionUsage() != null); + this.heapSupported = heapMax() > 0 && (!emitters.isEmpty() || collectionUsageAvailable); + } + + @Override + public boolean isCpuSupported() { + if (operatingSystemBean == null) { + return false; + } + try { + return operatingSystemBean.getProcessCpuTime() >= 0; + } catch (UnsupportedOperationException e) { + return false; + } + } + + @Override + public boolean isHeapSupported() { + return heapSupported; + } + + /** + * @return CPU used since the previous call as a fraction of the CPUs available to the process; NaN on the first + * call and when unavailable + */ + @Override + public double sampleCpu() { + try { + long cpuTime = operatingSystemBean.getProcessCpuTime(); + long sampledAt = System.nanoTime(); + long previousCpuTime = lastCpuTime; + long previousSampledAt = lastCpuSampledAt; + lastCpuTime = cpuTime; + lastCpuSampledAt = sampledAt; + if (cpuTime < 0 || previousCpuTime < 0 || sampledAt <= previousSampledAt) { + return Double.NaN; + } + return cpuLoad(cpuTime - previousCpuTime, sampledAt - previousSampledAt, availableCpus()); + } catch (RuntimeException e) { + return Double.NaN; + } + } + + static double cpuLoad(long cpuNanos, long elapsedNanos, double cpus) { + double load = (double) cpuNanos / ((double) elapsedNanos * cpus); + return Double.isFinite(load) ? Math.min(1, Math.max(0, load)) : Double.NaN; + } + + /** + * The CPUs the process may use: the container CPU quota when one is set, which may be fractional, otherwise the + * processors the JVM reports. Read on every sample, so a resized container takes effect. + */ + static double availableCpus() { + int processors = Runtime.getRuntime().availableProcessors(); + double quota = cgroupCpuLimit(); + return quota > 0 ? Math.min(processors, quota) : processors; + } + + private static double cgroupCpuLimit() { + try { + if (Files.isReadable(CGROUP_V2_CPU_MAX)) { + return parseCpuMax(Files.readString(CGROUP_V2_CPU_MAX)); + } + if (Files.isReadable(CGROUP_V1_CPU_QUOTA) && Files.isReadable(CGROUP_V1_CPU_PERIOD)) { + return parseCpuQuota(Files.readString(CGROUP_V1_CPU_QUOTA), Files.readString(CGROUP_V1_CPU_PERIOD)); + } + } catch (Exception e) { + LOGGER.debug("Unable to read the cgroup CPU limit", e); + } + return -1; + } + + /** + * @param cpuMax cgroup v2 {@code cpu.max}: {@code " "} or {@code "max "} + * @return the limit in CPUs, or -1 when unlimited or unreadable + */ + static double parseCpuMax(String cpuMax) { + String[] parts = cpuMax.trim().split("\\s+"); + if (parts.length != 2 || "max".equals(parts[0])) { + return -1; + } + return parseCpuQuota(parts[0], parts[1]); + } + + /** + * @return {@code quota / period} in CPUs, or -1 when the quota is unlimited ({@code -1}) or unreadable + */ + static double parseCpuQuota(String quota, String period) { + try { + long quotaMicros = Long.parseLong(quota.trim()); + long periodMicros = Long.parseLong(period.trim()); + return quotaMicros > 0 && periodMicros > 0 ? (double) quotaMicros / periodMicros : -1; + } catch (NumberFormatException e) { + return -1; + } + } + + @Override + public HeapSample sampleHeap() { + try { + long max = heapMax(); + if (max <= 0) { + return null; + } + long collectionCount = totalCollectionCount(); + if (collectionCount <= 0) { + // No collection yet: current used heap is an upper bound on live data. + return new HeapSample((double) memoryBean.getHeapMemoryUsage().getUsed() / (double) max, + AdaptiveConcurrencyController.NONE); + } + CollectionResult notified = lastNotified; + if (notified != null) { + if (notified != lastSeenNotification) { + lastSeenNotification = notified; + collectionCountAtNotification = collectionCount; + // Keep the fallback primed so a later stall can detect the next collection and its time. + lastCollectionCount = collectionCount; + lastCollectionAt = notified.collectionAtNanos(); + if (notificationStallLogged) { + notificationStallLogged = false; + LOGGER.info("GC notifications resumed"); + } + } + if (collectionCount - collectionCountAtNotification <= NOTIFICATION_STALL_COLLECTIONS) { + return new HeapSample(notified.utilization(), notified.collectionAtNanos()); + } + if (!notificationStallLogged) { + notificationStallLogged = true; + LOGGER.warn("{} collections happened without a GC notification; reading collection usage " + + "until notifications resume", + collectionCount - collectionCountAtNotification); + } + } + return sampleCollectionUsage(max, collectionCount); + } catch (RuntimeException e) { + LOGGER.debug("Heap sample failed", e); + return null; + } + } + + @Override + public void close() { + for (NotificationEmitter emitter : emitters) { + try { + emitter.removeNotificationListener(gcListener); + } catch (Exception e) { + LOGGER.debug("Unable to remove GC notification listener", e); + } + } + emitters.clear(); + } + + private HeapSample sampleCollectionUsage(long max, long collectionCount) { + // Collection usage is per pool and only updated by collections of that pool, which is the rule we want. + long used = 0; + double longLived = 0; + for (MemoryPoolMXBean pool : heapPools) { + MemoryUsage usage = pool.getCollectionUsage(); + if (usage != null) { + used += usage.getUsed(); + if (longLivedPoolNames.contains(pool.getName())) { + longLived = Math.max(longLived, poolUtilization(usage, max)); + } + } + } + if (collectionCount != lastCollectionCount) { + // A collection happened since the previous tick; the tick that observed it is its time. + // Collections that completed before sampling started carry no time. + lastCollectionAt = lastCollectionCount < 0 ? AdaptiveConcurrencyController.NONE : System.nanoTime(); + lastCollectionCount = collectionCount; + } + return new HeapSample(Math.max((double) used / (double) max, longLived), lastCollectionAt); + } + + private void onCollection(Notification notification, Object handback) { + if (!GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION.equals(notification.getType())) { + return; + } + try { + var info = GarbageCollectionNotificationInfo.from((CompositeData) notification.getUserData()); + long max = heapMax(); + if (max <= 0) { + return; + } + Map after = info.getGcInfo().getMemoryUsageAfterGc(); + long used = 0; + for (Map.Entry entry : after.entrySet()) { + if (heapPoolNames.contains(entry.getKey())) { + used += entry.getValue().getUsed(); + } + } + for (String pool : collectedLongLivedPools.getOrDefault(info.getGcName(), Set.of())) { + MemoryUsage usage = after.get(pool); + if (usage != null) { + longLivedUtilization.put(pool, poolUtilization(usage, max)); + } + } + double utilization = (double) used / (double) max; + for (double pool : longLivedUtilization.values()) { + utilization = Math.max(utilization, pool); + } + // GC end time is JVM uptime in milliseconds; convert it to the monotonic clock. + long agoMillis = Math.max(0, runtimeBean.getUptime() - info.getGcInfo().getEndTime()); + lastNotified = new CollectionResult(utilization, System.nanoTime() - agoMillis * 1_000_000L); + } catch (RuntimeException e) { + LOGGER.debug("Unable to read GC notification", e); + } + } + + private record CollectionResult(double utilization, long collectionAtNanos) {} + + /** Simulates a notification that no longer updates, for stall-detection tests. */ + @VisibleForTesting + void injectNotificationForTest(double utilization) { + lastNotified = new CollectionResult(utilization, System.nanoTime()); + } + + /** A pool's used share of its own capacity, or of the heap when the pool has no fixed maximum. */ + static double poolUtilization(MemoryUsage usage, long heapMax) { + long capacity = usage.getMax() > 0 ? Math.min(usage.getMax(), heapMax) : heapMax; + return Math.min(1, (double) usage.getUsed() / (double) capacity); + } + + private long heapMax() { + return memoryBean.getHeapMemoryUsage().getMax(); + } + + private long totalCollectionCount() { + long total = 0; + for (GarbageCollectorMXBean collector : collectorBeans) { + long count = collector.getCollectionCount(); + if (count > 0) { + total += count; + } + } + return total; + } +} diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/automator/ResourceSampler.java b/conductor-client/src/main/java/com/netflix/conductor/client/automator/ResourceSampler.java new file mode 100644 index 000000000..ecf82dc96 --- /dev/null +++ b/conductor-client/src/main/java/com/netflix/conductor/client/automator/ResourceSampler.java @@ -0,0 +1,49 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.client.automator; + +/** + * Source of JVM resource observations for {@link AdaptiveConcurrencyController}. Implementations are called from the + * controller's sampling thread, outside the controller lock. + */ +interface ResourceSampler { + + boolean isCpuSupported(); + + boolean isHeapSupported(); + + /** + * @return JVM process CPU utilization as a ratio in [0, 1], or {@code Double.NaN} when the sample is unavailable + */ + double sampleCpu(); + + /** + * @return the latest heap observation, or {@code null} when the sample failed + */ + HeapSample sampleHeap(); + + /** + * Releases listeners registered with the JVM. + */ + void close(); + + /** + * A heap observation. + * + * @param utilization post-collection heap used divided by heap max; current used heap before the first + * collection + * @param collectionAtNanos monotonic time of the collection that produced the value, or + * {@link AdaptiveConcurrencyController#NONE} before the first collection + */ + record HeapSample(double utilization, long collectionAtNanos) {} +} diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/automator/SlotReservation.java b/conductor-client/src/main/java/com/netflix/conductor/client/automator/SlotReservation.java new file mode 100644 index 000000000..f497b6dfc --- /dev/null +++ b/conductor-client/src/main/java/com/netflix/conductor/client/automator/SlotReservation.java @@ -0,0 +1,42 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.client.automator; + +/** + * Capacity granted to one poll request by {@link AdaptiveConcurrencyController#reserve}. Each slot is owned by the + * request until it is released by {@link AdaptiveConcurrencyController#completePoll} or transferred to a returned + * task. + * + * @param queue the logical queue the slots belong to + * @param count number of tasks the poll may request; every one has an open slot + * @param kind what the poll is for, which sets how long the server may hold it + */ +record SlotReservation(WorkerQueue queue, int count, Kind kind) { + + enum Kind { + /** Slots within the queue's share; the poll waits the worker's poll timeout. */ + SHARE, + /** A queue without free share keeps its domain active: one task, without waiting. */ + HEARTBEAT, + /** A queue without free share waits for its next task on a slot no share claims: one task, standby timeout. */ + STANDBY + } + + boolean heartbeat() { + return kind == Kind.HEARTBEAT; + } + + boolean standby() { + return kind == Kind.STANDBY; + } +} diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunner.java b/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunner.java index e8b70f28b..714c94682 100644 --- a/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunner.java +++ b/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunner.java @@ -25,6 +25,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.Semaphore; @@ -89,6 +90,9 @@ class TaskRunner { private Map> leaseExtendMap = new ConcurrentHashMap<>(); private final AtomicInteger activeWorkerCount = new AtomicInteger(0); private Runnable exitAction = () -> System.exit(1); + // Adaptive mode only; null in fixed mode. + private final AdaptiveConcurrencyController adaptiveController; + private final WorkerQueue workerQueue; TaskRunner(Worker worker, TaskClient taskClient, @@ -100,6 +104,29 @@ class TaskRunner { List pollFilters, EventDispatcher eventDispatcher, boolean useVirtualThreads) { + this(worker, taskClient, updateRetryCount, taskToDomain, workerNamePrefix, threadCount, taskPollTimeout, + pollFilters, eventDispatcher, useVirtualThreads, false, null, 0); + } + + /** + * @param enableUpdateV2 use update-and-fetch; the {@code taskUpdateV2} system property or + * environment variable also enables it + * @param adaptiveController controller for adaptive mode, or null for fixed mode + * @param adaptiveMaxConcurrency ceiling for this worker's adaptive concurrency; ignored in fixed mode + */ + TaskRunner(Worker worker, + TaskClient taskClient, + int updateRetryCount, + Map taskToDomain, + String workerNamePrefix, + int threadCount, + int taskPollTimeout, + List pollFilters, + EventDispatcher eventDispatcher, + boolean useVirtualThreads, + boolean enableUpdateV2, + AdaptiveConcurrencyController adaptiveController, + int adaptiveMaxConcurrency) { this.worker = worker; this.taskClient = taskClient; this.updateRetryCount = updateRetryCount; @@ -110,7 +137,8 @@ class TaskRunner { this.pollFilters = pollFilters; this.eventDispatcher = eventDispatcher; this.tasksTobeExecuted = new LinkedBlockingQueue<>(); - this.enableUpdateV2 = Boolean.parseBoolean(System.getProperty("taskUpdateV2", "false")) || Boolean.parseBoolean(System.getenv("taskUpdateV2")); + this.enableUpdateV2 = enableUpdateV2 || Boolean.parseBoolean(System.getProperty("taskUpdateV2", "false")) || Boolean.parseBoolean(System.getenv("taskUpdateV2")); + this.adaptiveController = adaptiveController; LOGGER.info("taskUpdateV2 is set to {} for {}", this.enableUpdateV2, worker.getTaskDefName()); //1. Is there a worker level override? @@ -135,21 +163,55 @@ class TaskRunner { this.errorAt = errorInterval; LOGGER.info("Polling errors will be sampled at every {} error (after the first 100 errors) for taskType {}", this.errorAt, taskType); this.uncaughtExceptionHandler = this::onUncaughtException; - ThreadFactory threadFactory = null; - if(useVirtualThreads) { - threadFactory = Thread.ofVirtual().name(workerNamePrefix).uncaughtExceptionHandler(uncaughtExceptionHandler).factory(); + if (adaptiveController != null) { + ThreadFactory threadFactory; + if (useVirtualThreads) { + // Numbered names; the pattern form leaves every virtual thread with the literal prefix. + threadFactory = Thread.ofVirtual() + .name(workerNamePrefix.replace("%d", ""), 0) + .uncaughtExceptionHandler(uncaughtExceptionHandler) + .factory(); + } else { + threadFactory = new BasicThreadFactory.Builder() + .namingPattern(workerNamePrefix) + .uncaughtExceptionHandler(uncaughtExceptionHandler) + .build(); + } + this.workerQueue = adaptiveController.register( + taskClient, + taskType, + domain, + adaptiveMaxConcurrency, + pollingIntervalInMillis, + this::isPollable, + threadFactory, + useVirtualThreads, + eventDispatcher); + this.executorService = workerQueue.executionPool(); + LOGGER.info("Starting adaptive Worker for taskType '{}' with maximum concurrency {}, {} threads, {} ms polling interval and domain {}", + taskType, + workerQueue.maximumConcurrency, + useVirtualThreads ? "virtual" : "platform", + pollingIntervalInMillis, + domain); } else { - threadFactory = new BasicThreadFactory.Builder() - .namingPattern(workerNamePrefix) - .uncaughtExceptionHandler(uncaughtExceptionHandler) - .build(); - } - this.executorService = Executors.newFixedThreadPool(threadCount, threadFactory); - LOGGER.info("Starting Worker for taskType '{}' with {} threads, {} ms polling interval and domain {}", - taskType, - threadCount, - pollingIntervalInMillis, - domain); + this.workerQueue = null; + ThreadFactory threadFactory = null; + if(useVirtualThreads) { + threadFactory = Thread.ofVirtual().name(workerNamePrefix).uncaughtExceptionHandler(uncaughtExceptionHandler).factory(); + } else { + threadFactory = new BasicThreadFactory.Builder() + .namingPattern(workerNamePrefix) + .uncaughtExceptionHandler(uncaughtExceptionHandler) + .build(); + } + this.executorService = Executors.newFixedThreadPool(threadCount, threadFactory); + LOGGER.info("Starting Worker for taskType '{}' with {} threads, {} ms polling interval and domain {}", + taskType, + threadCount, + pollingIntervalInMillis, + domain); + } LOGGER.info("Polling errors for taskType {} will be printed at every {} occurrence.", taskType, errorAt); LOGGER.info("Initialized the task lease extend executor"); @@ -168,6 +230,10 @@ void setExitAction(Runnable exitAction) { } public void pollAndExecute() { + if (workerQueue != null) { + pollAndExecuteAdaptive(); + return; + } Stopwatch stopwatch = null; while (pollingAndExecuting) { if (Thread.currentThread().isInterrupted()) { @@ -188,39 +254,86 @@ public void pollAndExecute() { LOGGER.trace("Poller for task {} waited for {} ms before getting {} tasks to execute", taskType, stopwatch.elapsed(TimeUnit.MILLISECONDS), tasks.size()); stopwatch = null; } - tasks.forEach(task -> { - Future taskFuture; - try { - taskFuture = this.executorService.submit(() -> this.processTask(task)); - } catch (java.util.concurrent.RejectedExecutionException e) { - LOGGER.error("Task rejected by executor (likely shutting down); dropping task {} of type {}", - task.getTaskId(), taskType, e); - permits.release(); - return; - } + tasks.forEach(this::dispatch); + } catch (Throwable t) { + LOGGER.error(t.getMessage(), t); + } + } + } - if (task.getResponseTimeoutSeconds() > 0 && worker.leaseExtendEnabled()) { - ScheduledFuture existingFuture = leaseExtendMap.remove(task.getTaskId()); - if (existingFuture != null) { - existingFuture.cancel(false); - } - - long delay = Math.round(task.getResponseTimeoutSeconds() * LEASE_EXTEND_DURATION_FACTOR); - ScheduledFuture leaseExtendFuture = leaseExtendExecutorService.scheduleWithFixedDelay( - extendLease(task, taskFuture), - delay, - delay, - TimeUnit.SECONDS - ); - leaseExtendMap.put(task.getTaskId(), leaseExtendFuture); - } - }); + private void pollAndExecuteAdaptive() { + while (pollingAndExecuting) { + if (Thread.currentThread().isInterrupted()) { + break; // Exit the loop if interrupted + } + + try { + List tasks = pollTasksAdaptive(); + tasks.forEach(this::dispatch); } catch (Throwable t) { LOGGER.error(t.getMessage(), t); } } } + /** + * Submits a task that owns a slot to the executor and schedules its lease extension. Used for polled tasks and + * for tasks returned by update-and-fetch. + */ + private void dispatch(Task task) { + Future taskFuture; + try { + taskFuture = this.executorService.submit(() -> this.processTask(task)); + } catch (Throwable e) { + LOGGER.error("Task rejected by executor (likely shutting down); dropping task {} of type {}", + task.getTaskId(), taskType, e); + releaseSlot(); + return; + } + + // The submitted task owns the slot now; a failure below must neither leak nor double-release it. + if (task.getResponseTimeoutSeconds() > 0 && worker.leaseExtendEnabled()) { + try { + ScheduledFuture existingFuture = leaseExtendMap.remove(task.getTaskId()); + if (existingFuture != null) { + existingFuture.cancel(false); + } + + long delay = Math.round(task.getResponseTimeoutSeconds() * LEASE_EXTEND_DURATION_FACTOR); + ScheduledFuture leaseExtendFuture = leaseExtendExecutorService.scheduleWithFixedDelay( + extendLease(task, taskFuture), + delay, + delay, + TimeUnit.SECONDS + ); + leaseExtendMap.put(task.getTaskId(), leaseExtendFuture); + } catch (RejectedExecutionException e) { + LOGGER.warn("Unable to schedule lease extension for task {} (likely shutting down); " + + "the task runs without lease extension", task.getTaskId(), e); + } + } + } + + private void releaseSlot() { + if (workerQueue != null) { + adaptiveController.releaseTaskSlot(workerQueue); + } else { + permits.release(); + } + } + + private boolean isPollable() { + if (worker.paused()) { + return false; + } + for (PollFilter filter : pollFilters) { + if (!filter.filter(taskType, domain)) { + return false; + } + } + return true; + } + public void shutdown(int timeout) { try { pollingAndExecuting = false; @@ -233,6 +346,11 @@ public void shutdown(int timeout) { LOGGER.warn("Lease extend executor service did not terminate within {} seconds, forcing shutdown", timeout); leaseExtendExecutorService.shutdownNow(); } + + // Pollers of one logical queue share its execution pool; the last one to leave drains it. + if (workerQueue != null && !adaptiveController.deregister(workerQueue)) { + return; + } executorService.shutdown(); if (executorService.awaitTermination(timeout, TimeUnit.SECONDS)) { @@ -294,34 +412,115 @@ private List pollTasksForWorker() { eventDispatcher.publish(new PollCompleted(taskType, elapsed)); } catch (Throwable e) { permits.release(pollCount - tasks.size()); + onPollError(e, stopwatch); + } - if (hasFatalAuthCause(e)) { - LOGGER.error("Fatal authentication failure — terminating JVM"); - exitAction.run(); - } + return tasks; + } - //For the first 100 errors, just print them as is... - boolean printError = pollingErrorCount < 100 || pollingErrorCount % errorAt == 0; - pollingErrorCount++; - if (pollingErrorCount > 10_000_000) { - //Reset after 10 million errors - pollingErrorCount = 0; - } - if (printError) { - LOGGER.error("Error polling for taskType: {}, error = {}", taskType, e.getMessage(), e); - } + /** + * Adaptive counterpart of {@link #pollTasksForWorker()}: slots come from the controller instead of the + * semaphore, and an empty response or error sets a backoff instead of sleeping for the polling interval. + */ + private List pollTasksAdaptive() { + if (worker.paused()) { + LOGGER.trace("Worker {} has been paused. Not polling anymore!", worker.getClass()); + eventDispatcher.publish(new TaskPaused(taskType)); + Uninterruptibles.sleepUninterruptibly(pollingIntervalInMillis, TimeUnit.MILLISECONDS); + return List.of(); + } - if (stopwatch.isRunning()) { - stopwatch.stop(); + for (PollFilter filter : pollFilters) { + if (!filter.filter(taskType, domain)) { + LOGGER.trace("Filter returned false, not polling."); + Uninterruptibles.sleepUninterruptibly(pollingIntervalInMillis, TimeUnit.MILLISECONDS); + return List.of(); } + } - long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS); - eventDispatcher.publish(new PollFailure(taskType, elapsed, e)); + long pollDelay = adaptiveController.pollDelayNanos(workerQueue); + if (pollDelay > 0) { + // Bounded so pause and shutdown are re-checked at least once a second. + Uninterruptibles.sleepUninterruptibly(Math.min(pollDelay, TimeUnit.SECONDS.toNanos(1)), TimeUnit.NANOSECONDS); + return List.of(); + } + + SlotReservation reservation = adaptiveController.reserve(workerQueue, () -> pollingAndExecuting); + if (reservation == null) { + return List.of(); + } + + eventDispatcher.publish(new PollStarted(taskType)); + long pollStartedAt = adaptiveController.now(); + Stopwatch stopwatch = Stopwatch.createStarted(); + List tasks; + try { + LOGGER.trace("Polling task of type: {} in domain: '{}' with size {}", taskType, domain, reservation.count()); + tasks = taskClient.batchPollTasksInDomain(taskType, domain, worker.getIdentity(), + reservation.count(), adaptivePollTimeout(reservation)); + } catch (Throwable e) { + adaptiveController.failPoll(reservation, pollStartedAt); + onPollError(e, stopwatch); + return List.of(); } + if (tasks == null) { + tasks = List.of(); + } + adaptiveController.completePoll(reservation, tasks.size(), pollStartedAt); + // The returned tasks own their slots now; a bookkeeping failure must not drop them. + try { + long receiveTime = System.currentTimeMillis(); + for (Task task : tasks) { + task.getExecutionMetadata().setClientReceiveTime(receiveTime); + } + stopwatch.stop(); + long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS); + LOGGER.trace("Time taken to poll {} task with a batch size of {} is {} ms", taskType, tasks.size(), elapsed); + eventDispatcher.publish(new PollCompleted(taskType, elapsed)); + } catch (Throwable e) { + LOGGER.warn("Poll bookkeeping failed for {}; the polled tasks still execute", taskType, e); + } return tasks; } + /** + * A heartbeat does not wait on the server: its slot is spare capacity other queues may need. A standby poll waits + * for the standby timeout, which bounds how long a queue whose share grows waits for that slot. + */ + private int adaptivePollTimeout(SlotReservation reservation) { + return switch (reservation.kind()) { + case HEARTBEAT -> 0; + case STANDBY -> (int) adaptiveController.getConfig().getStandbyPollTimeout().toMillis(); + case SHARE -> this.taskPollTimeout; + }; + } + + private void onPollError(Throwable e, Stopwatch stopwatch) { + if (hasFatalAuthCause(e)) { + LOGGER.error("Fatal authentication failure — terminating JVM"); + exitAction.run(); + } + + //For the first 100 errors, just print them as is... + boolean printError = pollingErrorCount < 100 || pollingErrorCount % errorAt == 0; + pollingErrorCount++; + if (pollingErrorCount > 10_000_000) { + //Reset after 10 million errors + pollingErrorCount = 0; + } + if (printError) { + LOGGER.error("Error polling for taskType: {}, error = {}", taskType, e.getMessage(), e); + } + + if (stopwatch.isRunning()) { + stopwatch.stop(); + } + + long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS); + eventDispatcher.publish(new PollFailure(taskType, elapsed, e)); + } + private List pollTask(int count) { if (count < 1) { return Collections.emptyList(); @@ -373,8 +572,9 @@ private Task processTask(Task task) { LOGGER.trace("Executing task: {} of type: {} in worker: {} at {}", task.getTaskId(), taskType, worker.getClass().getSimpleName(), worker.getIdentity()); LOGGER.trace("task {} is getting executed after {} ms of getting polled", task.getTaskId(), (System.currentTimeMillis() - task.getStartTime())); Stopwatch stopwatch = Stopwatch.createStarted(); + TaskInvocation invocation = new TaskInvocation(); try { - executeTask(worker, task); + executeTask(worker, task, invocation); stopwatch.stop(); long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS); LOGGER.trace( @@ -384,16 +584,21 @@ private Task processTask(Task task) { } catch (Throwable t) { task.setStatus(Task.Status.FAILED); TaskResult result = new TaskResult(task); - handleException(t, result, worker, task); + handleException(t, result, worker, task, invocation); } finally { cancelLeaseExtension(task.getTaskId()); - permits.release(); + if (invocation.replacementTask != null) { + // The replacement takes over this task's slot. + dispatch(invocation.replacementTask); + } else { + releaseSlot(); + } eventDispatcher.publish(new ActiveWorkersChanged(taskType, activeWorkerCount.decrementAndGet())); } return task; } - private void executeTask(Worker worker, Task task) { + private void executeTask(Worker worker, Task task, TaskInvocation invocation) { if (task == null || task.getTaskDefName().isEmpty()) { LOGGER.warn("Empty task {}", worker.getTaskDefName()); return; @@ -442,7 +647,7 @@ private void executeTask(Worker worker, Task task) { task.setStatus(Task.Status.FAILED); result = new TaskResult(task); } - handleException(e, result, worker, task); + handleException(e, result, worker, task, invocation); } LOGGER.trace( @@ -451,10 +656,10 @@ private void executeTask(Worker worker, Task task) { worker.getClass().getSimpleName(), worker.getIdentity(), result.getStatus()); - updateTaskResult(updateRetryCount, task, result, worker); + updateTaskResult(updateRetryCount, task, result, worker, invocation); } - private void updateTaskResult(int count, Task task, TaskResult result, Worker worker) { + private void updateTaskResult(int count, Task task, TaskResult result, Worker worker, TaskInvocation invocation) { Stopwatch updateStopwatch = Stopwatch.createStarted(); try { // upload if necessary @@ -475,14 +680,21 @@ private void updateTaskResult(int count, Task task, TaskResult result, Worker wo result.getExecutionMetadata().setClientSendTime(clientSendTime); LOGGER.debug("Task {} outbound send time: {}", task.getTaskId(), clientSendTime); - if(enableUpdateV2) { + if(enableUpdateV2 && isReplacementAllowed(invocation)) { Task nextTask = retryOperation( (TaskResult taskResult) -> taskClient.updateTaskV2(taskResult, taskType), count, result, "updateTaskV2"); if (nextTask != null) { - tasksTobeExecuted.add(nextTask); + if (workerQueue != null) { + // Dispatched with this task's slot once the invocation finishes. + adaptiveController.recordReplacement(workerQueue); + nextTask.getExecutionMetadata().setClientReceiveTime(System.currentTimeMillis()); + invocation.replacementTask = nextTask; + } else { + tasksTobeExecuted.add(nextTask); + } } } else { retryOperation( @@ -547,14 +759,34 @@ private R retryOperation(Function operation, int count, T input, St throw new RuntimeException("Exhausted retries performing " + opName); } - private void handleException(Throwable t, TaskResult result, Worker worker, Task task) { + private void handleException(Throwable t, TaskResult result, Worker worker, Task task, TaskInvocation invocation) { LOGGER.error(String.format("Error while executing task %s", task.toString()), t); result.setStatus(TaskResult.Status.FAILED); result.setReasonForIncompletion("Error while executing the task: " + t); StringWriter stringWriter = new StringWriter(); t.printStackTrace(new PrintWriter(stringWriter)); result.log(stringWriter.toString()); - updateTaskResult(updateRetryCount, task, result, worker); + updateTaskResult(updateRetryCount, task, result, worker, invocation); + } + + /** + * In adaptive mode a replacement is requested at most once per invocation, and only when the controller would + * grant the freed slot back to this worker. A worker that fails after a successful update reports again with + * the plain update. + */ + private boolean isReplacementAllowed(TaskInvocation invocation) { + if (workerQueue == null) { + return true; + } + return invocation.replacementTask == null && adaptiveController.isReplacementAllowed(workerQueue); + } + + /** + * Per-invocation state of {@link #processTask(Task)}. + */ + private static final class TaskInvocation { + // Task returned by update-and-fetch that takes over this invocation's slot; adaptive mode only. + Task replacementTask; } private Runnable extendLease(Task task, Future taskCompletableFuture) { diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunnerConfigurer.java b/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunnerConfigurer.java index 030246917..12852172e 100644 --- a/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunnerConfigurer.java +++ b/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunnerConfigurer.java @@ -26,6 +26,7 @@ import com.netflix.conductor.client.automator.filters.PollFilter; import com.netflix.conductor.client.config.ConductorClientConfiguration; +import com.netflix.conductor.client.config.PropertyFactory; import com.netflix.conductor.client.events.dispatcher.EventDispatcher; import com.netflix.conductor.client.events.listeners.ListenerRegister; import com.netflix.conductor.client.events.taskrunner.TaskRunnerEvent; @@ -55,6 +56,9 @@ public class TaskRunnerConfigurer { private final EventDispatcher eventDispatcher; private final MetricsCollector metricsCollector; private final boolean useVirtualThreads; + private final boolean updateV2; + private final AdaptiveConcurrencyController adaptiveConcurrencyController; + private final Map taskToAdaptiveMaxConcurrency; /** * @see TaskRunnerConfigurer.Builder @@ -77,6 +81,9 @@ private TaskRunnerConfigurer(TaskRunnerConfigurer.Builder builder) { this.eventDispatcher = builder.eventDispatcher; this.metricsCollector = builder.metricsCollector; this.useVirtualThreads = builder.useVirtualThreads; + this.updateV2 = builder.updateV2; + this.adaptiveConcurrencyController = builder.adaptiveConcurrencyController; + this.taskToAdaptiveMaxConcurrency = builder.taskToAdaptiveMaxConcurrency; builder.workers.forEach(this.workers::add); taskRunners = new LinkedList<>(); } @@ -138,6 +145,20 @@ public boolean isUseVirtualThreads() { return useVirtualThreads; } + /** + * @return true if workers use adaptive concurrency instead of fixed thread counts + */ + public boolean isAdaptiveConcurrencyEnabled() { + return adaptiveConcurrencyController != null; + } + + /** + * @return true if task results are reported with update-and-fetch + */ + public boolean isUpdateV2() { + return updateV2; + } + /** * Starts the polling. Must be called after {@link TaskRunnerConfigurer.Builder#build()} method. */ @@ -145,9 +166,31 @@ public synchronized void init() { this.scheduledExecutorService = Executors.newScheduledThreadPool(workers.size(), new BasicThreadFactory.Builder().namingPattern("TaskRunner %d") .build()); + if (adaptiveConcurrencyController != null) { + initAdaptive(); + return; + } workers.forEach(worker -> scheduledExecutorService.submit(() -> this.startWorker(worker))); } + /** + * Registers every worker with the controller before any poller starts, so the initial process target sees all + * logical queues. + */ + private void initAdaptive() { + List runners = new LinkedList<>(); + for (Worker worker : workers) { + runners.add(createTaskRunner(worker)); + } + synchronized (taskRunners) { + taskRunners.addAll(runners); + } + adaptiveConcurrencyController.activate(); + for (TaskRunner taskRunner : runners) { + scheduledExecutorService.submit(taskRunner::pollAndExecute); + } + } + /** * Invoke this method within a PreDestroy block within your application to facilitate a graceful * shutdown of your worker, during process termination. @@ -165,11 +208,11 @@ public void shutdown() { } } - private void startWorker(Worker worker) { + private TaskRunner createTaskRunner(Worker worker) { final Integer threadCountForTask = this.taskToThreadCount.getOrDefault(worker.getTaskDefName(), threadCount); final Integer taskPollTimeout = this.taskPollTimeout.getOrDefault(worker.getTaskDefName(), defaultPollTimeout); LOGGER.info("Domain map for tasks = {}", taskToDomain); - final TaskRunner taskRunner = new TaskRunner( + return new TaskRunner( worker, taskClient, updateRetryCount, @@ -179,7 +222,33 @@ private void startWorker(Worker worker) { taskPollTimeout, pollFilters, eventDispatcher, - useVirtualThreads); + useVirtualThreads, + updateV2, + adaptiveConcurrencyController, + adaptiveMaxConcurrency(worker.getTaskDefName())); + } + + /** + * Resolves a worker's adaptive ceiling: the builder map, then {@code conductor.worker..adaptive_max_concurrency} + * with the {@code conductor.worker.adaptive_max_concurrency} fallback, then the controller default. + */ + private int adaptiveMaxConcurrency(String taskType) { + if (adaptiveConcurrencyController == null) { + return 0; + } + int defaultMax = adaptiveConcurrencyController.getConfig().getDefaultMaxWorkerConcurrency(); + Integer value = taskToAdaptiveMaxConcurrency.get(taskType); + if (value == null) { + value = PropertyFactory.getInteger(taskType, Worker.PROP_ADAPTIVE_MAX_CONCURRENCY, defaultMax); + } + if (value < 1) { + throw new IllegalArgumentException("Adaptive maximum concurrency for " + taskType + " must be positive"); + } + return Math.min(value, adaptiveConcurrencyController.getConfig().getMaxProcessConcurrency()); + } + + private void startWorker(Worker worker) { + final TaskRunner taskRunner = createTaskRunner(worker); // startWorker(worker) is executed by several threads. // taskRunners.add(taskRunner) without synchronization could lead to a race condition and unpredictable behavior, // including potential null values being inserted or corrupted state. @@ -212,6 +281,9 @@ public static class Builder { private final List pollFilters = new LinkedList<>(); private final EventDispatcher eventDispatcher = new EventDispatcher<>(); private boolean useVirtualThreads; + private boolean updateV2; + private AdaptiveConcurrencyController adaptiveConcurrencyController; + private Map taskToAdaptiveMaxConcurrency = new HashMap<>(); /** * Returns the event dispatcher used by this builder, allowing direct @@ -316,11 +388,21 @@ public TaskRunnerConfigurer.Builder withTaskPollTimeout(Integer taskPollTimeout) return this; } + /** + * @deprecated never applied to polling; use adaptive concurrency, whose poll batch follows free slots up to + * {@link AdaptiveConcurrencyConfig#getMaxPollBatch()} + */ + @Deprecated public TaskRunnerConfigurer.Builder withTaskPollCount(Map taskPollCount) { this.taskPollCount = taskPollCount; return this; } + /** + * @deprecated never applied to polling; use adaptive concurrency, whose poll batch follows free slots up to + * {@link AdaptiveConcurrencyConfig#getMaxPollBatch()} + */ + @Deprecated public TaskRunnerConfigurer.Builder withTaskPollCount(int defaultPollCount) { this.defaultPollCount = defaultPollCount; return this; @@ -335,6 +417,9 @@ public TaskRunnerConfigurer.Builder withTaskPollCount(int defaultPollCount) { * @return Builder instance */ public TaskRunnerConfigurer build() { + if (adaptiveConcurrencyController != null) { + validateAdaptivePollTimeouts(); + } if (!metricsCollectorExplicitlySet) { var conductorClient = taskClient.getConductorClient(); if (conductorClient != null) { @@ -383,5 +468,49 @@ public Builder withUseVirtualThreads(boolean useVirtualThreads) { return this; } + /** + * @param updateV2 report results with update-and-fetch, which can return the next task in the same call. + * The {@code taskUpdateV2} system property or environment variable also enables it. + * @return Builder instance + */ + public Builder withUpdateV2(boolean updateV2) { + this.updateV2 = updateV2; + return this; + } + + /** + * Enables adaptive concurrency. Thread counts no longer apply: the controller divides one process target + * among workers in proportion to queue depth, grows it under demand while CPU and heap have headroom, and + * shrinks it under pressure. Share one controller among all configurers of a process. + * + * @param adaptiveConcurrencyController the shared controller + * @return Builder instance + */ + public Builder withAdaptiveConcurrency(AdaptiveConcurrencyController adaptiveConcurrencyController) { + this.adaptiveConcurrencyController = adaptiveConcurrencyController; + return this; + } + + /** + * @param taskToAdaptiveMaxConcurrency per-task ceilings for adaptive concurrency; tasks not listed use + * {@link AdaptiveConcurrencyConfig#getDefaultMaxWorkerConcurrency()} + * @return Builder instance + */ + public Builder withTaskAdaptiveMaxConcurrency(Map taskToAdaptiveMaxConcurrency) { + if (taskToAdaptiveMaxConcurrency.values().stream().anyMatch(v -> v == null || v < 1)) { + throw new IllegalArgumentException("Adaptive maximum concurrency cannot be less than 1"); + } + this.taskToAdaptiveMaxConcurrency = taskToAdaptiveMaxConcurrency; + return this; + } + + private void validateAdaptivePollTimeouts() { + int limit = AdaptiveConcurrencyController.MAX_POLL_TIMEOUT_MILLIS; + if (defaultPollTimeout > limit || taskPollTimeout.values().stream().anyMatch(t -> t != null && t > limit)) { + throw new IllegalArgumentException("Poll timeout cannot exceed " + limit + + " ms; the server rejects longer batch polls"); + } + } + } } diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/automator/WorkerQueue.java b/conductor-client/src/main/java/com/netflix/conductor/client/automator/WorkerQueue.java new file mode 100644 index 000000000..6b66a2ef1 --- /dev/null +++ b/conductor-client/src/main/java/com/netflix/conductor/client/automator/WorkerQueue.java @@ -0,0 +1,193 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.client.automator; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.function.BooleanSupplier; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.conductor.client.events.dispatcher.EventDispatcher; +import com.netflix.conductor.client.events.taskrunner.TaskRunnerEvent; +import com.netflix.conductor.client.http.TaskClient; + +/** + * Controller state for one logical worker queue: server/client scope, task type and resolved domain. All pollers + * registered for the same logical queue share this state and its execution pool. + * + *

Mutable fields are guarded by the controller lock. + */ +final class WorkerQueue { + + private static final Logger LOGGER = LoggerFactory.getLogger(WorkerQueue.class); + private static final long EXECUTION_THREAD_KEEP_ALIVE_SECONDS = 60; + + /** + * Identity of a logical queue. {@link TaskClient} compares by identity, so distinct client scopes are never + * merged. + */ + record Key(TaskClient taskClient, String taskType, String domain) {} + + final Key key; + final String queueName; + final int maximumConcurrency; + final int pollingIntervalMillis; + final Condition slotAvailable; + // Replaced by later registrations; read outside the lock by the depth sampler. + private volatile EventDispatcher eventDispatcher; + private volatile BooleanSupplier pollable; + private final ExecutorService executionPool; + private final ThreadPoolExecutor platformPool; + private int configuredPoolSize = 1; + + int registrations; + int targetConcurrency; + int publishedTarget = -1; + int heldSlots; + int depth; + long depthObservedAt = AdaptiveConcurrencyController.NONE; + int outstandingPolls; + int consecutiveEmptyPolls; + /** Sampled depth when the latest empty response settled; arrivals since then count in full. */ + long depthAtEmptyPoll; + long nextPollAllowedAt; + long nextHeartbeatAt; + long nextStandbyAt; + /** Outstanding standby polls; their slots are not part of the backlog. */ + int standbyPolls; + /** Slots held by outstanding poll requests; the rest of {@link #heldSlots} back accepted tasks. */ + int reservedPollSlots; + /** When a task slot was last released, or tasks last started arriving; a wedged worker's age grows from here. */ + long lastTaskSlotReleaseAt; + /** Pause and poll filter result at the latest depth sample. */ + boolean pollableAtSample = true; + /** Pollers waiting for an open slot for a due heartbeat. */ + int heartbeatWaiters; + int waiters; + int processWaiters; + long processWaitSequence; + + WorkerQueue(Key key, + int maximumConcurrency, + int pollingIntervalMillis, + BooleanSupplier pollable, + ThreadFactory threadFactory, + boolean useVirtualThreads, + EventDispatcher eventDispatcher, + Condition slotAvailable, + long now) { + this.key = key; + this.queueName = queueName(key.taskType(), key.domain()); + this.maximumConcurrency = maximumConcurrency; + this.pollingIntervalMillis = pollingIntervalMillis; + this.pollable = pollable; + this.eventDispatcher = eventDispatcher; + this.slotAvailable = slotAvailable; + this.nextPollAllowedAt = now; + this.nextHeartbeatAt = now; + this.nextStandbyAt = now; + this.lastTaskSlotReleaseAt = now; + if (useVirtualThreads) { + this.platformPool = null; + this.executionPool = Executors.newThreadPerTaskExecutor(threadFactory); + } else { + this.platformPool = new ThreadPoolExecutor( + 1, + 1, + EXECUTION_THREAD_KEEP_ALIVE_SECONDS, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(maximumConcurrency), + threadFactory); + this.platformPool.allowCoreThreadTimeOut(true); + this.executionPool = platformPool; + } + } + + /** + * @return the server's name for this queue: the task type, or {@code domain:taskType} for a domain queue + */ + static String queueName(String taskType, String domain) { + return StringUtils.isBlank(domain) ? taskType : domain + ":" + taskType; + } + + String taskType() { + return key.taskType(); + } + + EventDispatcher eventDispatcher() { + return eventDispatcher; + } + + void updateRegistration(BooleanSupplier pollable, EventDispatcher eventDispatcher) { + this.pollable = pollable; + this.eventDispatcher = eventDispatcher; + } + + String domain() { + return key.domain(); + } + + ExecutorService executionPool() { + return executionPool; + } + + int deficit() { + return targetConcurrency - heldSlots; + } + + /** + * @return arrival order among queues waiting for a process slot; later than any waiter when not waiting + */ + long waitSequence() { + return processWaiters > 0 ? processWaitSequence : Long.MAX_VALUE; + } + + /** + * Evaluates worker pause and poll filters. Runs user code, so it is never called under the controller lock. + */ + boolean isPollable() { + try { + return pollable.getAsBoolean(); + } catch (RuntimeException e) { + LOGGER.warn("Pause or poll filter check failed for {}; treating the worker as pollable", queueName, e); + return true; + } + } + + /** + * Sizes the platform pool to cover the target and all promised capacity. Threads are created lazily on + * submission and idle threads above the size retire after the keep-alive. + */ + void resizePool() { + if (platformPool == null) { + return; + } + int required = Math.max(1, Math.max(targetConcurrency, heldSlots)); + if (required > configuredPoolSize) { + platformPool.setMaximumPoolSize(required); + platformPool.setCorePoolSize(required); + } else if (required < configuredPoolSize) { + platformPool.setCorePoolSize(required); + platformPool.setMaximumPoolSize(required); + } + configuredPoolSize = required; + } +} diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/events/listeners/ListenerRegister.java b/conductor-client/src/main/java/com/netflix/conductor/client/events/listeners/ListenerRegister.java index 3e1adcd4c..45f11bec0 100644 --- a/conductor-client/src/main/java/com/netflix/conductor/client/events/listeners/ListenerRegister.java +++ b/conductor-client/src/main/java/com/netflix/conductor/client/events/listeners/ListenerRegister.java @@ -17,6 +17,7 @@ import com.netflix.conductor.client.events.task.TaskPayloadUsedEvent; import com.netflix.conductor.client.events.task.TaskResultPayloadSizeEvent; import com.netflix.conductor.client.events.taskrunner.ActiveWorkersChanged; +import com.netflix.conductor.client.events.taskrunner.AdaptiveConcurrencyChanged; import com.netflix.conductor.client.events.taskrunner.PollCompleted; import com.netflix.conductor.client.events.taskrunner.PollFailure; import com.netflix.conductor.client.events.taskrunner.PollStarted; @@ -70,6 +71,7 @@ public static void register(TaskRunnerEventsListener listener, EventDispatcher dispatcher) { @@ -87,6 +89,7 @@ public static void unregister(TaskRunnerEventsListener listener, EventDispatcher dispatcher.unregister(TaskPaused.class, listener); dispatcher.unregister(ThreadUncaughtException.class, listener); dispatcher.unregister(ActiveWorkersChanged.class, listener); + dispatcher.unregister(AdaptiveConcurrencyChanged.class, listener); } // --- TaskClientListener --- diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/events/listeners/TaskRunnerEventsListener.java b/conductor-client/src/main/java/com/netflix/conductor/client/events/listeners/TaskRunnerEventsListener.java index 7ed10793e..5b91e6f1b 100644 --- a/conductor-client/src/main/java/com/netflix/conductor/client/events/listeners/TaskRunnerEventsListener.java +++ b/conductor-client/src/main/java/com/netflix/conductor/client/events/listeners/TaskRunnerEventsListener.java @@ -13,6 +13,7 @@ package com.netflix.conductor.client.events.listeners; import com.netflix.conductor.client.events.taskrunner.ActiveWorkersChanged; +import com.netflix.conductor.client.events.taskrunner.AdaptiveConcurrencyChanged; import com.netflix.conductor.client.events.taskrunner.PollCompleted; import com.netflix.conductor.client.events.taskrunner.PollFailure; import com.netflix.conductor.client.events.taskrunner.PollStarted; @@ -57,4 +58,6 @@ default void consume(ThreadUncaughtException e) {} default void consume(ActiveWorkersChanged e) {} + default void consume(AdaptiveConcurrencyChanged e) {} + } diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/events/taskrunner/AdaptiveConcurrencyChanged.java b/conductor-client/src/main/java/com/netflix/conductor/client/events/taskrunner/AdaptiveConcurrencyChanged.java new file mode 100644 index 000000000..4f7d8f018 --- /dev/null +++ b/conductor-client/src/main/java/com/netflix/conductor/client/events/taskrunner/AdaptiveConcurrencyChanged.java @@ -0,0 +1,57 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.client.events.taskrunner; + +import lombok.Getter; +import lombok.ToString; + +/** + * Published by the adaptive concurrency controller when a worker's share or the process target changes, and after + * each queue-depth sample. + */ +@Getter +@ToString +public final class AdaptiveConcurrencyChanged extends TaskRunnerEvent { + private final String domain; + private final int targetConcurrency; + private final int heldSlots; + /** Seconds since a task slot was last released while any are held; a wedged worker's age keeps growing. */ + private final long taskSlotAgeSeconds; + private final int queueDepth; + private final int processTargetConcurrency; + private final int processHeldSlots; + private final boolean admissionPaused; + private final String reason; + + public AdaptiveConcurrencyChanged(String taskType, + String domain, + int targetConcurrency, + int heldSlots, + long taskSlotAgeSeconds, + int queueDepth, + int processTargetConcurrency, + int processHeldSlots, + boolean admissionPaused, + String reason) { + super(taskType); + this.domain = domain; + this.targetConcurrency = targetConcurrency; + this.heldSlots = heldSlots; + this.taskSlotAgeSeconds = taskSlotAgeSeconds; + this.queueDepth = queueDepth; + this.processTargetConcurrency = processTargetConcurrency; + this.processHeldSlots = processHeldSlots; + this.admissionPaused = admissionPaused; + this.reason = reason; + } +} diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/http/TaskClient.java b/conductor-client/src/main/java/com/netflix/conductor/client/http/TaskClient.java index 6dfd78170..0b9455f16 100644 --- a/conductor-client/src/main/java/com/netflix/conductor/client/http/TaskClient.java +++ b/conductor-client/src/main/java/com/netflix/conductor/client/http/TaskClient.java @@ -469,15 +469,25 @@ public void removeTaskFromQueue(String taskType, String taskId) { public int getQueueSizeForTask(String taskType) { Validate.notBlank(taskType, "Task type cannot be blank"); + Integer queueSize = getQueueSizesForTasks(List.of(taskType)).get(taskType); + return queueSize != null ? queueSize : 0; + } + + /** + * Retrieve the sizes of several task queues in one request. + * + * @param queueNames queue names: a task type for the default queue, or {@code domain:taskType} for a domain queue + * @return queue size keyed by the names passed; a name the server has never seen may be absent + */ + public Map getQueueSizesForTasks(List queueNames) { + Validate.notEmpty(queueNames, "Queue names cannot be empty"); ConductorClientRequest request = ConductorClientRequest.builder() .method(Method.GET) .path("/tasks/queue/sizes") - .addQueryParams("taskType", List.of(taskType)) + .addQueryParams("taskType", queueNames) .build(); ConductorClientResponse> response = client.execute(request, STRING_INT_MAP_TYPE); - - Integer queueSize = response.getData().get(taskType); - return queueSize != null ? queueSize : 0; + return response.getData(); } public int getQueueSizeForTask(String taskType, String domain, String isolationGroupId, String executionNamespace) { diff --git a/conductor-client/src/main/java/com/netflix/conductor/client/worker/Worker.java b/conductor-client/src/main/java/com/netflix/conductor/client/worker/Worker.java index bc5a7cb8d..523109f76 100644 --- a/conductor-client/src/main/java/com/netflix/conductor/client/worker/Worker.java +++ b/conductor-client/src/main/java/com/netflix/conductor/client/worker/Worker.java @@ -38,6 +38,7 @@ public interface Worker { String PROP_POLLER_COUNT = "poller_count"; String PROP_LEASE_EXTEND_ENABLED = "leaseExtendEnabled"; String PROP_PAUSED = "paused"; + String PROP_ADAPTIVE_MAX_CONCURRENCY = "adaptive_max_concurrency"; /** * Retrieve the name of the task definition the worker is currently working on. diff --git a/conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorkerExecutor.java b/conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorkerExecutor.java index e2ed6d107..8b4488db2 100644 --- a/conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorkerExecutor.java +++ b/conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorkerExecutor.java @@ -19,6 +19,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.netflix.conductor.client.automator.AdaptiveConcurrencyController; import com.netflix.conductor.client.automator.TaskRunnerConfigurer; import com.netflix.conductor.client.http.TaskClient; import com.netflix.conductor.client.metrics.MetricsCollector; @@ -48,8 +49,16 @@ public class AnnotatedWorkerExecutor { protected Map workerDomains = new HashMap<>(); + protected Map workerToAdaptiveMaxConcurrency = new HashMap<>(); + private MetricsCollector metricsCollector; + private AdaptiveConcurrencyController adaptiveConcurrencyController; + + private boolean useVirtualThreads; + + private boolean updateV2; + private final Set scannedPackages = new HashSet<>(); private final WorkerConfiguration workerConfiguration; @@ -208,6 +217,11 @@ private void addMethod(WorkerTask annotation, Method method, Object bean) { workerDomains.put(name, domain); } + int adaptiveMaxConcurrency = workerConfiguration.getAdaptiveMaxConcurrency(name); + if (adaptiveMaxConcurrency > 0) { + workerToAdaptiveMaxConcurrency.put(name, adaptiveMaxConcurrency); + } + AnnotatedWorker executor = new AnnotatedWorker(name, method, bean); executor.setPollingInterval(workerToPollingInterval.get(name)); @@ -243,10 +257,16 @@ public void startPolling() { var builder = new TaskRunnerConfigurer.Builder(taskClient, workers) .withTaskThreadCount(workerToThreadCount) .withTaskPollTimeout(workerToPollTimeout) - .withTaskToDomain(workerDomains); + .withTaskToDomain(workerDomains) + .withUseVirtualThreads(useVirtualThreads) + .withUpdateV2(updateV2); if (metricsCollector != null) { builder.withMetricsCollector(metricsCollector); } + if (adaptiveConcurrencyController != null) { + builder.withAdaptiveConcurrency(adaptiveConcurrencyController) + .withTaskAdaptiveMaxConcurrency(workerToAdaptiveMaxConcurrency); + } var oldTaskRunner = Optional.ofNullable(taskRunner); @@ -276,4 +296,32 @@ public MetricsCollector getMetricsCollector() { public void setMetricsCollector(MetricsCollector metricsCollector) { this.metricsCollector = metricsCollector; } + + public AdaptiveConcurrencyController getAdaptiveConcurrencyController() { + return adaptiveConcurrencyController; + } + + /** + * @param adaptiveConcurrencyController enables adaptive concurrency for the annotated workers; thread counts no + * longer apply. Takes effect at the next {@link #startPolling()}. + */ + public void setAdaptiveConcurrencyController(AdaptiveConcurrencyController adaptiveConcurrencyController) { + this.adaptiveConcurrencyController = adaptiveConcurrencyController; + } + + public boolean isUseVirtualThreads() { + return useVirtualThreads; + } + + public void setUseVirtualThreads(boolean useVirtualThreads) { + this.useVirtualThreads = useVirtualThreads; + } + + public boolean isUpdateV2() { + return updateV2; + } + + public void setUpdateV2(boolean updateV2) { + this.updateV2 = updateV2; + } } diff --git a/conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/WorkerConfiguration.java b/conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/WorkerConfiguration.java index d4946fb22..2d8de2f16 100644 --- a/conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/WorkerConfiguration.java +++ b/conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/WorkerConfiguration.java @@ -41,4 +41,11 @@ public int getPollTimeout(String taskName) { public int getPollerCount(String taskName) { return 0; } + + /** + * @return the worker's adaptive concurrency ceiling, or 0 to use the controller default + */ + public int getAdaptiveMaxConcurrency(String taskName) { + return 0; + } } diff --git a/conductor-client/src/test/java/com/netflix/conductor/client/automator/AdaptiveConcurrencyControllerTest.java b/conductor-client/src/test/java/com/netflix/conductor/client/automator/AdaptiveConcurrencyControllerTest.java new file mode 100644 index 000000000..454fbf84f --- /dev/null +++ b/conductor-client/src/test/java/com/netflix/conductor/client/automator/AdaptiveConcurrencyControllerTest.java @@ -0,0 +1,1051 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.client.automator; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.netflix.conductor.client.events.dispatcher.EventDispatcher; +import com.netflix.conductor.client.http.TaskClient; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class AdaptiveConcurrencyControllerTest { + + private static final long SECOND = TimeUnit.SECONDS.toNanos(1); + + private final AtomicLong clock = new AtomicLong(TimeUnit.HOURS.toNanos(1)); + private final FakeResourceSampler sampler = new FakeResourceSampler(); + private final TaskClient taskClient = mock(TaskClient.class); + private AdaptiveConcurrencyController controller; + + @BeforeEach + void setUp() { + sampler.cpu = 0.20; + sampler.heap = new ResourceSampler.HeapSample(0.30, AdaptiveConcurrencyController.NONE); + } + + @AfterEach + void tearDown() { + if (controller != null) { + List.copyOf(registered).forEach(controller::deregister); + } + } + + private final java.util.List registered = new java.util.ArrayList<>(); + + private AdaptiveConcurrencyController controller(int initialProcessConcurrency) { + return controller(new AdaptiveConcurrencyConfig.Builder() + .withInitialProcessConcurrency(initialProcessConcurrency) + .build()); + } + + /** Standby polls disabled, so a queue without free share only heartbeats. */ + private AdaptiveConcurrencyController heartbeatOnlyController(int initialProcessConcurrency) { + return controller(new AdaptiveConcurrencyConfig.Builder() + .withInitialProcessConcurrency(initialProcessConcurrency) + .withStandbyPollTimeout(Duration.ZERO) + .build()); + } + + private AdaptiveConcurrencyController controller(AdaptiveConcurrencyConfig config) { + controller = new AdaptiveConcurrencyController(config, () -> sampler, clock::get, false); + return controller; + } + + private WorkerQueue register(String taskType, String domain) { + WorkerQueue queue = controller.register(taskClient, taskType, domain, 64, 100, () -> true, + Executors.defaultThreadFactory(), false, new EventDispatcher<>()); + registered.add(queue); + return queue; + } + + private void depths(Map sizes) { + when(taskClient.getQueueSizesForTasks(anyList())).thenReturn(sizes); + controller.sampleDepths(); + } + + private void advance(long nanos) { + clock.addAndGet(nanos); + } + + private SlotReservation reserve(WorkerQueue queue) { + Object outcome = controller.tryReserve(queue, clock.get()); + return outcome instanceof SlotReservation reservation ? reservation : null; + } + + @Test + void sharesFollowDepthWithLargestRemainderRounding() { + controller(32); + WorkerQueue renderPdf = register("render_pdf", null); + WorkerQueue sendEmail = register("send_email", null); + WorkerQueue nightlyReport = register("nightly_report", null); + controller.activate(); + renderPdf.heldSlots = 12; + + depths(Map.of("render_pdf", 48, "send_email", 6, "nightly_report", 0)); + + assertEquals(29, renderPdf.targetConcurrency); + assertEquals(3, sendEmail.targetConcurrency); + assertEquals(0, nightlyReport.targetConcurrency); + renderPdf.heldSlots = 0; + } + + @Test + void sharesFollowDepthBeyondTheWorkerMaximum() { + // Backlogs deeper than the worker maximum must still be weighted by depth, not flattened to the maximum. + controller(16); + WorkerQueue deep = register("deep", null); + WorkerQueue medium = register("medium", null); + WorkerQueue shallow = register("shallow", null); + WorkerQueue trickle = register("trickle", null); + controller.activate(); + + depths(Map.of("deep", 600, "medium", 300, "shallow", 150, "trickle", 30)); + + assertEquals(9, deep.targetConcurrency); + assertEquals(4, medium.targetConcurrency); + assertEquals(2, shallow.targetConcurrency); + assertEquals(1, trickle.targetConcurrency); + } + + @Test + void capReachedByOneQueueIsRedistributedToTheOthers() { + controller(new AdaptiveConcurrencyConfig.Builder().withInitialProcessConcurrency(20).build()); + WorkerQueue capped = controller.register(taskClient, "capped", null, 4, 100, () -> true, + Executors.defaultThreadFactory(), false, new EventDispatcher<>()); + registered.add(capped); + WorkerQueue other = register("other", null); + WorkerQueue third = register("third", null); + controller.activate(); + + depths(Map.of("capped", 1000, "other", 100, "third", 100)); + + assertEquals(4, capped.targetConcurrency, "share never exceeds the worker maximum"); + assertEquals(8, other.targetConcurrency); + assertEquals(8, third.targetConcurrency); + } + + @Test + void unknownDepthGivesEveryQueueAShareOfOne() { + controller(1); + WorkerQueue a = register("a", null); + WorkerQueue b = register("b", null); + WorkerQueue c = register("c", null); + controller.activate(); + + assertEquals(1, a.targetConcurrency); + assertEquals(1, b.targetConcurrency); + assertEquals(1, c.targetConcurrency); + } + + @Test + void depthIsSampledWithOneRequestNamingEveryQueue() { + controller(4); + register("a", null); + WorkerQueue b = register("b", "tenant1"); + controller.activate(); + + depths(Map.of("a", 3, "tenant1:b", 5)); + + verify(taskClient, times(1)).getQueueSizesForTasks(List.of("a", "tenant1:b")); + assertEquals(5, b.depth); + } + + @Test + void failedDepthSampleCountsAsOne() { + controller(8); + WorkerQueue a = register("a", null); + controller.activate(); + depths(Map.of("a", 50)); + assertEquals(8, a.targetConcurrency); + + when(taskClient.getQueueSizesForTasks(anyList())).thenThrow(new RuntimeException("unavailable")); + controller.sampleDepths(); + + assertEquals(1, a.targetConcurrency); + } + + @Test + void pausedWorkerHasNoWaitingDemand() { + controller(8); + WorkerQueue a = controller.register(taskClient, "a", null, 64, 100, () -> false, + Executors.defaultThreadFactory(), false, new EventDispatcher<>()); + registered.add(a); + controller.activate(); + + depths(Map.of("a", 50)); + + assertEquals(0, a.targetConcurrency); + } + + @Test + void reservationIsSizedByShareProcessTargetAndBatch() { + controller(32); + WorkerQueue a = register("a", null); + controller.activate(); + depths(Map.of("a", 100)); + + SlotReservation first = reserve(a); + assertNotNull(first); + assertEquals(16, first.count()); + SlotReservation second = reserve(a); + assertNotNull(second); + assertEquals(16, second.count()); + assertEquals(32, controller.getHeldSlots()); + assertTrue(reserve(a) == null); + } + + @Test + void emptyResponseLimitsDemandToOneTaskUntilDelivery() { + controller(32); + WorkerQueue a = register("a", null); + controller.activate(); + depths(Map.of("a", 48)); + assertEquals(32, a.targetConcurrency); + + SlotReservation reservation = reserve(a); + controller.completePoll(reservation, 0, clock.get()); + + assertEquals(1, a.targetConcurrency); + assertEquals(0, controller.getHeldSlots()); + assertTrue(controller.pollDelayNanos(a) > 0); + + advance(SECOND); + reservation = reserve(a); + assertNotNull(reservation); + assertEquals(1, reservation.count()); + controller.completePoll(reservation, 1, clock.get()); + depths(Map.of("a", 48)); + assertEquals(32, a.targetConcurrency); + } + + @Test + void freedProcessSlotGoesToTheQueueFurthestBelowItsShare() { + controller(10); + WorkerQueue deep = register("deep", null); + WorkerQueue shallow = register("shallow", null); + controller.activate(); + depths(Map.of("deep", 80, "shallow", 20)); + assertEquals(8, deep.targetConcurrency); + assertEquals(2, shallow.targetConcurrency); + + // The deep queue waits for a process slot with a larger deficit than the shallow queue. + deep.processWaiters = 1; + deep.processWaitSequence = 1; + assertFalse(controller.tryReserve(shallow, clock.get()) instanceof SlotReservation reservation + && reservation.count() > 0, "no task for the shallow queue while the deep one waits"); + + deep.processWaiters = 0; + Object outcome = controller.tryReserve(shallow, clock.get()); + assertTrue(outcome instanceof SlotReservation reservation && reservation.count() > 0); + } + + @Test + void queueWithoutShareSendsOneHeartbeatPerInterval() { + heartbeatOnlyController(4); + WorkerQueue idle = register("idle", null); + WorkerQueue busy = register("busy", null); + controller.activate(); + depths(Map.of("idle", 0, "busy", 2)); + assertEquals(0, idle.targetConcurrency); + + SlotReservation heartbeat = reserve(idle); + assertNotNull(heartbeat); + assertTrue(heartbeat.heartbeat()); + assertEquals(1, heartbeat.count()); + controller.completePoll(heartbeat, 0, clock.get()); + + advance(SECOND); + assertTrue(reserve(idle) == null, "heartbeat is at most once per interval"); + + advance(5 * SECOND); + assertNotNull(reserve(idle), "the next interval allows the next heartbeat"); + } + + @Test + void idleQueuesTakeTurnsOnTheFreeSlotWithinOneInterval() { + // An idle process decays to one slot. Every idle queue must still heartbeat each interval, or the server's + // domain fallback stops routing to the domains that lose, yet no queue is polled without an open slot. + heartbeatOnlyController(1); + List idle = List.of(register("a", "partner"), register("b", null), register("c", null), + register("d", null)); + controller.activate(); + depths(Map.of("partner:a", 0, "b", 0, "c", 0, "d", 0)); + + for (WorkerQueue queue : idle) { + SlotReservation heartbeat = reserve(queue); + assertNotNull(heartbeat, queue.key.taskType() + " heartbeat"); + assertTrue(heartbeat.heartbeat()); + assertEquals(1, heartbeat.count()); + assertEquals(1, controller.getHeldSlots(), "one open slot, one poll"); + for (WorkerQueue other : idle) { + if (other != queue && other.outstandingPolls == 0 && other.nextHeartbeatAt <= clock.get()) { + assertTrue(reserve(other) == null, "no poll without an open slot"); + } + } + controller.completePoll(heartbeat, 0, clock.get()); + advance(TimeUnit.MILLISECONDS.toNanos(20)); + } + assertEquals(0, controller.getHeldSlots()); + assertTrue(idle.stream().allMatch(q -> q.nextHeartbeatAt > clock.get()), "every queue polled this interval"); + } + + @Test + void noPollWhileAdmissionIsPaused() { + controller(4); + WorkerQueue a = register("a", "partner"); + controller.activate(); + depths(Map.of("partner:a", 0)); + sampler.heap = new ResourceSampler.HeapSample(0.95, clock.get()); + controller.sampleResources(); + assertTrue(controller.isAdmissionPaused()); + + assertTrue(reserve(a) == null); + } + + @Test + void queueWithHeldShareKeepsOneStandbyPollOpen() { + controller(4); + WorkerQueue idle = register("idle", null); + WorkerQueue busy = register("busy", null); + controller.activate(); + depths(Map.of("idle", 0, "busy", 2)); + assertEquals(0, idle.targetConcurrency); + + SlotReservation standby = reserve(idle); + assertNotNull(standby); + assertTrue(standby.standby()); + assertEquals(1, standby.count()); + assertTrue(reserve(idle) == null, "one standby poll at a time"); + + controller.completePoll(standby, 0, clock.get()); + assertEquals(0, idle.consecutiveEmptyPolls, "a standby wait running out is not an empty queue"); + assertEquals(0, controller.pollDelayNanos(idle)); + advance(SECOND); + SlotReservation next = reserve(idle); + assertNotNull(next, "the next standby poll follows when the previous one's wait has passed"); + assertTrue(next.standby()); + controller.completePoll(next, 0, clock.get()); + assertEquals(0, controller.getHeldSlots()); + } + + @Test + void standbyUsesOnlySlotsNoShareClaims() { + controller(3); + WorkerQueue busy = register("busy", null); + WorkerQueue idle = register("idle", null); + controller.activate(); + depths(Map.of("busy", 3, "idle", 0)); + assertEquals(3, busy.targetConcurrency); + idle.nextHeartbeatAt = clock.get() + 5 * SECOND; + + assertTrue(reserve(idle) == null, "all three slots belong to the busy queue's share, polled or not"); + + depths(Map.of("busy", 2, "idle", 0)); + SlotReservation standby = reserve(idle); + assertNotNull(standby, "the slot no share claims"); + assertTrue(standby.standby()); + SlotReservation work = reserve(busy); + assertEquals(2, work.count(), "the share is still whole"); + controller.completePoll(work, 0, clock.get()); + controller.completePoll(standby, 0, clock.get()); + } + + @Test + void standbyNeedsAFreeProcessSlot() { + // Unclaimed slots are computed over registered queues, so a deregistered worker's draining tasks could + // otherwise let a standby poll admit past the process target. + controller(2); + WorkerQueue leaving = register("leaving", null); + WorkerQueue idle = register("idle", null); + controller.activate(); + depths(Map.of("leaving", 5, "idle", 0)); + + SlotReservation work = reserve(leaving); + assertEquals(2, work.count()); + controller.deregister(leaving); // its two accepted tasks are still draining + registered.remove(leaving); + idle.nextHeartbeatAt = clock.get() + 5 * SECOND; + + assertTrue(reserve(idle) == null, "every process slot is still held by the departed queue's tasks"); + assertEquals(2, controller.getHeldSlots()); + controller.releaseTaskSlot(leaving); + controller.releaseTaskSlot(leaving); + } + + @Test + void doubleSlotReleaseIsClampedAndReported() { + controller(4); + WorkerQueue queue = register("pay", null); + controller.activate(); + depths(Map.of("pay", 2)); + SlotReservation work = reserve(queue); + assertEquals(2, work.count()); + controller.completePoll(work, 2, clock.get()); // both delivered: the tasks own the slots + + controller.releaseTaskSlot(queue); + controller.releaseTaskSlot(queue); + controller.releaseTaskSlot(queue); // a release bug fires once too often + + assertEquals(0, controller.getHeldSlots(), "the breach is clamped instead of inflating future grants"); + assertEquals(0, queue.heldSlots); + SlotReservation next = reserve(queue); + assertEquals(2, next.count(), "the deficit is not inflated by the breach"); + controller.completePoll(next, 0, clock.get()); + } + + @Test + void taskSlotAgeRunsFromDeliveryOrLastRelease() { + controller(4); + WorkerQueue queue = register("age", null); + controller.activate(); + depths(Map.of("age", 3)); + + SlotReservation work = reserve(queue); + assertEquals(3, queue.reservedPollSlots, "poll reservations are not task slots"); + advance(10 * SECOND); + controller.completePoll(work, 2, clock.get()); + assertEquals(0, queue.reservedPollSlots); + assertEquals(2, queue.heldSlots); + assertEquals(clock.get(), queue.lastTaskSlotReleaseAt, "the age restarts when an idle queue accepts work"); + + advance(10 * SECOND); + controller.releaseTaskSlot(queue); + assertEquals(clock.get(), queue.lastTaskSlotReleaseAt, "each task release restarts the age"); + controller.releaseTaskSlot(queue); + } + + @Test + void standbyDeliveryKeepsItsSlotAndCountsAsBacklog() { + controller(4); + WorkerQueue queue = register("a", null); + controller.activate(); + depths(Map.of("a", 0)); + assertEquals(0, queue.targetConcurrency); + + SlotReservation standby = reserve(queue); + controller.completePoll(standby, 1, clock.get()); + assertEquals(1, controller.getHeldSlots(), "the task keeps the slot"); + assertEquals(0, queue.standbyPolls); + depths(Map.of("a", 0)); + assertEquals(1, queue.targetConcurrency, "the running task counts as backlog"); + + SlotReservation next = reserve(queue); + assertNotNull(next, "the next standby poll goes out at once"); + assertTrue(next.standby()); + controller.completePoll(next, 0, clock.get()); + controller.releaseTaskSlot(queue); + } + + @Test + void failedStandbyPollBacksOff() { + controller(4); + WorkerQueue queue = register("a", null); + controller.activate(); + depths(Map.of("a", 0)); + + controller.failPoll(reserve(queue), clock.get()); + assertEquals(1, queue.consecutiveEmptyPolls); + assertTrue(controller.pollDelayNanos(queue) > 0); + assertEquals(0, controller.getHeldSlots()); + } + + @Test + void idleProcessKeepsRoomForOneStandbyPerQueue() { + controller(1); + register("a", null); + register("b", "partner"); + register("c", null); + controller.activate(); + for (int i = 0; i < 12; i++) { + advance(5 * SECOND); + controller.sampleResources(); + depths(Map.of("a", 0, "partner:b", 0, "c", 0)); + controller.controlTick(); + } + assertEquals(3, controller.getTargetConcurrency(), "one standby slot per idle queue, no more"); + } + + @Test + void noStandbyPollWhenDisabled() { + heartbeatOnlyController(4); + WorkerQueue queue = register("a", null); + controller.activate(); + depths(Map.of("a", 0)); + + SlotReservation reservation = reserve(queue); + assertTrue(reservation.heartbeat()); + controller.completePoll(reservation, 0, clock.get()); + for (int i = 0; i < 24; i++) { + advance(5 * SECOND); + controller.sampleResources(); + depths(Map.of("a", 0)); + controller.controlTick(); + } + assertEquals(1, controller.getTargetConcurrency(), "no room is kept for standby polls"); + } + + @Test + void dueHeartbeatGetsTheNextOpenSlotBeforeQueuesBelowTheirShare() { + // Under saturation freed slots went to queues below their share, so an idle domain never polled, went stale + // and its tasks fell back to another domain. A due heartbeat now takes the next open slot, once per interval. + controller(2); + WorkerQueue a = register("a", null); + WorkerQueue b = register("b", null); + WorkerQueue starved = register("starved", null); + WorkerQueue idle = register("idle", "partner"); + controller.activate(); + depths(Map.of("a", 100, "b", 100, "starved", 100, "partner:idle", 0)); + SlotReservation first = reserve(a); + SlotReservation second = reserve(b); + controller.completePoll(first, first.count(), clock.get()); + controller.completePoll(second, second.count(), clock.get()); + assertEquals(2, controller.getHeldSlots(), "the process is full"); + + assertTrue(reserve(idle) == null, "no open slot, no poll"); + assertTrue(reserve(starved) == null); + idle.heartbeatWaiters = 1; + controller.releaseTaskSlot(a); + + assertTrue(reserve(starved) == null, "the open slot goes to the due heartbeat first"); + SlotReservation heartbeat = reserve(idle); + assertNotNull(heartbeat); + assertEquals(1, heartbeat.count()); + idle.heartbeatWaiters = 0; + controller.completePoll(heartbeat, 0, clock.get()); + SlotReservation work = reserve(starved); + assertNotNull(work, "then to the queue below its share"); + controller.completePoll(work, work.count(), clock.get()); + controller.releaseTaskSlot(b); + controller.releaseTaskSlot(starved); + } + + @Test + void arrivalsAfterAnEmptyPollLiftTheCapAndTheBackoff() { + // After idle heartbeats came back empty, a burst of 200 waited five seconds on a share of one. + heartbeatOnlyController(16); + WorkerQueue a = register("a", null); + controller.activate(); + depths(Map.of("a", 0)); + for (int i = 0; i < 6; i++) { + SlotReservation heartbeat = reserve(a); + assertNotNull(heartbeat); + controller.completePoll(heartbeat, 0, clock.get()); + advance(6 * SECOND); + } + assertTrue(a.consecutiveEmptyPolls > 0); + + depths(Map.of("a", 200)); + assertEquals(16, a.targetConcurrency, "the new arrivals count in full"); + assertEquals(0, controller.pollDelayNanos(a), "and the queue polls without waiting out its backoff"); + } + + @Test + void undeliverableDepthStaysCappedToNewArrivals() { + controller(16); + WorkerQueue limited = register("limited", null); + controller.activate(); + depths(Map.of("limited", 1000)); + SlotReservation poll = reserve(limited); + controller.completePoll(poll, 0, clock.get()); + assertEquals(1, limited.targetConcurrency, "the server delivered nothing from a deep queue"); + + depths(Map.of("limited", 1003)); + assertEquals(4, limited.targetConcurrency, "only the three new tasks and one more count"); + } + + @Test + void processTargetGrowsTowardDemandWithHeadroom() { + controller(4); + WorkerQueue a = register("a", null); + controller.activate(); + depths(Map.of("a", 1000)); + controller.sampleResources(); + + controller.controlTick(); + assertEquals(5, controller.getTargetConcurrency()); + for (int i = 0; i < 50; i++) { + advance(5 * SECOND); + controller.sampleResources(); + depths(Map.of("a", 1000)); + controller.controlTick(); + } + assertEquals(64, controller.getTargetConcurrency(), "capped at demand, which the worker maximum caps"); + assertEquals(64, a.targetConcurrency); + } + + @Test + void idleProcessDecaysToTheMinimumInsteadOfOne() { + controller(new AdaptiveConcurrencyConfig.Builder() + .withInitialProcessConcurrency(8) + .withMinProcessConcurrency(32) + .build()); + register("a", null); + controller.activate(); + assertEquals(32, controller.getTargetConcurrency(), "the start is never below the minimum"); + + depths(Map.of("a", 1000)); + for (int i = 0; i < 30; i++) { + advance(5 * SECOND); + controller.sampleResources(); + depths(Map.of("a", 1000)); + controller.controlTick(); + } + assertEquals(64, controller.getTargetConcurrency()); + + for (int i = 0; i < 60; i++) { + advance(5 * SECOND); + controller.sampleResources(); + depths(Map.of("a", 0)); + controller.controlTick(); + } + assertEquals(32, controller.getTargetConcurrency(), "decay stops at the minimum"); + } + + @Test + void cpuPressureMayReduceBelowTheMinimum() { + controller(new AdaptiveConcurrencyConfig.Builder().withMinProcessConcurrency(8).build()); + register("a", null); + controller.activate(); + depths(Map.of("a", 1000)); + sampler.cpu = 1.0; + for (int i = 0; i < 4; i++) { + controller.sampleResources(); + advance(SECOND); + } + controller.controlTick(); + assertEquals(6, controller.getTargetConcurrency(), "safety outranks the minimum"); + + for (int i = 0; i < 12; i++) { + advance(5 * SECOND); + controller.sampleResources(); + depths(Map.of("a", 0)); + controller.controlTick(); + } + assertTrue(controller.getTargetConcurrency() <= 6, "decay never raises the target back to the minimum"); + } + + @Test + void minimumMustFitTheCeiling() { + assertThrows(IllegalArgumentException.class, () -> new AdaptiveConcurrencyConfig.Builder() + .withMaxProcessConcurrency(16).withMinProcessConcurrency(17).build()); + assertThrows(IllegalArgumentException.class, () -> new AdaptiveConcurrencyConfig.Builder() + .withMinProcessConcurrency(0).build()); + } + + @Test + void growthNeedsFreshMetrics() { + controller(4); + register("a", null); + controller.activate(); + depths(Map.of("a", 1000)); + controller.sampleResources(); + advance(6 * SECOND); + depths(Map.of("a", 1000)); + + controller.controlTick(); + + assertEquals(4, controller.getTargetConcurrency()); + } + + @Test + void sustainedCpuPressureReducesOncePerInterval() { + controller(40); + register("a", null); + controller.activate(); + depths(Map.of("a", 1000)); + sampler.cpu = 0.95; + for (int i = 0; i <= 3; i++) { + controller.sampleResources(); + advance(SECOND); + } + controller.controlTick(); + assertEquals(30, controller.getTargetConcurrency()); + + for (int i = 0; i < 7; i++) { + controller.sampleResources(); + advance(SECOND); + } + controller.controlTick(); + assertEquals(30, controller.getTargetConcurrency(), "next reduction waits for the reduction interval"); + + for (int i = 0; i < 8; i++) { + controller.sampleResources(); + advance(SECOND); + } + controller.controlTick(); + assertEquals(22, controller.getTargetConcurrency()); + } + + @Test + void highPostCollectionHeapPausesAdmissionAndResumesBelowThreshold() { + controller(20); + WorkerQueue a = register("a", null); + controller.activate(); + depths(Map.of("a", 1000)); + + sampler.heap = new ResourceSampler.HeapSample(0.95, clock.get()); + controller.sampleResources(); + assertTrue(controller.isAdmissionPaused()); + assertTrue(reserve(a) == null); + + advance(SECOND); + sampler.heap = new ResourceSampler.HeapSample(0.85, clock.get()); + controller.sampleResources(); + assertTrue(controller.isAdmissionPaused(), "a reading between the thresholds stays paused"); + + advance(SECOND); + sampler.heap = new ResourceSampler.HeapSample(0.60, clock.get()); + controller.sampleResources(); + assertFalse(controller.isAdmissionPaused()); + + controller.controlTick(); + assertEquals(15, controller.getTargetConcurrency(), "the high collection still costs one reduction"); + } + + @Test + void heapPressureCapsGrowthAtTheReducedTarget() { + // Re-growing past the level that caused heap pressure ran tasks out of memory end to end. + controller(20); + WorkerQueue a = register("a", null); + controller.activate(); + depths(Map.of("a", 1000)); + sampler.heap = new ResourceSampler.HeapSample(0.95, clock.get()); + controller.sampleResources(); + advance(SECOND); + sampler.heap = new ResourceSampler.HeapSample(0.30, clock.get()); + controller.sampleResources(); + controller.controlTick(); + assertEquals(15, controller.getTargetConcurrency()); + + for (int i = 0; i < 20; i++) { + advance(5 * SECOND); + sampler.heap = new ResourceSampler.HeapSample(0.30, clock.get()); + controller.sampleResources(); + depths(Map.of("a", 1000)); + controller.controlTick(); + } + assertEquals(15, controller.getTargetConcurrency(), "no growth past the pressure level for two minutes"); + + for (int i = 0; i < 6; i++) { + advance(5 * SECOND); + sampler.heap = new ResourceSampler.HeapSample(0.30, clock.get()); + controller.sampleResources(); + depths(Map.of("a", 1000)); + controller.controlTick(); + } + assertTrue(controller.getTargetConcurrency() > 15, "the ceiling relaxes after the hold"); + assertTrue(controller.getTargetConcurrency() <= 21, "one slot per control tick"); + assertEquals(controller.getTargetConcurrency(), a.targetConcurrency); + } + + @Test + void heapPressureAfterTheHoldReducesAgain() { + // End to end the target regrew after the hold, then sat through three more heap pauses unreduced. + controller(20); + register("a", null); + controller.activate(); + depths(Map.of("a", 1000)); + sampler.heap = new ResourceSampler.HeapSample(0.95, clock.get()); + controller.sampleResources(); + controller.controlTick(); + assertEquals(15, controller.getTargetConcurrency()); + for (int i = 0; i < 30; i++) { + advance(5 * SECOND); + sampler.heap = new ResourceSampler.HeapSample(0.30, clock.get()); + controller.sampleResources(); + depths(Map.of("a", 1000)); + controller.controlTick(); + } + int regrown = controller.getTargetConcurrency(); + assertTrue(regrown > 15); + + advance(5 * SECOND); + sampler.heap = new ResourceSampler.HeapSample(0.95, clock.get()); + controller.sampleResources(); + assertTrue(controller.isAdmissionPaused()); + controller.controlTick(); + assertEquals((int) Math.floor(0.75 * regrown), controller.getTargetConcurrency()); + } + + @Test + void controlTicksContinueAfterOutOfMemoryErrors() throws InterruptedException { + // End to end the target never moved again after the heap ran out during a tick. + var failures = new java.util.concurrent.atomic.AtomicInteger(3); + controller = new AdaptiveConcurrencyController(new AdaptiveConcurrencyConfig.Builder() + .withInitialProcessConcurrency(1) + .withControlInterval(Duration.ofMillis(100)) + .withResourceSampleInterval(Duration.ofMillis(20)) + .withMetricFreshness(Duration.ofSeconds(1)) + .withDepthSampleInterval(Duration.ofMillis(20)) + .withDepthFreshness(Duration.ofSeconds(1)) + .build(), () -> sampler, () -> { + if (failures.get() > 0 && failures.decrementAndGet() >= 0) { + throw new OutOfMemoryError("injected"); + } + return System.nanoTime(); + }, true); + when(taskClient.getQueueSizesForTasks(anyList())).thenReturn(Map.of("a", 1000)); + failures.set(0); + register("a", null); + controller.activate(); + failures.set(20); + + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (controller.getTargetConcurrency() < 4 && System.nanoTime() < deadline) { + Thread.sleep(20); + } + assertEquals(0, failures.get(), "the injected failures were hit"); + assertTrue(controller.getTargetConcurrency() >= 4, "control ticks continue after the failures"); + } + + @Test + void growthUsesTheHighestRecentHeapReading() { + // A low reading right after a high one let the target grow into an OutOfMemoryError end to end. + controller(5); + WorkerQueue a = register("a", null); + controller.activate(); + depths(Map.of("a", 1000)); + sampler.heap = new ResourceSampler.HeapSample(0.10, clock.get()); + controller.sampleResources(); + SlotReservation running = reserve(a); + controller.completePoll(running, running.count(), clock.get()); + assertEquals(5, controller.getHeldSlots()); + + advance(SECOND); + sampler.heap = new ResourceSampler.HeapSample(0.70, clock.get()); + controller.sampleResources(); + advance(4 * SECOND); + sampler.heap = new ResourceSampler.HeapSample(0.40, clock.get()); + controller.sampleResources(); + depths(Map.of("a", 1000)); + controller.controlTick(); + assertEquals(5, controller.getTargetConcurrency(), "0.70 of the last 30 seconds leaves no room"); + + for (int i = 0; i < 6; i++) { + advance(5 * SECOND); + sampler.heap = new ResourceSampler.HeapSample(0.40, clock.get()); + controller.sampleResources(); + depths(Map.of("a", 1000)); + controller.controlTick(); + } + assertEquals(6, controller.getTargetConcurrency(), "growth resumes once the high reading is 30 seconds old"); + for (int i = 0; i < 5; i++) { + controller.releaseTaskSlot(a); + } + } + + @Test + void growthStopsWhereProjectedHeapReachesTheGrowthThreshold() { + controller(10); + WorkerQueue a = register("a", null); + controller.activate(); + depths(Map.of("a", 1000)); + sampler.heap = new ResourceSampler.HeapSample(0.10, clock.get()); + controller.sampleResources(); + SlotReservation running = reserve(a); + assertEquals(10, running.count()); + controller.completePoll(running, 10, clock.get()); + // 0.06 of the heap per slot: at 0.70 one more slot would cross the 0.75 growth threshold. + advance(SECOND); + sampler.heap = new ResourceSampler.HeapSample(0.70, clock.get()); + controller.sampleResources(); + controller.controlTick(); + assertEquals(10, controller.getTargetConcurrency()); + + // 0.03 per slot at 0.40 once 0.70 is older than 30 seconds: room for eleven, so the ordinary step of two. + advance(31 * SECOND); + sampler.heap = new ResourceSampler.HeapSample(0.40, clock.get()); + controller.sampleResources(); + depths(Map.of("a", 1000)); + controller.controlTick(); + assertEquals(12, controller.getTargetConcurrency()); + for (int i = 0; i < 10; i++) { + controller.releaseTaskSlot(a); + } + } + + @Test + void usedHeapBeforeFirstCollectionNeverPauses() { + controller(4); + register("a", null); + controller.activate(); + + sampler.heap = new ResourceSampler.HeapSample(0.97, AdaptiveConcurrencyController.NONE); + controller.sampleResources(); + + assertFalse(controller.isAdmissionPaused()); + } + + @Test + void idlePausedProcessResumesAtOneAfterDrain() { + controller(20); + register("a", null); + controller.activate(); + long collectionAt = clock.get(); + sampler.heap = new ResourceSampler.HeapSample(0.95, collectionAt); + controller.sampleResources(); + assertTrue(controller.isAdmissionPaused()); + + sampler.heap = new ResourceSampler.HeapSample(0.85, collectionAt + 1); + advance(29 * SECOND); + controller.sampleResources(); + assertTrue(controller.isAdmissionPaused()); + + advance(SECOND); + controller.sampleResources(); + assertFalse(controller.isAdmissionPaused()); + assertEquals(1, controller.getTargetConcurrency()); + + advance(SECOND); + controller.sampleResources(); + assertFalse(controller.isAdmissionPaused(), "the stale reading does not re-enter the pause"); + } + + @Test + void surplusDecaysTowardDemand() { + controller(32); + register("a", null); + controller.activate(); + depths(Map.of("a", 2)); + + controller.controlTick(); + advance(30 * SECOND); + depths(Map.of("a", 2)); + controller.controlTick(); + assertEquals(16, controller.getTargetConcurrency()); + + for (int i = 0; i < 5; i++) { + advance(30 * SECOND); + depths(Map.of("a", 2)); + controller.controlTick(); + } + assertEquals(2, controller.getTargetConcurrency(), "never below demand"); + } + + @Test + void replacementNeedsTheFreedSlotToComeBackToTheSameQueue() { + controller(10); + WorkerQueue a = register("a", null); + WorkerQueue b = register("b", null); + controller.activate(); + depths(Map.of("a", 50, "b", 50)); + SlotReservation reservation = reserve(a); + controller.completePoll(reservation, reservation.count(), clock.get()); + + assertTrue(controller.isReplacementAllowed(a)); + + b.processWaiters = 1; + b.processWaitSequence = 1; + assertFalse(controller.isReplacementAllowed(a), "b is further below its share"); + b.processWaiters = 0; + + sampler.heap = new ResourceSampler.HeapSample(0.95, clock.get()); + controller.sampleResources(); + assertFalse(controller.isReplacementAllowed(a), "no replacement while paused"); + + for (int i = 0; i < reservation.count(); i++) { + controller.releaseTaskSlot(a); + } + } + + @Test + void emptyPollBackoffIsCapped() { + controller(4); + WorkerQueue a = register("a", null); + controller.activate(); + for (int i = 0; i < 12; i++) { + SlotReservation reservation = reserve(a); + if (reservation == null) { + advance(10 * SECOND); + reservation = reserve(a); + } + controller.completePoll(reservation, 0, clock.get()); + } + long delay = controller.pollDelayNanos(a); + assertTrue(delay <= 6 * SECOND && delay >= 4 * SECOND, "backoff capped at about five seconds: " + delay); + } + + @Test + void startupFailsWithoutMetricsUnlessDemandOnly() { + sampler.cpuSupported = false; + sampler.heapSupported = false; + controller(4); + register("a", null); + assertThrows(IllegalStateException.class, controller::activate); + + controller(new AdaptiveConcurrencyConfig.Builder().withDemandOnlyEnabled(true).build()); + registered.clear(); + WorkerQueue a = register("a", null); + controller.activate(); + assertEquals(1, a.targetConcurrency); + } + + @Test + void configRejectsInconsistentThresholds() { + assertThrows(IllegalArgumentException.class, + () -> new AdaptiveConcurrencyConfig.Builder().withHeapResumeThreshold(0.95).build()); + assertThrows(IllegalArgumentException.class, + () -> new AdaptiveConcurrencyConfig.Builder().withMaxPollBatch(0).build()); + assertThrows(IllegalArgumentException.class, + () -> new AdaptiveConcurrencyConfig.Builder().withDepthFreshness(java.time.Duration.ofMillis(500)).build()); + } + + static class FakeResourceSampler implements ResourceSampler { + volatile boolean cpuSupported = true; + volatile boolean heapSupported = true; + volatile double cpu; + volatile HeapSample heap; + + @Override + public boolean isCpuSupported() { + return cpuSupported; + } + + @Override + public boolean isHeapSupported() { + return heapSupported; + } + + @Override + public double sampleCpu() { + return cpu; + } + + @Override + public HeapSample sampleHeap() { + return heap; + } + + @Override + public void close() {} + } +} diff --git a/conductor-client/src/test/java/com/netflix/conductor/client/automator/JvmResourceSamplerTest.java b/conductor-client/src/test/java/com/netflix/conductor/client/automator/JvmResourceSamplerTest.java new file mode 100644 index 000000000..2d8707a6f --- /dev/null +++ b/conductor-client/src/test/java/com/netflix/conductor/client/automator/JvmResourceSamplerTest.java @@ -0,0 +1,132 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.client.automator; + +import java.lang.management.MemoryUsage; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class JvmResourceSamplerTest { + + private final JvmResourceSampler sampler = new JvmResourceSampler(); + + @AfterEach + void tearDown() { + sampler.close(); + } + + @Test + void readsHeapAfterACollection() throws Exception { + assertTrue(sampler.isHeapSupported()); + + System.gc(); + ResourceSampler.HeapSample sample = null; + for (int i = 0; i < 50; i++) { + sample = sampler.sampleHeap(); + if (sample != null && sample.collectionAtNanos() != AdaptiveConcurrencyController.NONE) { + break; + } + Thread.sleep(20); + } + + assertNotNull(sample); + assertTrue(sample.utilization() > 0 && sample.utilization() < 1, "utilization " + sample.utilization()); + assertNotEquals(AdaptiveConcurrencyController.NONE, sample.collectionAtNanos()); + assertTrue(sample.collectionAtNanos() <= System.nanoTime()); + } + + @Test + void cpuReadingIgnoresOtherReadersOfProcessCpuLoad() throws Exception { + // getProcessCpuLoad() measures from the previous caller anywhere in the JVM; a scrape just before the + // sampler used to turn a busy interval into zero. + assertTrue(sampler.isCpuSupported()); + var osBean = (com.sun.management.OperatingSystemMXBean) java.lang.management.ManagementFactory + .getOperatingSystemMXBean(); + assertTrue(Double.isNaN(sampler.sampleCpu()), "the first call has no interval"); + long end = System.nanoTime() + 300_000_000L; + long x = 0; + while (System.nanoTime() < end) { + x += System.nanoTime() & 7; + } + osBean.getProcessCpuLoad(); + double cpu = sampler.sampleCpu(); + double oneCore = 1.0 / JvmResourceSampler.availableCpus(); + assertTrue(x > 0); + assertTrue(cpu >= 0.5 * oneCore && cpu <= 1, "cpu " + cpu + " after a busy interval (one core " + oneCore + ")"); + } + + @Test + void cpuLoadIsRelativeToTheCpuQuota() { + assertEquals(1.0, JvmResourceSampler.cpuLoad(500_000_000L, 1_000_000_000L, 0.5), 1e-9); + assertEquals(0.25, JvmResourceSampler.cpuLoad(500_000_000L, 1_000_000_000L, 2), 1e-9); + assertEquals(1.0, JvmResourceSampler.cpuLoad(3_000_000_000L, 1_000_000_000L, 2), 1e-9, "clamped"); + assertEquals(0.5, JvmResourceSampler.parseCpuMax("50000 100000\n"), 1e-9); + assertEquals(-1, JvmResourceSampler.parseCpuMax("max 100000"), 1e-9); + assertEquals(2.0, JvmResourceSampler.parseCpuQuota("200000", "100000"), 1e-9); + assertEquals(-1, JvmResourceSampler.parseCpuQuota("-1", "100000"), 1e-9); + } + + @Test + void poolUtilizationIsRelativeToThePoolCapacity() { + long mb = 1024 * 1024; + // A Serial old generation of 85 MB in a 123 MB heap, full after collection. + assertEquals(0.99, JvmResourceSampler.poolUtilization(new MemoryUsage(0, 84 * mb + 15 * mb / 100, 85 * mb, + 85 * mb), 123 * mb), 0.01); + // G1 and ZGC report the heap as the old generation's maximum, or none at all. + assertEquals(0.5, JvmResourceSampler.poolUtilization(new MemoryUsage(0, 64 * mb, 64 * mb, 128 * mb), + 128 * mb), 1e-9); + assertEquals(0.25, JvmResourceSampler.poolUtilization(new MemoryUsage(0, 32 * mb, 32 * mb, -1), 128 * mb), + 1e-9); + } + + @Test + void cpuReadingIsARatioOrUnavailable() { + if (!sampler.isCpuSupported()) { + return; + } + double cpu = sampler.sampleCpu(); + assertTrue(Double.isNaN(cpu) || (cpu >= 0 && cpu <= 1), "cpu " + cpu); + } + + @Test + void stalledNotificationsFallBackToCollectionUsage() throws Exception { + // A notification pipeline that stops while collections continue must not freeze the heap estimate at a + // stale value that still looks fresh. + assertTrue(sampler.isHeapSupported()); + System.gc(); // at least one collection has happened + Thread.sleep(50); + sampler.close(); // notifications stop; collections continue + Thread.sleep(50); // let any in-flight notification land first + sampler.injectNotificationForTest(0.123); + assertEquals(0.123, sampler.sampleHeap().utilization(), 1e-9, "a fresh notification is trusted"); + + ResourceSampler.HeapSample sample = null; + for (int i = 0; i < 50; i++) { + System.gc(); + Thread.sleep(20); + sample = sampler.sampleHeap(); + if (sample != null && Math.abs(sample.utilization() - 0.123) > 1e-9) { + break; + } + } + assertNotNull(sample); + assertNotEquals(0.123, sample.utilization(), 1e-9); + assertTrue(sample.utilization() > 0 && sample.utilization() < 1, "utilization " + sample.utilization()); + } +} diff --git a/conductor-client/src/test/java/com/netflix/conductor/client/automator/TaskRunnerAdaptiveTest.java b/conductor-client/src/test/java/com/netflix/conductor/client/automator/TaskRunnerAdaptiveTest.java new file mode 100644 index 000000000..77ad9f1a7 --- /dev/null +++ b/conductor-client/src/test/java/com/netflix/conductor/client/automator/TaskRunnerAdaptiveTest.java @@ -0,0 +1,246 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.client.automator; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import com.netflix.conductor.client.events.dispatcher.EventDispatcher; +import com.netflix.conductor.client.events.taskrunner.TaskRunnerEvent; +import com.netflix.conductor.client.http.TaskClient; +import com.netflix.conductor.client.worker.Worker; +import com.netflix.conductor.common.metadata.tasks.Task; +import com.netflix.conductor.common.metadata.tasks.TaskResult; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class TaskRunnerAdaptiveTest { + + private static final String TASK_TYPE = "adaptive_task"; + + private final List runners = new ArrayList<>(); + private final List pollThreads = new ArrayList<>(); + + @AfterEach + void tearDown() { + runners.forEach(runner -> runner.shutdown(1)); + pollThreads.forEach(Thread::interrupt); + } + + private static AdaptiveConcurrencyController controller() { + var config = new AdaptiveConcurrencyConfig.Builder() + .withDemandOnlyEnabled(true) + .withInitialProcessConcurrency(4) + .build(); + var noMetrics = new AdaptiveConcurrencyControllerTest.FakeResourceSampler(); + noMetrics.cpuSupported = false; + noMetrics.heapSupported = false; + return new AdaptiveConcurrencyController(config, () -> noMetrics, System::nanoTime, false); + } + + private static Worker worker(AtomicInteger executions, Set threadNames, CountDownLatch done) { + Worker worker = mock(Worker.class); + when(worker.getTaskDefName()).thenReturn(TASK_TYPE); + when(worker.getPollingInterval()).thenReturn(10); + when(worker.getIdentity()).thenReturn("adaptive-worker-1"); + when(worker.execute(any(Task.class))).thenAnswer(invocation -> { + Task task = invocation.getArgument(0); + executions.incrementAndGet(); + threadNames.add(Thread.currentThread().getName()); + TaskResult result = new TaskResult(task); + result.setStatus(TaskResult.Status.COMPLETED); + done.countDown(); + return result; + }); + return worker; + } + + private static Task task() { + Task task = new Task(); + task.setTaskId(UUID.randomUUID().toString()); + task.setTaskDefName(TASK_TYPE); + task.setTaskType(TASK_TYPE); + task.setStatus(Task.Status.IN_PROGRESS); + return task; + } + + private static TaskClient taskClient(ConcurrentLinkedQueue backlog) { + TaskClient taskClient = mock(TaskClient.class); + when(taskClient.batchPollTasksInDomain(anyString(), any(), anyString(), anyInt(), anyInt())) + .thenAnswer(invocation -> { + int count = invocation.getArgument(3); + List tasks = new ArrayList<>(); + Task next; + while (tasks.size() < count && (next = backlog.poll()) != null) { + tasks.add(next); + } + return tasks; + }); + when(taskClient.getQueueSizesForTasks(anyList())) + .thenAnswer(invocation -> Map.of(TASK_TYPE, backlog.size())); + return taskClient; + } + + private TaskRunner runner(Worker worker, TaskClient taskClient, AdaptiveConcurrencyController controller, + boolean useVirtualThreads, boolean updateV2) { + @SuppressWarnings("unchecked") + EventDispatcher eventDispatcher = + (EventDispatcher) mock(EventDispatcher.class); + TaskRunner runner = new TaskRunner(worker, taskClient, 3, Map.of(), "adaptive-worker-%d", 1, 100, List.of(), + eventDispatcher, useVirtualThreads, updateV2, controller, 8); + runners.add(runner); + return runner; + } + + private void start(TaskRunner runner) { + Thread thread = new Thread(runner::pollAndExecute); + thread.setDaemon(true); + thread.start(); + pollThreads.add(thread); + } + + private static void awaitHeldSlots(AdaptiveConcurrencyController controller, int expected) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (controller.getHeldSlots() != expected && System.nanoTime() < deadline) { + Thread.sleep(10); + } + assertEquals(expected, controller.getHeldSlots()); + } + + @Test + void polledTasksExecuteAndReleaseTheirSlots() throws Exception { + var backlog = new ConcurrentLinkedQueue(); + for (int i = 0; i < 20; i++) { + backlog.add(task()); + } + var executions = new AtomicInteger(); + var done = new CountDownLatch(20); + TaskClient taskClient = taskClient(backlog); + AdaptiveConcurrencyController controller = controller(); + TaskRunner runner = runner(worker(executions, ConcurrentHashMap.newKeySet(), done), taskClient, controller, + false, false); + controller.activate(); + controller.sampleDepths(); + + start(runner); + + assertTrue(done.await(10, TimeUnit.SECONDS), "all tasks executed"); + awaitHeldSlots(controller, 0); + assertEquals(20, executions.get()); + verify(taskClient, never()).updateTaskV2(any(), anyString()); + } + + @Test + void updateAndFetchReplacementReusesTheSlot() throws Exception { + var executions = new AtomicInteger(); + var done = new CountDownLatch(5); + var backlog = new ConcurrentLinkedQueue(); + backlog.add(task()); + var replacements = new ConcurrentLinkedQueue(); + for (int i = 0; i < 4; i++) { + replacements.add(task()); + } + TaskClient taskClient = taskClient(backlog); + when(taskClient.updateTaskV2(any(TaskResult.class), anyString())).thenAnswer(invocation -> replacements.poll()); + AdaptiveConcurrencyController controller = controller(); + TaskRunner runner = runner(worker(executions, ConcurrentHashMap.newKeySet(), done), taskClient, controller, + false, true); + controller.activate(); + + start(runner); + + assertTrue(done.await(10, TimeUnit.SECONDS), "polled task and four replacements executed"); + awaitHeldSlots(controller, 0); + assertEquals(5, executions.get()); + } + + @Test + void virtualThreadsRunEachTaskOnANumberedVirtualThread() throws Exception { + var backlog = new ConcurrentLinkedQueue(); + for (int i = 0; i < 6; i++) { + backlog.add(task()); + } + var threadNames = ConcurrentHashMap.newKeySet(); + var done = new CountDownLatch(6); + AdaptiveConcurrencyController controller = controller(); + TaskRunner runner = runner(worker(new AtomicInteger(), threadNames, done), taskClient(backlog), controller, + true, false); + controller.activate(); + controller.sampleDepths(); + + start(runner); + + assertTrue(done.await(10, TimeUnit.SECONDS)); + awaitHeldSlots(controller, 0); + assertTrue(threadNames.stream().allMatch(name -> name.matches("adaptive-worker-\\d+")), threadNames.toString()); + } + + @Test + void pollersOfOneQueueShareStateAndPool() throws Exception { + var backlog = new ConcurrentLinkedQueue(); + var executions = new AtomicInteger(); + Worker worker = worker(executions, ConcurrentHashMap.newKeySet(), new CountDownLatch(0)); + TaskClient taskClient = taskClient(backlog); + AdaptiveConcurrencyController controller = controller(); + TaskRunner first = runner(worker, taskClient, controller, false, false); + TaskRunner second = runner(worker, taskClient, controller, false, false); + controller.activate(); + + WorkerQueue queue = extractQueue(first); + assertSame(queue, extractQueue(second)); + assertEquals(2, queue.registrations); + + first.shutdown(1); + runners.remove(first); + assertTrue(!queue.executionPool().isShutdown(), "the remaining poller keeps the shared pool"); + second.shutdown(1); + runners.remove(second); + assertTrue(queue.executionPool().isShutdown()); + } + + @Test + void pollTimeoutAboveServerLimitIsRejected() { + var builder = new TaskRunnerConfigurer.Builder(mock(TaskClient.class), List.of()) + .withTaskPollTimeout(6000) + .withAdaptiveConcurrency(controller()); + assertThrows(IllegalArgumentException.class, builder::build); + } + + private static WorkerQueue extractQueue(TaskRunner runner) throws Exception { + var field = TaskRunner.class.getDeclaredField("workerQueue"); + field.setAccessible(true); + return (WorkerQueue) field.get(runner); + } +} diff --git a/conductor-client/src/test/java/com/netflix/conductor/client/http/TaskClientQueueSizesTest.java b/conductor-client/src/test/java/com/netflix/conductor/client/http/TaskClientQueueSizesTest.java new file mode 100644 index 000000000..694bb0d50 --- /dev/null +++ b/conductor-client/src/test/java/com/netflix/conductor/client/http/TaskClientQueueSizesTest.java @@ -0,0 +1,68 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.client.http; + +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class TaskClientQueueSizesTest { + + private MockWebServer server; + private TaskClient taskClient; + + @BeforeEach + void setUp() throws Exception { + server = new MockWebServer(); + server.start(); + taskClient = new TaskClient(ConductorClient.builder().basePath(server.url("/api").toString()).build()); + } + + @AfterEach + void tearDown() throws Exception { + server.shutdown(); + } + + @Test + void sizesEveryQueueInOneRequest() throws Exception { + server.enqueue(new MockResponse() + .setHeader("Content-Type", "application/json") + .setBody("{\"render_pdf\": 48, \"tenant1:send_email\": 6}")); + + Map sizes = taskClient.getQueueSizesForTasks(List.of("render_pdf", "tenant1:send_email")); + + assertEquals(Map.of("render_pdf", 48, "tenant1:send_email", 6), sizes); + RecordedRequest request = server.takeRequest(); + assertEquals("/api/tasks/queue/sizes", request.getRequestUrl().encodedPath()); + assertEquals(List.of("render_pdf", "tenant1:send_email"), request.getRequestUrl().queryParameterValues("taskType")); + assertEquals(1, server.getRequestCount()); + } + + @Test + void singleQueueSizeUsesTheSameRequest() { + server.enqueue(new MockResponse() + .setHeader("Content-Type", "application/json") + .setBody("{\"render_pdf\": 7}")); + + assertEquals(7, taskClient.getQueueSizeForTask("render_pdf")); + } +} diff --git a/design/adaptive-worker-concurrency.md b/design/adaptive-worker-concurrency.md new file mode 100644 index 000000000..8b6dbc0de --- /dev/null +++ b/design/adaptive-worker-concurrency.md @@ -0,0 +1,904 @@ +# Adaptive worker polling and concurrency + +Status: implemented; see `AdaptiveConcurrencyController` and the automator package in `conductor-client`. This document is the design of record for that implementation. Numeric defaults are starting points for validation through load testing. + +## Purpose and scope + +Introduce an opt-in mode that adjusts worker execution concurrency and polling batch sizes according to queue depth, actual task delivery, JVM process CPU utilization, and post-collection JVM heap utilization. One controller coordinates participating workers in a process and divides its capacity among them in proportion to their backlogs. + +Adaptive mode uses separate concurrency limits. Existing thread-count configuration continues to govern fixed mode. Poller count stays configured and fixed; the controller adjusts execution capacity and the number of tasks requested per poll. + +The feature uses existing Conductor APIs and task execution semantics. Existing SDK cleanup, unrelated reliability changes, native process-memory measurement, and container instrumentation are outside this proposal. + +JVM CPU and heap thresholds guide acceptance of new work. They are not hard resource guarantees: tasks already running can increase resource use. Heap utilization does not measure total process or container memory. + +## Relationship to the existing runner + +The current runner already has most of the shape this design needs. Adaptive mode changes the source of permits and the sizing of requests; it keeps the loop, the execution path, the update path, the event model, and the configuration mechanisms. Implementers should modify these existing pieces, not build parallel ones. + +| Existing mechanism | Where | Adaptive mode | +|---|---|---| +| One polling thread per worker registration | `TaskRunnerConfigurer.init()` and `startWorker()` | Retained. Registrations for the same logical queue are grouped and share one controller-managed worker state. | +| Acquire all free permits, poll with `count` equal to permits, release unused permits | `TaskRunner.pollTasksForWorker()` using `Semaphore permits` | Same shape. Permits come from controller targets instead of a fixed semaphore, a queue with no demand polls only as a heartbeat, and a queue whose last response was empty requests one task. | +| Submit each task to the executor, then schedule lease extension when `responseTimeoutSeconds > 0` and `leaseExtendEnabled()` | `TaskRunner.pollAndExecute()` | Factored into one `dispatch(task)` method used for polled tasks and for update-and-fetch replacements, so both paths get lease extension. | +| `processTask()` runs the worker, `executeTask()` builds the result, `updateTaskResult()` uploads large payloads and retries the update, `finally` cancels lease extension and releases the permit | `TaskRunner` | Retained. `updateTaskResult()` returns the slot disposition, the `finally` releases the controller slot unless it was transferred, and execution and reporting durations are recorded. | +| Replacement task from update-v2 is placed on `tasksTobeExecuted` and drained by the next poll | `TaskRunner.pollTask()` and `updateTaskResult()` | The in-memory queue is not used in adaptive mode. A replacement task is dispatched directly with the finishing task's slot. | +| Fixed pool `Executors.newFixedThreadPool(threadCount, threadFactory)`, platform or virtual thread factory | `TaskRunner` constructor | Platform threads: resizable `ThreadPoolExecutor` with the same factory and uncaught-exception handler. Virtual threads: thread-per-task executor bounded by slots; no pool. | +| `useVirtualThreads` opt-in on the builder only; the virtual factory uses `Thread.ofVirtual().name(prefix)`, so every thread carries the literal prefix pattern | `TaskRunnerConfigurer.Builder`, `TaskRunner` constructor | Flag retained and also exposed through Spring `conductor.client.*` and the annotated-worker configuration. Virtual threads are named with the numbered `name(prefix, start)` form. | +| `worker.paused()`, `PollFilter.filter(taskType, domain)`, `worker.getPollingInterval()`, per-task poll timeout | `TaskRunner.pollTasksForWorker()` and constructor | Retained as inputs. The polling interval becomes the base of the empty-response backoff. | +| Domain resolved from `conductor.worker..domain`, then `conductor.worker.all.domain`, then the builder map | `TaskRunner` constructor | Retained. The logical queue identity uses this resolved domain. | +| Per-worker properties via `PropertyFactory` (`conductor.worker..` with `conductor.worker.` fallback) | `PropertyFactory` | Per-worker adaptive settings use the same keys. Process-level settings use the builder and, in Spring, `conductor.client.*`. | +| Events published through `EventDispatcher` to `TaskRunnerEventsListener` and `MetricsCollector` | `events` and `metrics` packages, `PrometheusMetricsCollector` | New event types are added as default methods on the listener so existing collectors remain source-compatible. | +| Fatal authentication exit, `shutdown(timeout)` draining the executor and lease executor | `TaskRunner` | Unchanged. | +| Update-v2 enabled by the JVM-wide `taskUpdateV2` system property or environment variable | `TaskRunner` constructor | Retained as a fallback; a builder option is added so adaptive mode can authorize replacements per runner. | +| `withTaskPollCount(...)` accepted but never passed to the runner | `TaskRunnerConfigurer.Builder` | Superseded by the adaptive batch maximum and deprecated. | +| `getQueueSizeForTask(taskType)` calls `/tasks/queue/sizes` with a single name and reads one entry of the returned map | `TaskClient` | Generalized to take a list of queue names, `taskType` or `domain:taskType`, and return the whole map, so one request sizes every logical queue of the process. The domain overload, which drops its domain, is not used by this feature; fixing it is a separate bug. | + +Behavior change to document for users: today a poller count of N registers the same worker N times, and each registration gets its own runner, semaphore, and fixed pool, so fixed-mode capacity is N multiplied by the thread count. In adaptive mode the N pollers share one worker target and one execution pool. Poller count affects only how many concurrent poll requests a logical queue can have outstanding. + +## Server and SDK responsibilities + +| Conductor server | Java SDK | +|---|---| +| Owns queues and determines which tasks are eligible for delivery | Determines how much work this process can accept | +| Applies task-definition concurrency and rate limits | Adjusts local execution concurrency within configured ceilings | +| Tracks execution state, retries, and timeouts | Executes accepted tasks and reports their results | +| Distributes queued tasks among polling consumers | Coordinates local workers sharing CPU and heap | +| Reports queue depth per task type and domain | Samples depth and divides its process budget among workers in proportion to it, each capped at what the worker can use | +| Routes domain-fallback tasks to domains with a recent poll | Keeps a heartbeat poll on idle queues so routing and poll-data views keep working | + +The server marks delivered tasks `IN_PROGRESS`. A poll therefore accepts work; it is not a read-only preview of the queue. The SDK reserves execution capacity before requesting tasks and keeps accepted work accounted for until local execution and result-reporting handling finish. + +Batch poll semantics matter for interpreting responses. The inspected Postgres queue implementation returns as soon as at least one task is available, up to `count`, and waits for the timeout only while the queue is empty; its comments state the Redis implementation behaves the same. A response smaller than `count` therefore means fewer than `count` tasks were available at that instant. It does not mean the queue is draining, and it does not mean the arrival rate is below what one slot can process. An empty response means the queue stayed empty for the whole timeout, or that server-side limits or other consumers took the work. + +Queue depth is the server's size for the logical queue. It is a shared snapshot, not work reserved for this process: other consumers, task-definition concurrency and rate limits, and delayed tasks all mean a positive depth does not guarantee delivery. Backends also differ in what they count. In the inspected source the Postgres queue's size query counts every message row for the queue, including messages delivered to a consumer and not yet acknowledged, while its all-queues query and the Redis queue count only waiting messages. Depth is therefore an upper bound on waiting work. The controller uses it to divide capacity and to bound growth; it never admits work on the strength of depth alone, because every slot is still backed by a delivered task. An overstated depth inflates a queue's share, and the empty responses that follow shrink that queue's requests to one task until a delivery, so the cost is bounded. Backends should report waiting messages; aligning the Postgres size query with its all-queues query is a server-side fix to file alongside this feature. + +The server also uses polls as a liveness signal. `WorkflowExecutorOps.getActiveDomain` routes a task with an ordered domain list to the first domain some worker polled within `conductor.app.activeWorkerLastPollTimeout`, ten seconds by default, and `ExecutionService.poll` refreshes that poll data on every poll, empty or not. A worker that stops polling a domain therefore stops being offered that domain's tasks. Adaptive mode keeps a heartbeat poll on every queue so this routing, and the poll-data views built on the same record, keep working. + +### Existing APIs + +Paths are relative to the configured API base URL, usually ending in `/api`. + +| Operation | API | Use | +|---|---|---| +| Batch poll | `GET /tasks/poll/batch/T?workerid=W&domain=D&count=N&timeout=M` | Accept up to the reserved capacity | +| Result update | `POST /tasks` | Report task results | +| Update and fetch | `POST /tasks/update-v2` | Existing optional replacement-task path | +| Queue depth | `GET /tasks/queue/sizes?taskType=N1&taskType=N2` | One request per second naming every logical queue; a domain queue is named `domain:taskType` | + +Omit `domain` for the default queue. The server rejects a batch poll whose `timeout` exceeds 5000 milliseconds with an error rather than clamping it. Validate configured poll timeouts against that bound at configuration time. The batch sizes endpoint takes repeated `taskType` parameters and returns a map keyed by the names passed. In the inspected source each name goes straight to the queue store's size lookup, so it sizes whatever queue it names, and the server names a domain queue `domain:taskType` (`QueueUtils.getQueueName`, separator `:`). Passing that form sizes the domain queue in the same request. The server marks the endpoint deprecated in favor of the single-queue `GET /tasks/queue/size`; this design depends on it as the only call that sizes many queues at once and asks that it be kept. Isolation groups and execution namespaces are not part of the polling path and are not passed. + +No new server protocol or task-definition modification is required. + +```text +Queue depth per worker ----+ +Actual poll responses -----+--> Adaptive controller <-- JVM CPU and post-GC heap +Task completion -----------+ | + v + Process target, divided among workers by demand + | + v + Reserve capacity before polling + | + v + Poll with count = min(free worker slots, free process slots, batch maximum) + | + v + Execute tasks and report results + | + v + Release capacity +``` + +## Configuration and terminology + +An execution slot represents capacity promised to one pending task fetch or owned by one accepted task. It is not a guarantee that an execution thread is currently busy. In the existing runner a slot is a semaphore permit; adaptive mode issues slots from the controller instead. + +`heldSlots` includes pending fetch reservations, accepted tasks awaiting dispatch, executing tasks, and tasks undergoing result-reporting handling. A target can temporarily be below `heldSlots` after a reduction; no new slots are admitted until enough work drains. + +A logical worker queue is identified by server/client scope, task type, and resolved domain. The resolved domain is the one the runner computes today from worker properties and the builder map. Multiple configured pollers for that logical queue share one worker concurrency target and execution pool. Distinct server/client scopes are not merged merely because their task names match. + +Applications supply one shared adaptive controller to participating runners in the process. Framework integration shares one controller across participating worker registrations. The controller observes process resources but controls only the workers registered with it; unrelated application work is reflected in resource pressure. + +### Proposed defaults + +| Setting | Default | Purpose | +|---|---:|---| +| Adaptive mode | Disabled | Explicit opt-in | +| Initial process concurrency | min(logical queues, available processors), clamped to [minimum, process maximum] | Aggregate startup target. Set it to the expected working level to skip the ramp from a small start; the auto value is 1 in a pod with one CPU or less | +| Minimum process concurrency | 1 | Lowest target demand decay settles at, so an idle process keeps a ready budget and the next burst does not ramp from one slot. Idle queues still hold no threads, because their shares are zero. CPU and heap pressure may reduce the target below it, and decay never raises a target that pressure lowered | +| Maximum process concurrency | 256 | Aggregate admission ceiling | +| Default maximum worker concurrency | 64 | Per-queue ceiling, explicitly overridable | +| Maximum polling batch | 16 | Ceiling for a single request | +| Resource sampling interval | 1 second | Observe JVM CPU and post-GC heap | +| CPU smoothing | EWMA, alpha 0.2 | Filter one-second sample noise out of the growth decision | +| Control interval | 5 seconds, ±20% jitter | Evaluate target adjustments | +| Growth thresholds | Smoothed CPU below 75% and latest CPU sample below 90%; post-GC heap below 75% | Headroom required for growth | +| CPU pressure threshold | CPU samples at or above 90% for 3 seconds | Reduce targets | +| Heap pause threshold | Post-GC heap at or above 90%, from a collection newer than the last resume | Stop new acquisition immediately; also reduction evidence | +| Heap resume threshold | Post-GC heap below 80% | Resume at current targets; a reading below 80% can only come from a collection after the pause | +| Drained resume | Paused, `heldSlots == 0`, and 30 seconds since the high collection that caused or renewed the pause | Resume at concurrency one so ordinary allocation can trigger the collection that refreshes the estimate | +| Pre-collection heap reading | Current used heap until the first collection | Upper bound on live data; gates growth before any GC, never enters a pause | +| Reduction factor | 0.75 | Withdraw 25% of the process target under pressure; shares shrink with it | +| Reduction interval | 15 seconds | Minimum spacing between reductions, and growth cooldown after one | +| Growth step | 20% of the process target, rounded up, clamped to 1–16 slots, never past total demand | Bounded expansion per control tick | +| Demand decay | After 30 seconds with total demand below the process target, shrink the target to the larger of demand and half of itself, floor 1 | Return surplus capacity gradually so the next burst ramps under the growth gate | +| Depth sampling interval | 1 second | One `GET /tasks/queue/sizes` request per interval naming every logical queue, on a dedicated thread | +| Depth freshness | 5 seconds | A depth older than this is unknown and counts as one; must cover a full sweep of the process's queues | +| Heartbeat interval | 5 seconds | A queue with no share and no standby poll polls one task this often on an open slot, without a server wait, keeping the server's poll data inside its 10-second active-poller window and verifying a zero depth | +| Standby poll timeout | 1 second | Server wait of the single-task long poll a queue keeps open on a slot no share claims, so a task is picked up as it arrives; at most 5 seconds, zero disables standby polls. An idle queue sends about one request per timeout | +| Metric freshness | 5 seconds | A metric older than this freezes growth; a gap longer than this resets smoothing and pressure timing | +| Demand-only operation | Disabled | Startup fails if neither resource metric is supported unless explicitly enabled | +| Execution threads | Platform; `useVirtualThreads` selects virtual | Existing opt-in; chooses the executor strategy below | +| Execution-thread idle timeout | 60 seconds | Retire idle platform execution threads; not applicable to virtual threads | + +All concurrency and batch ceilings must be positive. Initial process concurrency must not exceed the process maximum. Per-worker maxima remain subject to the process ceiling. Validate finite thresholds in `(0, 1]`, with growth < heap resume < heap pause and growth < CPU pressure. Durations must be positive; each freshness must exceed its sampling interval; the heartbeat interval should stay below the server's active-poller window. The standby poll timeout is between zero and 5000 milliseconds. Configurable smoothing uses `0 < alpha <= 1`. Configured long-poll timeouts must not exceed 5000 milliseconds and must fit inside the HTTP read timeout with room for overhead. + +Expose adaptive options through the task-runner builder and worker configuration path, including framework bindings. Required concepts are enablement, the shared controller, process maximum, default/per-worker maximum, batch maximum, resource and depth sampling intervals and freshness, heartbeat interval, control interval, resource thresholds, demand-only operation, update-and-fetch enablement, and virtual-thread selection. Virtual threads stay opt-in with the existing `useVirtualThreads` flag; that flag is additionally exposed as a Spring property and through the annotated-worker configuration, since today only the builder sets it. Per-worker values use the existing `conductor.worker..` property path with the `conductor.worker.` fallback. Do not reinterpret existing `threadCount` values as adaptive ceilings. + +Existing worker polling intervals, long-poll timeouts, pause/filter behavior, and configured poller counts remain inputs to this algorithm. Adaptive configuration must use distinct names for execution concurrency, requested batch count, and poller count. The existing `withTaskPollCount` builder option, which is stored but never applied today, is deprecated in favor of the adaptive batch maximum. + +## Controller state and invariants + +```text +ProcessState: + maximumConcurrency + targetConcurrency + heldSlots // reservations plus accepted tasks, all queues + admissionPaused // heap-pause latch + lastResumeAt // -infinity at startup + lastReductionAt // -infinity at startup + surplusSince // when total demand last fell below the target; NONE while demand covers it + heapGrowthCeiling // growth limit left by heap pressure; unlimited until a heap reduction or pause sets it + heapCeilingSince // when the ceiling was set; it holds 120 seconds, then rises one slot per control tick + + cpu: supported, raw, smoothed, observedAt, pressureSince // observedAt is NONE after a failed sample + heap: supported, postGcUtilization, collectionAt, observedAt, + lastHighCollectionAt, // collectionAt of the latest collection at or above the pause threshold + baseline, // lowest post-collection utilization seen: the heap the process needs without tasks, or close to it + recentReadings // post-collection readings of the last 30 seconds; their peak gates growth + +WorkerQueueState: + queueIdentity // scope, task type, resolved domain + maximumConcurrency + targetConcurrency // this queue's share of the process target; derived by allocate() + heldSlots // reservations plus accepted tasks + depth // last sampled queue size + depthObservedAt // NONE until the first successful sample and after a failed one + outstandingPolls + nextHeartbeatAt // now at startup; pushed forward by every granted poll; a refused heartbeat stays due + nextStandbyAt // now at startup; a delivery resets it, an expired standby wait pushes it one standby timeout + standbyPolls // outstanding standby polls; their slots are not part of the backlog + reservedPollSlots // slots held by outstanding polls; the rest of heldSlots back accepted tasks + lastTaskSlotReleaseAt // last task-slot release, or when tasks started arriving; drives the slot-age observation + consecutiveEmptyPolls // poll errors count as empty responses + depthAtEmptyPoll // sampled depth when the latest empty response settled; zero if the depth was stale + pollableAtSample // worker pause and poll filter result at the latest depth sweep + nextPollAllowedAt + configuredPollInterval + configuredPollTimeout + executionPool +``` + +The following invariants apply: + +1. Rule one: a queue is polled only with an open slot for every task the poll may return. A reservation never exceeds the free worker target or the free process target at the time it is made, except that a heartbeat may take one open process slot beyond the queue's share, never beyond the worker maximum. Held slots exceed a target only after that target was lowered by a reduction, a decay, or a reallocation, and then only until work drains. Nothing polls while admission is paused or every slot is held. +2. Each slot has exactly one owner: an outstanding request or an accepted task. +3. Reservations are transferred or released exactly once. Network calls do not run under the controller lock. +4. Reducing a target does not revoke work already accepted or a reservation backing an outstanding fetch. +5. Tasks already owned by the process continue execution and reporting while acquisition is paused. +6. All pollers for the same logical queue share its counters and target. +7. Worker targets are derived. `allocate()` recomputes every one of them from the process target and the queues' demands; nothing adjusts a worker target individually. + +`applyTargets()` is the single exit for every target or admission change: it runs `allocate()`, publishes the targets, resizes platform execution pools to their required size, emits the adjustment observation, and wakes worker waiters of each queue up to its new deficit and process waiters in deficit order until the free process slots are covered, or every waiter when the pause lifts. Unchanged targets cause no wakeups. `NONE` timestamps compare as negative infinity, so a comparison such as `lastHighCollectionAt > lastReductionAt` is false while both are unset. + +Use monotonic time for controller intervals, freshness, backoff, and duration comparisons. Resource sampling and control scheduling use a controller executor separate from task execution and task-result updates. The depth sampler runs on its own thread because it makes network calls; the controller executor's tasks never do, so a slow server cannot delay a CPU or heap sample. + +## Startup + +```text +function start(workerQueues): + cpu.supported = platform OperatingSystemMXBean reports getProcessCpuTime() + heap.supported = heap max is positive AND (GC notifications are available + OR at least one heap pool reports collection usage) + log each unsupported metric once; expose in observability + if NOT cpu.supported AND NOT heap.supported AND NOT demandOnlyEnabled: + fail startup with a configuration error + + process.targetConcurrency = configured initial value, or + clamp(configured initial OR min(count(workerQueues), availableProcessors), minimumConcurrency, maximumConcurrency) + process.heldSlots = 0 + process.admissionPaused = false + process.lastResumeAt = -infinity + process.lastReductionAt = -infinity + process.surplusSince = NONE + process.heapGrowthCeiling = unlimited; process.heapCeilingSince = NONE + cpu.observedAt = NONE; cpu.pressureSince = NONE + heap.observedAt = NONE; heap.lastHighCollectionAt = NONE + heap.baseline = undefined; heap.recentReadings = empty + + for each queue: + queue.heldSlots = 0 + queue.outstandingPolls = 0 + queue.depth = 0; queue.depthObservedAt = NONE // unknown until the first sample + queue.consecutiveEmptyPolls = 0; queue.depthAtEmptyPoll = 0 + queue.standbyPolls = 0; queue.pollableAtSample = true + queue.nextPollAllowedAt = now + queue.nextHeartbeatAt = now; queue.nextStandbyAt = now + initialize adaptive execution pool with size 1 + allocate(now) // unknown depth counts as one, so every queue starts with a one-task share + + start resource sampler + start depth sampler + start control timer + start configured pollers // existing one-thread-per-registration model +``` + +If there are more queues than process slots, grants rotate among queues with equal deficits. Startup does not reserve one process slot permanently for every worker. Queues registered after the controller starts enter the next allocation like any other and do not raise the process target. + +## Monitoring JVM capacity + +### CPU + +Read JVM process CPU as the growth of `OperatingSystemMXBean.getProcessCpuTime()` between samples, divided by the elapsed time and by the CPUs available to the process: the container quota when one is set (cgroup v2 `cpu.max`, or v1 `cpu.cfs_quota_us` over `cpu.cfs_period_us`), which may be fractional, capped by `Runtime.availableProcessors()`. The quota is re-read on every sample so a resized container takes effect. Clamp to `[0, 1]`; the first sample has no interval and is an invalid observation, not evidence that the metric is unsupported. Do not use `getProcessCpuLoad()`: it measures the interval since the previous call by any caller in the JVM, so a metrics library reading it, as Micrometer's `ProcessorMetrics` does on every scrape and Spring Boot Actuator registers by default, leaves this sampler a window of milliseconds. End-to-end testing measured readings of zero on a saturated core with another reader 20 milliseconds earlier, and the controller grew while CPU was saturated. `availableProcessors()` alone is not enough either, because it rounds a fractional quota up: at 0.5 CPU a saturated process would read 0.5 and never reach the pressure threshold. The metric is unsupported only when the platform bean does not implement the `com.sun.management` interface or reports no process CPU time. + +CPU samples are one-second averages and are noisy. Growth therefore uses an exponentially weighted moving average, and the latest sample vetoes growth only at the pressure threshold. Pressure uses the raw samples directly: a run of samples at or above 90% lasting three seconds is pressure, and the start of that run is the evidence time. + +### Heap + +Current used heap from `MemoryMXBean.getHeapMemoryUsage()` includes garbage that has not yet been collected. On a healthy JVM it routinely rises above 90% just before a young collection: under G1 defaults the eden alone can be most of the heap, and under ParallelGC it is a third. Used heap is therefore not a pressure signal. The controlling signal is heap usage measured immediately after a collection, which is the JVM's best available estimate of live data. + +```text +heapMax = MemoryMXBean.getHeapMemoryUsage().getMax() // must be positive +postGcHeapUsed = sum over heap pools of usage after the most recent collection +longLived[pool] = used / min(pool max, heapMax) after the most recent collection OF that pool + for each heap pool that supports usage thresholds // old generation; never eden or survivor +postGcUtilization = max(postGcHeapUsed / heapMax, max(longLived)) // controlling value +``` + +The whole-heap ratio alone misses pressure under collectors with a fixed-size old generation. SerialGC, the JVM's default in containers with less than two CPUs or 1792 MB, and ParallelGC give the old generation about two thirds of the heap; data that outlives a young collection can only live there. End-to-end testing with tasks holding multi-megabyte payloads in a 123 MB heap found the old generation 99% full after collection while the whole-heap ratio read 69%, because the empty eden counted as headroom; growth continued and tasks ran out of memory. The fullest long-lived pool measured against its own capacity catches this. Long-lived pools are identified by `MemoryPoolMXBean.isUsageThresholdSupported()`, which is true for Tenured Gen, PS Old Gen, G1 Old Gen, ZGC Old Generation, ZHeap and Shenandoah and false for every eden, survivor and ZGC young pool. A pool's reading is taken only from collections by a collector whose `getMemoryPoolNames()` include it, because after a young collection the old generation still holds garbage that only a full collection frees; the reading is kept until the next such collection. For G1, ZGC and Shenandoah the long-lived pool's capacity is the whole heap, so the reading equals the whole-heap ratio or less and nothing changes. + +Obtain `postGcHeapUsed` in one of two ways, chosen at startup: + +1. Preferred: register a listener on each `GarbageCollectorMXBean` that implements `NotificationEmitter`. On each collection notification, sum `getMemoryUsageAfterGc()` across heap pools, update the long-lived pools that collector collects, and store the resulting utilization with the collection end time converted from JVM uptime to the controller's monotonic clock. The listener only stores the latest value; the sampler reads it on its next tick, so a collector running many short collections per second produces at most one observation per sampling interval and the listener never does controller work. +2. Fallback: on each sampling tick, sum `MemoryPoolMXBean.getCollectionUsage().getUsed()` across heap pools that support it, and take each long-lived pool's collection usage against its capacity; collection usage is only updated by collections of that pool, which is the rule above. Collection usage carries no timestamp, so a collection is detected by a change in the sum of `GarbageCollectorMXBean.getCollectionCount()` across collectors between ticks, and `collectionAt` is the tick that observed the change. This path is accurate to within one sampling interval; the notification path is exact. + +The choice is also revisited at runtime: if collections keep advancing without a notification, the sampler reads collection usage until notifications resume, so a stalled listener cannot freeze the estimate at a stale value that still looks fresh. + +Either way the sampler reads the current post-collection value every tick, so the heap metric produces a fresh observation each second even when no collection has occurred. A quiet JVM is not an unavailable one. Each heap observation carries two times. `observedAt` is the sampler tick and drives freshness. `collectionAt` is the time of the collection that produced the value and drives pause entry and reduction evidence, so re-reading the same value never counts as new evidence. + +Before the first collection there is no post-collection value; on both paths this is the state in which every collector's collection count is zero. Until the first collection, the heap reading is current used heap with `collectionAt` undefined. Used heap is an upper bound on live data and is small on a fresh JVM, so the metric is valid from the first tick and growth is gated normally. A reading without a `collectionAt` never enters a pause and never counts as pressure evidence, because without a collection it may be mostly garbage. + +Post-collection values are not smoothed. They are already an aggregate over a collection cycle and change only when a collection happens; smoothing them would delay every decision without removing any noise. Reject a nonpositive or undefined heap maximum. If neither mechanism is available, the heap metric is unsupported. Unknown values are never interpreted as zero. + +Known limitation: a live-set increase is visible only after the next collection. Between collections the post-collection estimate can understate live data by at most the allocation since that collection, which is bounded by free heap. The JVM collects before exhausting the heap, so the lag is one collection cycle. + +### Sampling and freshness + +Read CPU and heap independently: an exception or unavailable value from one must not prevent observing or acting on the other. Use fixed-delay sampling, with the next attempt scheduled one interval after the previous attempt finishes. Never replay missed timer ticks after a JVM pause. Timestamp observations when obtained. + +A metric is fresh when its latest observation is at most five seconds old; a failed sample clears the observation time, so a metric whose last attempt failed is not fresh. Check age at the time of each decision, not only when the sampler publishes, so a sampler that stops is noticed. Preserve the heap-pause latch across missing measurements. All metric state is read and written under the controller lock; only the MXBean calls themselves run outside it. + +```text +function sampleResources(now): + // MXBean calls run outside the controller lock; each is attempted independently + cpuReading = cpu.supported ? processCpuTimeDelta / (elapsed * availableCpus) : FAILED + heapReading = heap.supported ? (latest post-collection usage and its collection time, + or (current used heap, UNDEFINED) before the first collection) : FAILED + + under controller lock: + if cpu.supported: + if cpuReading is finite AND 0 <= cpuReading <= 1: + if now - cpu.observedAt > freshness: // also true after a failed sample + cpu.smoothed = cpuReading + cpu.pressureSince = NONE + else: + cpu.smoothed = alpha * cpuReading + (1 - alpha) * cpu.smoothed + cpu.raw = cpuReading + cpu.observedAt = now + if cpuReading >= cpuPressureThreshold: + if cpu.pressureSince == NONE: cpu.pressureSince = now + else: + cpu.pressureSince = NONE + else: + cpu.observedAt = NONE + cpu.pressureSince = NONE + + if heap.supported: + if heapReading != FAILED: + (used, collectionAt) = heapReading + heap.postGcUtilization = double(used) / double(heapMax) + heap.collectionAt = collectionAt + heap.observedAt = now + if collectionAt is defined AND heap.postGcUtilization >= heapPauseThreshold: + heap.lastHighCollectionAt = collectionAt + if collectionAt is defined: + heap.baseline = min(heap.baseline, heap.postGcUtilization) // the first collection sets it + append (collectionAt, now, heap.postGcUtilization) to heap.recentReadings + if collectionAt is newer than the last entry's + else: + heap.observedAt = NONE + + reconcileHeapPause(now) +``` + +### Heap pause and resume + +A fresh post-collection observation at or above 90%, produced by a collection newer than the last resume, closes the admission gate immediately. Outstanding fetch reservations remain valid. Ordinary result reporting and lease handling continue. The high collection is also reduction evidence; the control tick applies the reduction when the reduction interval permits, whether or not the pause has ended by then, so a heap that flaps across the thresholds shrinks targets at most once per interval, the same bound CPU pressure has. + +```text +function reconcileHeapPause(now): + heapFresh = heap.supported AND now - heap.observedAt <= freshness + if NOT process.admissionPaused: + if heapFresh AND heap.collectionAt is defined + AND heap.postGcUtilization >= heapPauseThreshold + AND heap.collectionAt > process.lastResumeAt: + process.admissionPaused = true + limitHeapGrowth(floor(0.75 * max(process.targetConcurrency, process.heldSlots)), now) + applyTargets() // publishes the pause + return + + if heapFresh AND heap.postGcUtilization < heapResumeThreshold: + resumedByDrain = false // a collection after the pause shows headroom + else if process.heldSlots == 0 AND now - heap.lastHighCollectionAt >= 30 seconds: + resumedByDrain = true + process.targetConcurrency = 1 // allocate() derives the shares + else: + return + + process.admissionPaused = false + process.lastResumeAt = resumedByDrain ? now : heap.collectionAt // pause re-entry needs a collection newer than this + applyTargets() // wakes capacity waiters +``` + +The estimate path is the normal one. The pause was entered on a reading at or above 90%, so a reading below 80% can only come from a later collection; admission resumes at the current targets, and the pause's reduction lands at the next permitted control tick if it has not already. A brief spike costs one 25% reduction and one cooldown, not a restart from concurrency one. + +The drain path handles a pause whose estimate is stuck between the resume and pause thresholds because nothing allocates enough to trigger a collection. With no held slots the process is idle, so resuming at concurrency one lets ordinary allocation trigger the collection that refreshes the estimate. Pause entry requires a collection newer than the last resume. On the estimate path the resume time is the time of the collection that justified it, so any later high collection re-enters, even one that completed just before the resume was observed. On the drain path there is no justifying collection, so the resume time is the current time; without that rule the stale reading would re-enter the pause on the next tick. After a drained resume the process runs at concurrency one until a new collection reports. Growth needs no separate rule for this case: the stale reading is at or above the pause threshold, which is above the growth threshold, so growth is blocked until a newer collection shows headroom. If that collection is at or above the pause threshold the pause re-enters and the drain timer restarts from that collection; if the process is still idle, it resumes again 30 seconds later, so a process whose live set genuinely sits above the threshold alternates between paused and concurrency one rather than staying paused forever. The drain timer is anchored on `lastHighCollectionAt`, the same field that carries reduction evidence, so a further high collection during a pause renews the pause and a collection between the thresholds does not. + +The controller does not force garbage collection. Sustained genuine heap pressure keeps acquisition at or near zero; this design cannot guarantee progress if the process lacks memory headroom. + +### Monitoring state + +The monitoring state is derived for observability and is not consulted by the algorithm: + +| State | Condition | +|---|---| +| `PAUSED` | `admissionPaused` | +| `PRESSURED` | CPU `pressureSince` is set, or a high collection is newer than the last reduction | +| `STALE` | A supported metric is not fresh | +| `HEALTHY` | None of the above | + +Precedence is the order listed. An unsupported metric is absent from every condition. With only one supported metric, growth, pressure, and pause decisions use that metric alone. With neither supported and demand-only operation enabled, the controller skips resource gating entirely and growth is bounded by demand and configured ceilings; this is logged at startup as a warning. + +## Adjusting concurrency targets + +The process target bounds aggregate admission. Worker targets are its division among queues in proportion to backlog, each capped at the queue's demand. The process target changes on the control timer and on a drained resume. Worker targets are recomputed whenever the process target changes, after every depth sweep, and after an empty response; never after every task. + +### Demand and allocation + +A queue's backlog is the slots it holds plus the tasks waiting for it. Its demand is the work it could run now: the backlog capped by its maximum. Shares are weighted by backlog, not demand, so a queue with 600 waiting tasks outweighs one with 100 even when both exceed a maximum of 64; weighting by the capped demand would give them equal shares exactly when backlogs are largest. + +```text +function backlog(queue, now): + depth = (now - queue.depthObservedAt <= depthFreshness) ? queue.depth : 1 // unknown depth counts as one + if queue.consecutiveEmptyPolls > 0: // the server delivered nothing from this depth + depth = min(depth, 1 + max(0, depth - queue.depthAtEmptyPoll)) // one, plus tasks that arrived since + return (queue.heldSlots - queue.standbyPolls) + depth // a standby slot counts once it delivers a task + +function demand(queue, now): + return min(queue.maximumConcurrency, backlog(queue, now)) + +function standbyDemand(queue, now): // the slot of the queue's standby poll + held = queue.heldSlots - queue.standbyPolls + return 1 if standby polls are enabled AND the queue was pollable at the last sweep + AND backlog(queue, now) == held AND held < queue.maximumConcurrency + else 0 + +function allocate(now): + b[q] = backlog(q, now); d[q] = demand(q, now); total = sum(d) + if total <= process.targetConcurrency: + share[q] = d[q] // every queue gets all it can use + else: + remaining = process.targetConcurrency; open = queues with d[q] > 0 + repeat: // water-filling + quota[q] = remaining * b[q] / sum(b over open) for q in open + for each q in open with quota[q] >= d[q]: + share[q] = d[q]; remaining -= d[q]; remove q from open // capped at its demand + until no queue was capped + share[q] = floor(quota[q]) for q in open + share[q] = 1 for q in open with share[q] == 0, while slots remain // floors before rounding + give one more slot to the remaining queues in open + with the largest fractional parts of quota // largest-remainder rounding: shares sum to the target + share[q] = max(1, share[q]) for each queue with d[q] > 0 // floor; can push the sum above the target + queue.targetConcurrency = share[q] for each queue // 0 where d[q] == 0 +``` + +Shares are proportional to backlog. A queue whose proportional share would exceed its demand is held at its demand and the excess goes to the other queues, so no slot is assigned that a queue cannot use. Floors are taken from the leftover before rounding, so they push the sum above the target only when there are more queues with work than slots. Largest-remainder rounding keeps the shares summing exactly to the process target, so rounding never leaves a slot that no queue may use; the floor of one keeps every queue with waiting work polling; and when total demand is at or below the process target every queue gets its whole demand. A share never exceeds demand, so a request never asks for more than the sampled depth. Floors can push the sum of the shares above the process target; the process target still bounds admission, and the grant rule in "Reserving poll capacity" decides who gets the difference. A queue holding long-running tasks keeps its share while they run, because held slots are demand, and releases it as they finish; nothing is preempted. + +Two rules make depth safe to rely on. Unknown counts as one: while the depth endpoint is unavailable, every queue asks for one slot beyond what it holds, so a busy queue grows one slot per tick as it fills and an idle queue costs one single-task poll per backoff interval. An empty response caps the queue at one waiting task until a delivery, plus whatever arrived after it: server limits, other consumers, or a backend that counts in-flight messages can all leave a positive depth undeliverable, and without this rule such a queue would hold a full-size request for every poll timeout. Tasks that arrive after the empty response are new evidence, so they count in full, and a sweep that sees them also cancels the queue's empty-response backoff. Without that exception an idle queue, whose heartbeats all come back empty, met a burst of 200 tasks with a share of one and waited out a five-second backoff before polling, despite a warm process target of 16. + +Depth is sampled once per interval on a dedicated thread. A sweep is one batch request that names every logical queue the process polls, a domain queue as `domain:taskType`, and returns every size at once. The sampler also evaluates `worker.paused()` and the poll filters there, outside the lock, and records a depth of zero for a worker that would not poll, so a paused or filtered worker keeps the share its running tasks need and nothing more, and user code never runs under the controller lock. + +```text +function queueName(queue): + return queue.domain == null ? queue.taskType : queue.domain + ":" + queue.taskType // the server's own naming + +function sampleDepths(now): + // user code and the network call run outside the lock + pollable[q] = NOT worker.paused() AND poll filters accept the queue, for each queue + names = queueName(q) for each queue with pollable[q] + sizes = names is empty ? {} : taskClient.getQueueSizes(names) // GET /tasks/queue/sizes, one request + + under controller lock: + for each queue: + if NOT pollable[q]: queue.depth = 0; queue.depthObservedAt = now + else if the request succeeded: queue.depth = max(0, sizes[queueName(q)]); queue.depthObservedAt = now + else: queue.depthObservedAt = NONE // unknown counts as one + applyTargets() // re-divides the process target; wakes queues whose share rose +``` + +A name missing from a successful response is read as zero, which is what the server returns for a queue it has never seen; a failed request leaves every queue unknown until the next sweep. + +Use fixed-delay scheduling, a finite request timeout, and the existing sampled error logging. The one request must finish within the depth freshness window; a server that cannot answer in that time makes every depth unknown, which degrades to delivery-driven growth rather than stalling. Depth reads never reserve slots and never run in the poll loop. Discovery latency for an idle queue is one sampling interval plus one poll. Server load: one request per process per second replaces the ten idle polls per second per queue that the default 100-millisecond polling interval produces today, and idle queues stop polling except for heartbeats. The server still runs one count per name behind the request, and on Postgres each is a count over the queue's rows, so very large fleets may want a longer interval. + +### Control tick + +```text +function controlTick(now): + under controller lock: + reconcileHeapPause(now) + + cpuFresh = cpu.supported AND now - cpu.observedAt <= freshness + heapFresh = heap.supported AND now - heap.observedAt <= freshness + + cpuPressure = cpuFresh AND cpu.pressureSince != NONE AND now - cpu.pressureSince >= 3 seconds + heapPressure = heap.supported AND heap.lastHighCollectionAt > process.lastReductionAt + + if (cpuPressure OR heapPressure) AND now - process.lastReductionAt >= reductionInterval: + process.targetConcurrency = max(1, floor(0.75 * process.targetConcurrency)) + process.lastReductionAt = now + if cpu.pressureSince != NONE: cpu.pressureSince = now // the next reduction needs 3 more seconds of pressure + if heapPressure: limitHeapGrowth(process.targetConcurrency, now) + applyTargets() // allocate() cuts every share with it + return + + total = sum(demand(queue, now) + standbyDemand(queue, now) for each queue) + + if total >= process.targetConcurrency: + process.surplusSince = NONE + else if process.surplusSince == NONE: + process.surplusSince = now + else if now - process.surplusSince >= decayInterval: // 30 seconds of surplus + process.targetConcurrency = min(process.targetConcurrency, // decay only lowers + max(minimumConcurrency, total, ceil(process.targetConcurrency / 2))) + process.surplusSince = now + + heapPeak = highest post-collection utilization of the last 30 seconds, and the latest + growthAllowed = + total > process.targetConcurrency + AND NOT process.admissionPaused + AND now - process.lastReductionAt >= reductionInterval + AND (NOT cpu.supported OR (cpuFresh AND cpu.smoothed < growthThreshold + AND cpu.raw < cpuPressureThreshold)) + AND (NOT heap.supported OR (heapFresh AND heapPeak < growthThreshold)) + relaxHeapGrowthCeiling(now) // after 120 s without heap pressure: +1 per tick + if growthAllowed: + increment = clamp(ceil(0.20 * process.targetConcurrency), 1, 16) + increment = min(increment, heapGrowthCeiling - process.targetConcurrency) + taskSlots = process.heldSlots - slots held by standby polls + if heapFresh AND taskSlots > 0 AND heapPeak > heap.baseline: + perSlot = (heapPeak - heap.baseline) / taskSlots + increment = min(increment, floor((growthThreshold - heapPeak) / perSlot)) + if increment > 0: + process.targetConcurrency = min(process.maximumConcurrency, total, + process.targetConcurrency + increment) + + applyTargets() + +function limitHeapGrowth(ceiling, now): // on a heap reduction, and on a pause with 0.75 * max(target, held) + heapGrowthCeiling = max(1, min(heapGrowthCeiling, ceiling)); heapCeilingSince = now +``` + +Reduction needs evidence newer than the previous reduction. For CPU the evidence is a run of high samples; resetting `pressureSince` at the reduction means the next reduction needs a run that starts after this one. For heap the evidence is the collection time of the latest high collection; re-reading a stale post-collection value is never new evidence, so one high collection yields at most one reduction. The first reduction has no interval to wait for because `lastReductionAt` starts at negative infinity. Growth waits for the same interval after a reduction, which is the growth cooldown. A reduction cuts only the process target; every share shrinks with it through `allocate()`. + +The process target rises while total demand exceeds it, never past total demand or its ceiling. A single deep queue therefore carries it, a queue holding long-running tasks keeps counting through its held slots, and idle queues contribute nothing. Growth is 20% per tick with a floor of one slot, so a process at one slot still ramps and aggregate growth stays bounded and resource-gated. Depth is shared by every process polling the queue, so total demand overstates what this process alone will receive. That only means the target can run ahead of delivered work, which costs idle threads and nothing else, and decay returns it. + +Heap needs two more limits, because running out of it fails tasks rather than slowing them. End-to-end testing with tasks that hold multi-megabyte payloads in a 123 MB heap showed both failure modes. First, the growth gate reads the last post-collection value, which does not yet include what newly granted slots will allocate, so a 20% step taken at 70% can land above 100%. Growth is therefore also limited to the slots whose observed cost fits under the growth threshold: the heap held by running tasks is the post-collection utilization minus the baseline, the lowest post-collection utilization seen, and dividing it by the slots held for tasks gives the cost of one more. Growth decisions use the highest post-collection reading of the last 30 seconds, not the latest: tasks that hold payloads for seconds make each reading depend on which of them a collection happened to find live or promote, and on a host with a 128 MB heap a low reading taken just after a high one let the target grow two slots into an `OutOfMemoryError`. Pause and resume still use the latest reading, since they must react to it. Second, after a pause and a reduction the controller regrew past the level that had caused the pressure and ran tasks out of memory, twelve times in one run. A heap reduction therefore leaves a growth ceiling at the reduced target, and a pause leaves one at three quarters of what was running; the ceiling holds for two minutes, then rises one slot per control tick, so the controller probes back toward the pressure point slowly enough for a collection to report before it gets there. CPU has no such ceiling because CPU saturation slows tasks without failing them. + +The control tick runs from a fixed-delay timer that checks a jittered due time, not from a task that reschedules itself, and every timer catches and drops anything thrown, including failures to log it. When the heap ran out end to end, a self-rescheduling tick stopped for good: the target never changed again, through three more heap pauses. + +When total demand has been below the process target for thirty seconds, the target shrinks to the largest of the configured minimum, total demand and half of itself, and again every thirty seconds while the surplus lasts. It never drops below demand, and demand is never below held slots, so decay never creates an overshoot. A process that goes idle reaches the minimum, one by default, within a few minutes, and the next burst ramps from there under the growth gate instead of jumping to a stale ceiling. A higher minimum trades nothing while idle, since idle queues have no shares and their pools release their threads, for a burst that starts with that many slots ready. Decay is not a reduction: it does not set `lastReductionAt` and starts no growth cooldown. + +If the process is at its ceiling, capacity released by one queue is used by others through ordinary slot grants without raising the ceiling. The controller never terminates running tasks to rebalance capacity. + +## Reserving poll capacity + +### Shares and the grant rule + +Shares are targets, and the grant rule makes holdings follow them. A queue's deficit is `targetConcurrency - heldSlots`, how far it is below its share. A freed process slot, or a slot opened by a higher process target, goes to the waiting queue with the largest positive deficit; equal deficits are served in arrival order, and a queue without a positive deficit is never granted a process slot. A queue at or above its share waits on its own worker target, not on the process, so it cannot take process slots from queues below theirs. When floors make the shares sum to more than the process target, deep queues reach their shares first and the remainder rotates among the queues with floor shares, each of which gets a slot in turn as tasks finish. + +Idle queues have no share and poll only with standby polls on slots no share claims, or as heartbeats, one short single-task request per interval on an open slot, so they cannot starve busy ones. The only queues that poll without receiving are those whose depth cannot be delivered, and the empty-response rule holds each of them to one slot. An empty long poll still holds that slot for the whole timeout, so deployments with many such queues should keep poll timeouts short. + +```text +function deficit(queue): + return queue.targetConcurrency - queue.heldSlots + +function reserveBatch(queue, now): + under controller lock: + if process.admissionPaused: + return WAIT_FOR_PROCESS_SLOT // no poll without an open slot; the resume wakes every waiter + + processAvailable = max(0, process.targetConcurrency - process.heldSlots) + workerAvailable = max(0, deficit(queue)) + + if workerAvailable == 0: // no share, or the share is already held + if standby polls are enabled AND queue.outstandingPolls == 0 AND now >= queue.nextStandbyAt + AND queue.heldSlots < queue.maximumConcurrency + AND unclaimedSlots() > dueHeartbeats: + count = 1 // standby: one task, server waits the standby timeout + queue.standbyPolls += 1 + queue.nextStandbyAt = now + standbyPollTimeout + else if queue.outstandingPolls > 0 OR now < queue.nextHeartbeatAt: + return WAIT_FOR_WORKER_SLOT // the wait ends at nextStandbyAt or nextHeartbeatAt + else if processAvailable == 0 OR queue.heldSlots >= queue.maximumConcurrency + OR (processAvailable == 1 AND a more overdue heartbeat is waiting): + return WAIT_FOR_HEARTBEAT_SLOT // stays due; woken first when a slot opens + else: + count = 1 // heartbeat: one task on the open slot, no server wait + else: + dueHeartbeats = queues waiting with a heartbeat due + if processAvailable <= dueHeartbeats: + return WAIT_FOR_PROCESS_SLOT // due heartbeats take open slots first + if some queue waiting for a process slot has deficit(w) > deficit(queue): + return WAIT_FOR_PROCESS_SLOT // the slot belongs to a queue further below its share + count = min(workerAvailable, processAvailable - dueHeartbeats, maximumBatch) + + ensure execution pool can accommodate queue.heldSlots + count + queue.heldSlots += count + queue.outstandingPolls += 1 + queue.nextHeartbeatAt = now + heartbeatInterval // any poll is a heartbeat for the server + process.heldSlots += count + return Reservation(count) + +function unclaimedSlots(): // neither held nor promised to a share + return max(0, process.targetConcurrency - sum(max(q.targetConcurrency, q.heldSlots) for each queue q)) +``` + +A poller that cannot reserve waits on the resource it lacks. Each logical queue keeps a FIFO of pollers waiting for its worker target, since pollers of one queue are interchangeable. Every wait is bounded: a poller re-attempts at least once per second, so it re-checks worker pause and shutdown on that cadence and a missed wakeup costs at most that bound rather than a hang. The process keeps the set of queues waiting for a process slot and grants by deficit as described, so every wakeup goes to a poller that can use the freed slot and hundreds of waiting queues never stampede. A finished task frees a worker slot and a process slot, so it wakes one worker waiter of its queue and the process waiter with the largest positive deficit. `applyTargets()` wakes as many waiters per resource as the new targets admit, in deficit order for the process, and every waiter when the pause lifts; a sweep that raises an idle queue's share from zero wakes it this way. It also wakes any process waiter whose deficit is no longer positive, so that poller re-enters `reserveBatch()` and parks on its worker target instead; the process waiting set therefore holds only queues that a freed slot could serve. The deficit check in `reserveBatch()` means a poller that arrives while a needier queue waits joins the waiters instead of taking the slot, so one queue cannot monopolize freed capacity by re-polling immediately. A woken poller re-checks worker pause and shutdown before reserving. + +An outstanding empty long poll holds only the slots it was granted, and all requests have a finite timeout. The design provides fair opportunities rather than preemption: an outstanding request cannot be revoked safely because a different queue became busy. + +### Heartbeat polls + +A queue with no share and no standby poll still polls once per heartbeat interval, and like every poll the heartbeat needs an open slot: it reserves one process slot and requests one task for it. It asks the server not to wait, so the slot is held only for the request. A heartbeat that cannot get a slot stays due and waits; when a slot opens, due heartbeats are woken first, oldest first, ahead of queues below their share. That priority costs each busy queue at most one short request per idle queue per interval, and it is what keeps idle domains alive under load: end-to-end testing showed that when freed slots always went to queues below their share, an idle domain never polled, went stale, and its next task fell back to the default queue, which no worker polled. Staying due rather than retrying an interval later matters for the idle case: an idle process decays to one slot, and when refused heartbeats were pushed a full interval the same queues kept losing to each other, two of four going 26 seconds unpolled. With every slot held for longer than the server's active-poller window, nothing polls and domain routing may lapse until a slot opens; fixed mode behaves the same way when a task type's threads are all busy, and the minimum process concurrency keeps slots available in an idle process. Nothing polls while admission is paused. The heartbeat exists for the server: domain fallback routes a task to the first domain polled within the active-poller window, every poll refreshes that record, and a worker that stopped polling an idle domain would stop being offered its tasks within ten seconds. The heartbeat also checks a zero depth against the queue itself, which protects against a backend whose size query undercounts, and keeps the poll-data views operators rely on. A heartbeat that returns a task resets the empty streak, the task holds the slot like any other, and the next sweep counts it in the queue's demand; one that returns nothing takes the empty-response path. A paused worker sends no heartbeat, because the poll loop honors the pause before reserving, so the server stops routing to a domain whose worker is paused, exactly as today. The poller of a share-less queue waits in its worker FIFO with a timed wait that ends at `nextStandbyAt` or `nextHeartbeatAt`, and every poll pushes the heartbeat one interval forward, because any poll refreshes the server's record. + +### Standby polls + +Shares follow depth samples, so a queue that has caught up, holding exactly its share, learns of its next task only at the next sweep, and an idle queue only at its next heartbeat. End-to-end, that was the largest latency gap against fixed threads, which keep polling every queue: tasks waited about a second to be picked up after idle, against 30 to 100 milliseconds with fixed threads, and steady-state medians were 1.3 to 2.6 times higher. + +A queue whose share is fully held therefore keeps one standby poll open: a single-task long poll whose server wait is the standby timeout. The server answers it the moment a task arrives, so pickup costs one round trip. Like every poll it holds an open slot for the task it may return, but only a slot no queue's share claims: the process target minus, for each queue, the larger of its share and its held slots. It also requires a free process slot outright, because unclaimed slots are computed over registered queues and can overstate free capacity while a deregistered worker's tasks drain. A standby poll therefore never takes a slot that a queue below its share could use; when a share grows, the slots it needs are either unclaimed or held by standby polls that return within the standby timeout, which bounds the delay. Due heartbeats still go first. Under contention every slot is claimed, standby polls stop, and idle queues fall back to heartbeats. + +The control tick counts one standby slot for each pollable queue that has nothing waiting beyond what it holds, so with resource headroom the process target keeps room for every queue's standby poll; an idle process settles at one slot per queue, or at the minimum process concurrency if that is higher. A standby slot holds no thread: execution threads are created when a task is submitted and retire when idle. Standby slots are not part of a queue's backlog until the poll delivers a task, which then holds the slot and counts like any other; allocation and the replacement check ignore standby slots, so standby polls never change how shares are divided. + +A standby poll that runs out its wait on an empty queue is expected, not a sign of an undeliverable queue: it does not count toward the empty streak or the backoff, and the next one follows when the wait has passed, so an idle queue sends one request per standby timeout, never faster even if the server answers early. A standby poll that delivers a task is followed by the next one at once. A failed standby poll is a poll error and backs off like any other. Each standby poll also refreshes the server's active-poller window, so a queue with standby polls needs no heartbeat. + +## Task polling algorithm + +The existing `pollAndExecute()` and `pollTasksForWorker()` loop is retained. Its steps today are: check `worker.paused()`, apply poll filters, acquire every free permit, poll with `count` equal to the permits acquired, release the permits not matched by returned tasks, submit each task to the executor, schedule lease extension, and on an empty result sleep for the polling interval. Adaptive mode changes four things in that loop: + +1. Permits come from `reserveBatch(queue)` instead of a fixed semaphore. +2. Submission and lease-extension scheduling move into a shared `dispatch(task)` so update-and-fetch replacements take the same path. +3. The empty-result sleep becomes the backoff described below; a poll error takes the same path. +4. The in-memory `tasksTobeExecuted` queue is not consulted. + +```text +function pollLoop(queue): + while running: + honor existing worker pause and poll filters // unchanged + wait until queue.nextPollAllowedAt + + reservation = controller.reserveBatch(queue, now) + if reservation is a WAIT_FOR result: + wait in that resource's FIFO until woken or shut down, + or, in the worker FIFO, until queue.nextHeartbeatAt or queue.nextStandbyAt, + and at most one second before re-attempting + continue + + pollStartedAt = now + try: + tasks = taskClient.batchPollTasksInDomain( // existing client call + taskType, domain, worker.getIdentity(), + reservation.count, + 0 for a heartbeat, standbyPollTimeout for a standby poll, else configuredPollTimeout + ) + set clientReceiveTime on each task // unchanged + record requested count, returned count, and poll duration + + under controller lock: + unused = reservation.count - size(tasks) + queue.heldSlots -= unused + process.heldSlots -= unused + if reservation is a standby poll: queue.standbyPolls -= 1 + + if tasks is empty AND reservation is a standby poll: + pass // its wait ran out; nextStandbyAt paces the next one + else if tasks is empty: + queue.consecutiveEmptyPolls += 1 + queue.depthAtEmptyPoll = (now - queue.depthObservedAt <= depthFreshness) ? queue.depth : 0 + queue.nextPollAllowedAt = max(now, pollStartedAt + emptyDelay(queue)) + applyTargets() // the share now counts one waiting task; see "Demand and allocation" + else: + queue.consecutiveEmptyPolls = 0 + queue.nextPollAllowedAt = now + queue.nextStandbyAt = now + + for each returned task: + dispatch(task) // the task owns one worker slot and one process slot + + catch poll error: + under controller lock: + release the reservation // count worker and process slots + same bookkeeping as an empty response // streak, backoff, applyTargets() + record poll error // existing sampled logging and PollFailure event + if fatal authentication cause: existing exit action + + finally: + under controller lock: + queue.outstandingPolls -= 1 + wake capacity waiters + +function dispatch(task): + if task is a replacement from update-and-fetch: // it reuses the finishing task's slots + under controller lock: + queue.consecutiveEmptyPolls = 0 + try: + future = queue.executionPool.submit(() -> executeWithSlot(task)) + catch rejected execution: + release the task's worker and process slots // existing behavior logs and drops the task + return + if task.responseTimeoutSeconds > 0 AND worker.leaseExtendEnabled(): + schedule lease extension for task against future // existing extendLease logic +``` + +Request cleanup never releases slots already transferred to tasks. An error after receiving a batch must not treat already accepted tasks as an empty response or fetch a replacement batch outside the capacity budget. Use the existing task failure/reporting lifecycle for accepted work; do not introduce a new requeue protocol. + +Never send a task poll with `count=0`. Waiting for capacity is local and does not count as an empty poll or server failure. + +### Poll timing + +```text +Empty response or poll error: + delay = min( + max(configuredPollInterval, 5 seconds), + configuredPollInterval * 2^(emptyStreak - 1) + ) + nextPollAllowedAt = max(now, pollStartedAt + delay) // the long poll's own wait counts toward the delay +``` + +Apply ±20% jitter, bounded arithmetic, and a positive delay. Measuring from poll start means a long-poll timeout that already covered the delay adds no second wait, while short timeouts still back off exponentially. Protecting busy queues from idle ones is not this delay's job; allocation does that, since a queue without depth has no share and polls only as a heartbeat. Honor applicable server retry instructions. A nonempty response resets the streak. A poll error is an empty response for backoff and demand: the existing runner also treats a failed poll as an empty result, and the capped backoff keeps a process with many queues from hammering an unavailable server. Existing terminal authentication behavior remains authoritative. + +After a nonempty response, poll again immediately if capacity is available. Task completion wakes capacity waiters but does not bypass an active empty-poll or server-error delay. + +Keep configured long-poll timeout and poller count stable in this version. The server rejects timeouts above 5000 milliseconds; configuration validation enforces that bound. The timeout must also fit inside the HTTP read timeout with room for request/response overhead. + +## Adjusting execution threads + +Each logical queue has one execution pool shared by its configured pollers. Adaptive concurrency describes admitted task slots independently of thread implementation: the controller bounds concurrency with slots, and the executor exists to run the tasks those slots admit. Two executor strategies are supported, selected by the existing `useVirtualThreads` flag. Platform threads are the default; virtual threads are opt-in. The controller, resource sampler, and lease-extension executors stay small platform scheduled executors under either strategy and are never resized from task targets. + +| | Platform threads (default) | Virtual threads (`useVirtualThreads`) | +|---|---|---| +| Executor | Resizable `ThreadPoolExecutor` built with the existing platform thread factory and uncaught-exception handler | `Executors.newThreadPerTaskExecutor(factory)` built with the existing virtual thread factory and uncaught-exception handler | +| Concurrency bound | Slots, and the pool size that follows them | Slots only; one virtual thread per admitted task | +| Reaction to a target change | Pool resized as described below | Nothing to resize | +| Idle threads | Retire after 60 seconds | None exist; a virtual thread ends with its task | +| Thread cost at the 256 process ceiling | 256 platform threads and stacks | Negligible; stacks live on the heap and are covered by the heap signal | +| CPU-bound tasks | Limited by the pool size the controller chooses | Still limited by carrier threads, whose count defaults to available processors; the CPU signal governs | + +### Platform threads + +The pool must accommodate capacity already promised, even when a target has just fallen: + +```text +requiredPoolSize = max(1, queue.targetConcurrency, queue.heldSlots) + +function resizePool(pool, requiredSize): + if requiredSize > currentConfiguredSize: + pool.setMaximumPoolSize(requiredSize) + pool.setCorePoolSize(requiredSize) + else if requiredSize < currentConfiguredSize: + pool.setCorePoolSize(requiredSize) + pool.setMaximumPoolSize(requiredSize) +``` + +Prepare capacity before granting a new reservation, and recompute the required size after target changes and slot releases. Core and maximum are kept equal, so the pool must be created with `allowCoreThreadTimeOut(true)` and a 60-second keep-alive for idle threads to retire. Threads are created lazily as tasks are submitted, so raising the size does not start threads immediately. On reduction, running tasks continue; threads above the new requirement retire as they become idle. + +Bound the executor handoff queue by the worker adaptive maximum. Slots bound accepted work, so a task waits in the handoff queue only for the moment between one thread finishing and picking up the next; the queue is not a prefetch buffer. Thread creation or submission failure must not leak a slot, and accepted tasks must follow the existing failure/lifecycle policy. + +### Virtual threads + +Today the runner pools virtual threads inside `Executors.newFixedThreadPool`, which works but contradicts the JDK guidance that virtual threads are not pooled. In adaptive mode the slot controller is the concurrency bound, so the virtual-thread strategy uses a thread-per-task executor and skips pool sizing entirely. `requiredPoolSize` and `resizePool` are not used. Submission failure is handled exactly as for platform threads: the slots are released and the task follows the existing failure policy. + +The virtual thread factory keeps the existing uncaught-exception handler. Thread naming uses the numbered form `Thread.ofVirtual().name(prefix, 0)` so threads are distinguishable; the current fixed-name form leaves every virtual thread named with the literal prefix pattern. + +Operational notes for the opt-in: + +- Carrier parallelism defaults to available processors and is tunable with `jdk.virtualThreadScheduler.parallelism`. CPU-bound workers gain nothing from virtual threads; the CPU signal still limits their concurrency. +- Before JDK 24, a virtual thread that blocks inside a `synchronized` block or method pins its carrier. Worker code and libraries that do blocking I/O under `synchronized` can exhaust carriers at high slot counts. Recommend JDK 24 or later for the opt-in and `-Djdk.tracePinnedThreads=full` for diagnosis on earlier releases. +- Thread-locals and large per-task stacks are heap allocations per virtual thread. The heap signal covers them; no separate accounting is added. +- Thread-count metrics from `ThreadMXBean` do not include virtual threads. Observability reports active slots and the executor strategy rather than thread counts. + +## Execution, reporting, and replacement tasks + +### Ordinary execution + +The existing `processTask()`, `executeTask()`, `updateTaskResult()`, `handleException()`, and `retryOperation()` chain is retained with these changes. `updateTaskResult()` returns the slot disposition, `SLOT_TO_RELEASE` or `SLOT_TRANSFERRED`, instead of void. The `finally` block of `processTask()`, which today cancels lease extension, releases a semaphore permit, and publishes `ActiveWorkersChanged`, releases the slots only if no call in the invocation transferred them and records execution and reporting durations. `updateTaskResult()` can run twice for one task when the worker fails after a successful update, because the outer catch in `processTask()` reports again through `handleException()`. A replacement is therefore attempted at most once per invocation, and once the slots are transferred every later update in that invocation uses the plain update call. + +```text +function executeWithSlot(task): + try: + existing processTask body: + execute task using existing worker behavior + report result using existing update and retry behavior + record execution and reporting observations + finally: + cancel lease extension for this task // unchanged + release the task's worker and process slots exactly once, unless transferred + publish ActiveWorkersChanged // unchanged + recompute required execution pool size + wake capacity waiters +``` + +The slots stay held through result-reporting retries. A successfully reported `IN_PROGRESS` result releases them when the invocation finishes. Adaptive mode does not change task-result semantics, retry policy, callback behavior, or server timeout handling. + +### Update and fetch + +Today update-v2 is enabled process-wide by the `taskUpdateV2` system property or environment variable, the returned task is appended to the in-memory `tasksTobeExecuted` queue, the next poll drains that queue before calling the server, and lease extension for the drained task is scheduled by the poll loop. In adaptive mode the in-memory queue is not used, a builder option enables update-v2 per runner with the system property as a fallback, and the replacement is dispatched directly through `dispatch(task)` so it is scheduled for lease extension exactly like a polled task. + +When update-and-fetch is enabled, the result-update request can itself accept another task. Authorize a replacement before sending that request. The replacement reuses the finishing task's slots instead of releasing them and later acquiring new ones. + +```text +function reportAndMaybeContinue(task, result): + replacementAllowed = + NOT process.admissionPaused + AND queue.heldSlots <= queue.targetConcurrency // within its share, counting the finishing task + AND process.heldSlots <= process.targetConcurrency + AND deficit(queue) + 1 >= largest positive deficit among queues waiting for a process slot + // the freed slot would be granted back to this queue + + if updateAndFetchEnabled AND replacementAllowed: + nextTask = taskClient.updateTaskV2(result, taskType) // existing call, existing retry wrapper + record reporting outcome for current task + if nextTask exists: + finish current task's invocation and lease bookkeeping + dispatch(nextTask) // takes over the finishing task's slots + return SLOT_TRANSFERRED + else: + taskClient.updateTask(result, taskType) // existing call, existing retry wrapper + record reporting outcome for current task + + return SLOT_TO_RELEASE +``` + +The caller releases only on `SLOT_TO_RELEASE`; its `finally` must not release transferred slots. Dispatch continuations through the executor handoff, not recursive calls that grow the stack. + +If pressure rises while an authorized update-and-fetch request is outstanding, its possible returned task remains backed by the slots. Handle that accepted task and reconsider replacement permission afterward. Using ordinary updates when permission is denied lets the grant rule take effect without a server change. + +## Shutdown and worker registration lifecycle + +Existing `TaskRunnerConfigurer.shutdown()` and `TaskRunner.shutdown(timeout)` behavior is retained: stop polling, cancel lease futures, shut down the lease executor, then shut down the execution pool and await termination within the grace period. Adaptive mode adds: + +1. Stop granting new polls and replacement requests to that worker. +2. Let outstanding polls resolve within their finite request timeouts. +3. Process and report accepted work under existing graceful-shutdown behavior. +4. Release reservations and remove the worker's pollers from the wait FIFOs and the process waiting set. +5. The shared controller reference-counts participating runners. Resource sampling, depth sampling, the control timer, and GC notification listeners start with the first registration and stop when the count returns to zero; a later registration restarts them and recomputes the initial process target from the queues registered at that moment. + +Newly registered queues enter the next allocation and compete for grants like any other. Initial process concurrency is computed once at controller start; later registration does not raise process capacity or override a reduction or admission pause. The annotated-worker executor already rebuilds the configurer on restart, which recomputes the initial target. + +## Observability + +Expose bounded controller observations through the existing event dispatcher and `TaskRunnerEventsListener`. Add new event types as default methods so existing `MetricsCollector` implementations compile unchanged, and extend `PrometheusMetricsCollector` with gauges for them: + +- Process and worker concurrency targets, ceilings, held slots, sampled depth and its age, demand, deficit, and the execution thread strategy per queue. +- Task-slot age per queue: seconds since a task slot was last released while any are held. Completing workers keep it near their task duration; a wedged worker's age grows without bound, which no throughput metric shows. +- Outstanding polls, requested and returned batch sizes, empty responses, heartbeat polls, poll errors, and depth sample failures. +- CPU raw and smoothed readings; post-GC heap reading; time since last collection; per-metric support, validity, and age; derived monitoring state; admission-pause latch. +- Last reduction time, pressure evidence times, and which resume path cleared the last pause. +- Execution duration and reporting duration per task, recorded separately so reporting delays are visible; successfully reported completions, failed results, and `IN_PROGRESS` results, recorded separately. The algorithm does not use completion throughput; see deferred work for the throughput probe. +- Adjustment reason: demand growth, CPU pressure, heap pressure, demand decay, reallocation, drained resume, or ceiling. + +Emit a bounded adjustment observation when targets change. Avoid per-task controller logs and task identifiers as metric labels. These observations explain the controller; control accounting itself remains direct and synchronous with slot ownership changes. + +## Validation and acceptance + +Use an injected monotonic clock, resource sampler, and fake Conductor client for deterministic algorithm tests. Validate task dispatch behavior against a real Conductor server without altering task-definition policies for the feature. The existing `harness` module with its simulated workers and Prometheus collector is the starting point for load tests. + +| Scenario | Expected behavior | +|---|---| +| Startup with queued work | Before the first depth sweep every queue's share is one task; after it, shares follow depth; process target is min(queues, processors) | +| Sustained backlog with resource headroom | The process target grows in bounded steps toward total demand; shares follow depth; requests are sized by free slots up to the batch maximum | +| Two queues with depths 10 000 and 100, process target 32 | Shares 32 and 1; the deep queue's deficit is served first; the shallow queue gets a slot as tasks finish | +| Twenty queues, one deep, process target 32 | The deep queue holds its share; the nineteen floor shares rotate through the remaining slots in deficit, then arrival, order | +| Underfilled nonempty responses | Share unchanged; the queue polls again immediately while it has a deficit | +| Server task concurrency or rate limit | SDK respects actual delivery and does not change server policy | +| CPU at or above 90% for 3 seconds | One 25% reduction; accepted tasks continue; the next reduction needs a new 3-second run and 15 seconds | +| CPU averaging 65% with one-second samples above 75% | Growth continues; smoothed stays below 75% and samples below 90% | +| CPU jumps from 20% to 100% | First sample vetoes growth despite a low average; reduction after 3 seconds | +| Healthy JVM whose used heap reaches 95% before each young collection | No pause and no reduction; post-GC utilization stays below thresholds | +| Post-GC heap at or above 90% | Admission closes immediately; execution and reporting continue; one reduction at the next permitted tick | +| Next collection below 80% | Admission resumes at current targets; the reduction still lands if the interval had not permitted it yet; growth resumes after cooldown once post-GC is below 75% | +| Collection during a pause shows 85% | Remains paused | +| Paused, tasks drained, 30 seconds since the high collection | Resume at concurrency one; the stale reading does not re-enter the pause; growth waits for a collection below 75%; a later collection at or above 90% re-pauses | +| One high collection, then no further collections | One pause entry and one reduction; stale re-reads produce neither | +| Heap flapping across 80% and 90% every few seconds | Admission closes on each high collection; at most one reduction per 15 seconds | +| Startup before the first collection | Used heap serves as the heap reading; valid from the first tick; growth proceeds when it is below 75%; no pause can occur | +| Collection-usage fallback path | Collections detected from collector count deltas; `collectionAt` accurate to one sampling interval | +| CPU metric unsupported by the runtime | Logged once; controller runs heap-only with full growth and pressure behavior | +| Both metrics unsupported | Startup fails with a configuration error; with demand-only enabled, growth follows demand within ceilings | +| Sampler stops after healthy readings | Stale metrics freeze growth; targets hold; the pause latch is preserved | +| JVM pause or delayed sampling | No catch-up samples; smoothing and pressure timing reset after a gap longer than the freshness window | +| Hundreds of queues registered at startup | One batch depth request per second; queues without depth poll only as heartbeats; admission bounded by the process target | +| Hundreds of idle queues and few process slots | Each idle queue takes an open slot for one non-waiting single-task request per interval; the busy queue keeps the rest of the process target | +| Idle queues in an idle process | The target keeps one slot per queue, and each queue holds a standby poll on it, so a task is picked up as it arrives and poll data stays fresh. With standby polls disabled or no slot unclaimed, heartbeats take turns on the open slots, oldest due first, so every queue polls every 5 seconds, poll data stays inside the server's 10-second active-poller window and domain fallback keeps routing to each domain | +| A queue's share grows while standby polls hold the unclaimed slots | The new share uses unclaimed and freed slots at once; slots held by standby polls return within the standby timeout, or sooner when the poll delivers a task | +| Idle domain queue while other queues wait for process slots | Its due heartbeat takes the next open slot ahead of them, once per interval, keeping domain routing on this worker | +| Every slot held for longer than the active-poller window | Nothing polls; domain routing may lapse until a slot opens, as in fixed mode when a task type's threads are all busy | +| Backend reports zero depth for a non-empty queue | The heartbeat returns a task, resets the empty streak, and the next sweep gives the queue a share | +| Positive depth the server does not deliver (task limits, other consumers, in-flight counting) | After the first empty response the queue asks for one task per backoff interval; its share counts one waiting task until a delivery | +| Queue with four pollers and a share of one | One outstanding poll of one slot; the other pollers wait on the worker target | +| Freed process slot while several queues wait | The queue with the largest deficit is woken; a poller arriving with a smaller deficit joins the waiters | +| Poller count of N for one busy queue | One shared share and pool; up to N outstanding polls while the queue has a deficit | +| Busy queue's depth drops to zero | Its share falls to its held slots; no further polls until a sweep shows depth; running tasks finish normally | +| Total demand below the process target for 30 seconds | The process target shrinks to the larger of demand and half of itself, repeatedly, never below demand | +| Depth endpoint unavailable or forbidden | Every depth counts as one; all queues poll with one-slot requests at backoff pace; busy queues grow one slot per tick as they fill; growth stays resource-gated | +| Depth request slower than the freshness window | Depths count as unknown; behavior as for an unavailable endpoint until the server recovers or the freshness is raised | +| Queue with a resolved domain | Named `domain:taskType` in the same request; its depth is the domain queue's, not the default queue's | +| Paused worker with a deep queue | Its share is its held slots; it takes no new process slots and sends no heartbeat while paused | +| Empty response with a long-poll timeout at or above the computed backoff | Delay is measured from poll start, so no second wait is added | +| Poll failure or timeout | Request-owned slots are released once; backoff and demand as for an empty response | +| Configured poll timeout above 5000 ms | Rejected at configuration time | +| Update-and-fetch | Slots transfer to the replacement or are released if none returns; the replacement is scheduled for lease extension and counts as a delivery | +| Update-and-fetch while paused, over share, or with a needier queue waiting for a process slot | Ordinary update used | +| Worker fails after a successful update-and-fetch transfer | The second report uses the plain update; no second replacement is requested; the failing invocation does not release the transferred slots | +| Virtual threads opted in | Concurrency equals admitted slots with no pool resizing; each task runs on its own virtual thread; a target reduction stops admission without cancelling running tasks | +| Virtual threads with blocking I/O under `synchronized` on a pre-24 JDK | Carriers pin and throughput plateaus; CPU and demand signals stop growth rather than oversubscribing; documented as an operational limit | +| Platform threads by default | Pool follows the target lazily; idle threads retire after 60 seconds; thread count never exceeds the worker target or held slots | +| Several processes consuming one queue | Depth is shared, so each process's demand overstates its own delivery; targets run ahead of delivered work and decay returns them; local decisions coexist with server dispatch | +| Adaptive mode disabled | Existing fixed-mode behavior remains available, including N × threadCount capacity for poller count N | +| Shutdown or worker removal | No new admission; existing lifecycle drains accepted work | + +Load tests cover CPU-heavy, blocking-I/O, long-running, allocation-heavy, and mixed queues, each run with platform threads and with virtual threads. Compare completion throughput, queue drain time, CPU and post-GC heap utilization, poll request rate, result-update latency, and task timeouts against representative fixed-concurrency settings. The allocation-heavy case must show that used heap crossing 90% between collections does not trigger a pause. + +Acceptance requires bounded slot ownership, growth under sustained demand, shares that follow depth under contention, backoff and recovery under resource pressure, and compatibility with existing server APIs. The numeric defaults remain subject to review and calibration; no claim of optimal throughput or protection against all memory exhaustion is made. + +## Deferred work + +- Sublinear weighting, such as the square root of demand, if load tests show that proportional shares leave shallow queues with too little when one queue is very deep. +- Removal of the deprecation marker on `GET /tasks/queue/sizes` and a documented guarantee that it sizes any queue name it is given, including `domain:taskType`, since this design depends on both. +- Server side: make the Postgres queue size query count only waiting messages, as its all-queues query already does, so depth means the same thing on every backend. +- Throughput comparison probe. A probe that withdraws growth which did not raise throughput can help when the bottleneck is a downstream dependency that CPU and heap do not reflect. If added, the pass threshold must be relative to the increment, the sample requirement must beat task-duration variance, and the probe must not hold a queue's target for the measurement window. +- Attribution of pressure to specific queues, so that a reduction can target the queue consuming resources rather than cutting every queue by the same factor. +- Container-aware memory, native process memory, and direct-buffer accounting. +- An opt-in explicit collection request when paused and idle, for operators who prefer it to the drained-resume path. +- A retreat to initial targets when metrics stay stale, if load testing shows that freezing growth alone is insufficient protection against a failed sampler. + +## Code and API references + +- [SDK runner configuration](https://github.com/conductor-oss/java-sdk/blob/9e335df4fcd214c0331910dde842a9cf3bad7c36/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunnerConfigurer.java) +- [SDK polling, execution, and task updates](https://github.com/conductor-oss/java-sdk/blob/9e335df4fcd214c0331910dde842a9cf3bad7c36/conductor-client/src/main/java/com/netflix/conductor/client/automator/TaskRunner.java) +- [SDK task HTTP client](https://github.com/conductor-oss/java-sdk/blob/9e335df4fcd214c0331910dde842a9cf3bad7c36/conductor-client/src/main/java/com/netflix/conductor/client/http/TaskClient.java) +- [SDK per-worker property resolution](https://github.com/conductor-oss/java-sdk/blob/9e335df4fcd214c0331910dde842a9cf3bad7c36/conductor-client/src/main/java/com/netflix/conductor/client/config/PropertyFactory.java) +- [SDK annotated worker registration and poller count](https://github.com/conductor-oss/java-sdk/blob/9e335df4fcd214c0331910dde842a9cf3bad7c36/conductor-client/src/main/java/com/netflix/conductor/sdk/workflow/executor/task/AnnotatedWorkerExecutor.java) +- [SDK Prometheus metrics collector](https://github.com/conductor-oss/java-sdk/blob/9e335df4fcd214c0331910dde842a9cf3bad7c36/conductor-client-metrics/src/main/java/com/netflix/conductor/client/metrics/prometheus/PrometheusMetricsCollector.java) +- [SDK load-test harness](https://github.com/conductor-oss/java-sdk/blob/9e335df4fcd214c0331910dde842a9cf3bad7c36/harness/src/main/java/io/orkes/conductor/harness/HarnessMain.java) +- [Conductor task resource endpoints](https://github.com/conductor-oss/conductor/blob/main/rest/src/main/java/com/netflix/conductor/rest/controllers/TaskResource.java) +- [Conductor polling and dispatch eligibility](https://github.com/conductor-oss/conductor/blob/main/core/src/main/java/com/netflix/conductor/service/ExecutionService.java) +- [Conductor Postgres queue long-poll semantics](https://github.com/conductor-oss/conductor/blob/main/postgres-persistence/src/main/java/com/netflix/conductor/postgres/dao/PostgresQueueDAO.java) +- [JVM process CPU measurements](https://docs.oracle.com/en/java/javase/21/docs/api/jdk.management/com/sun/management/OperatingSystemMXBean.html) +- [JVM heap measurements](https://docs.oracle.com/en/java/javase/21/docs/api/java.management/java/lang/management/MemoryMXBean.html) +- [JVM post-collection pool usage](https://docs.oracle.com/en/java/javase/21/docs/api/java.management/java/lang/management/MemoryPoolMXBean.html#getCollectionUsage()) +- [JVM collection notifications](https://docs.oracle.com/en/java/javase/21/docs/api/jdk.management/com/sun/management/GarbageCollectionNotificationInfo.html) +- [Resizable execution pools](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/util/concurrent/ThreadPoolExecutor.html) +- [Thread-per-task executors and virtual threads](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/util/concurrent/Executors.html#newThreadPerTaskExecutor(java.util.concurrent.ThreadFactory)) +- [JEP 444: Virtual Threads](https://openjdk.org/jeps/444) +- [JEP 491: Synchronize Virtual Threads without Pinning](https://openjdk.org/jeps/491) diff --git a/harness/README.md b/harness/README.md index 9a5ecdd80..5aade0144 100644 --- a/harness/README.md +++ b/harness/README.md @@ -67,6 +67,13 @@ kubectl rollout status deployment/java-sdk-harness-worker -n $NS | `HARNESS_WORKFLOWS_PER_SEC` | no | 2 | Workflows to start per second | | `HARNESS_BATCH_SIZE` | no | 20 | Thread count per worker (controls polling concurrency) | | `HARNESS_POLL_INTERVAL_MS` | no | 100 | Milliseconds between poll cycles | +| `HARNESS_ADAPTIVE` | no | false | Use adaptive concurrency instead of `HARNESS_BATCH_SIZE` threads per worker | +| `HARNESS_ADAPTIVE_MAX_PROCESS_CONCURRENCY` | no | 256 | Adaptive concurrency ceiling for the process | +| `HARNESS_ADAPTIVE_INITIAL_PROCESS_CONCURRENCY` | no | 0 | Adaptive starting target; 0 uses min(task types, processors) | +| `HARNESS_ADAPTIVE_MIN_PROCESS_CONCURRENCY` | no | 1 | Target an idle process decays to, so bursts do not start from one slot | +| `HARNESS_ADAPTIVE_STANDBY_POLL_TIMEOUT_MS` | no | 1000 | Server wait of the long poll each queue keeps open on spare capacity; 0 disables it | +| `HARNESS_ADAPTIVE_MAX_WORKER_CONCURRENCY` | no | 64 | Adaptive concurrency ceiling per worker | +| `HARNESS_VIRTUAL_THREADS` | no | false | Run tasks on virtual threads | ## Kubernetes diff --git a/harness/build.gradle b/harness/build.gradle index 0ddc2f181..a4466b9b4 100644 --- a/harness/build.gradle +++ b/harness/build.gradle @@ -4,7 +4,8 @@ plugins { } application { - mainClass = 'io.orkes.conductor.harness.HarnessMain' + // Override with -PharnessMain=, e.g. io.orkes.conductor.harness.ControllerOverheadMain + mainClass = providers.gradleProperty('harnessMain').getOrElse('io.orkes.conductor.harness.HarnessMain') } repositories { diff --git a/harness/e2e/.gitignore b/harness/e2e/.gitignore new file mode 100644 index 000000000..89f9ac04a --- /dev/null +++ b/harness/e2e/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/harness/e2e/README.md b/harness/e2e/README.md new file mode 100644 index 000000000..8881e91a9 --- /dev/null +++ b/harness/e2e/README.md @@ -0,0 +1,53 @@ +# Adaptive concurrency end-to-end tests + +End-to-end tests for adaptive worker concurrency against a real OSS Conductor server. Worker containers run with +the same limits as `harness/manifests/deployment.yaml` unless a scenario says otherwise, so the JVM picks its +production defaults (SerialGC and a 123 MB heap at 512Mi). + +Conductor runs on the host, started by `server.sh` with the Conductor CLI (`conductor server start`, OSS +the CLI's latest release unless `E2E_SERVER_VERSION` names one, SQLite). Each run starts the server in its own output directory, so it gets an +empty database, and stops it at the end. Workers and the downstream simulator run in Docker and reach the server +through `host.docker.internal`. + +Requirements: the Conductor CLI, Docker and JDK 21. Ports 8080 (server), 18090 (simulator) and 9991–9993 (worker +metrics) must be free. + +## Functional scenarios + +```shell +harness/e2e/run-functional.sh # all scenarios, about 30 minutes +harness/e2e/run-functional.sh weights heap # selected scenarios +``` + +One worker process polls four task types (`fn_a`..`fn_d`), one `TaskRunnerConfigurer` per type sharing one +controller. The driver loads each scenario's queues before the worker starts polling, samples the server's queue +sizes and the worker's metrics once per second, and checks the outcome against the design's rules. + +| Scenario | Setup | Checks | +|---|---|---| +| `weights` | Process target fixed at 16; backlogs 600/300/150/30 of 1 s tasks | Targets match the spec's water-filled shares; executions follow targets; no slot unassigned while demand exceeds the target; throughput while all queues are backlogged splits 9/4/2/1 | +| `rotation` | Process target 2; four queues of 40 tasks | Every queue starts a task in every 15 s window; concurrency never exceeds 2; equal backlogs finish together | +| `starvation` | Process target 4; three backlogs of 2 s tasks plus a domain-routed task every 3 s | Trickle tasks wait at most 5 s and never fall back out of their domain; heavy queues keep progressing | +| `scaling` | 1500 tasks of 2 s I/O on one queue, then 4 minutes idle | Growth steps of at most 20% (min 1, max 16) per control tick up to the ceiling; threads track the target; capacity is used; the target halves at most every 30 s down to one standby slot per queue; idle threads are released | +| `warmstart` | Initial target 32, minimum 16; 1500 tasks, idle, then a 200-task burst | Starts at 32 without a ramp; reaches the ceiling; idle decay stops at 16; idle threads are released; the burst after idle starts at least 16 tasks within 3 s | +| `cpu` | One CPU, 250 ms CPU-bound tasks, starting at target 8 | Reductions of 25% at least 15 s apart, no growth while CPU is saturated, the target settles at 1–2, throughput near one core | +| `heap` | Tasks holding 2–30 MB payloads for 3 s in a 123 MB heap | No `OutOfMemoryError` and every task completes; no poll and no task start while admission is paused | + +Every scenario also checks that all workflows completed, no task executed twice, and all slots were released. +Executions are recorded by the workers themselves (`logs/executions-*.csv`), independently of the SDK's metrics. + +## Mixed workload + +```shell +harness/e2e/run.sh adaptive # or: fixed, for tuned fixed thread counts +``` + +Two worker pods, four tasks shared unevenly by four workflows (dynamic fork fan-out, CPU-heavy JSON scoring, a +bounded connection pool, and a domain-routed long-running task with callbacks), a downstream simulator with limited +capacity, and phases for warmup, steady load, a 1000-workflow burst, a 5x downstream slowdown and idle. Phase lengths, +rates and sizes are environment variables read by `E2eDriver`. + +## Output + +Each run writes `out//`: `report.json` with every check and observation, `timeseries.csv` with the one-second +samples, the workers' execution logs, and the server and worker logs. diff --git a/harness/e2e/processes.sh b/harness/e2e/processes.sh new file mode 100755 index 000000000..91a526fd6 --- /dev/null +++ b/harness/e2e/processes.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Workers and the downstream simulator for the end-to-end tests, as JVMs on the host. Sourced by run.sh and +# run-functional.sh after server.sh. +# +# A worker JVM is started with -XX:ActiveProcessorCount and -XX:MaxRAM so the JVM and the SDK see the resources of the +# pod in harness/manifests/deployment.yaml: one CPU, 512Mi, hence SerialGC and a heap of about 123 MB. The host does not +# cap the CPU the process actually uses; the SDK reads CPU relative to one core and counts anything above it as full. + +HARNESS_LIB="$(cd "$(dirname "${BASH_SOURCE[0]}")/../build/install/harness/lib" && pwd)" +PIDS=() + +# start_jvm

: the caller's environment is passed through. +start_jvm() { + local name="$1" log="$2" cpus="$3" memory="$4" main="$5" + HOSTNAME="$name" java -XX:ActiveProcessorCount="$cpus" -XX:MaxRAM="$memory" -cp "$HARNESS_LIB/*" "$main" \ + > "$log" 2>&1 & + PIDS+=("$!") + eval "PID_${name//-/_}=$!" +} + +start_simulator() { + local log="$1" + SIM_PORT=18090 java -cp "$HARNESS_LIB/*" io.orkes.conductor.harness.e2e.DependencySimulator > "$log" 2>&1 & + PIDS+=("$!") + for _ in $(seq 1 30); do + curl -s -o /dev/null http://localhost:18090/stats && return 0 + sleep 1 + done + echo "Simulator did not start; see $log" >&2 + return 1 +} + +# process_status : one line with whether the JVM still runs and how often it ran out of memory. +process_status() { + local name="$1" log="$2" pid running=false oom + pid=$(eval "echo \${PID_${name//-/_}:-}") + [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null && running=true + oom=$(grep -c OutOfMemoryError "$log" 2>/dev/null) + echo "$name running=$running outOfMemory=${oom:-0}" +} + +# stop_jvm : stops one JVM started by start_jvm. +stop_jvm() { + local pid + pid=$(eval "echo \${PID_${1//-/_}:-}") + [ -n "$pid" ] && kill "$pid" 2>/dev/null && wait "$pid" 2>/dev/null + return 0 +} + +stop_processes() { + for pid in "${PIDS[@]:-}"; do + [ -n "$pid" ] && kill "$pid" 2>/dev/null + done + wait 2>/dev/null + PIDS=() +} diff --git a/harness/e2e/run-functional.sh b/harness/e2e/run-functional.sh new file mode 100755 index 000000000..850d7ece9 --- /dev/null +++ b/harness/e2e/run-functional.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Functional scenarios for adaptive concurrency against a local OSS Conductor started with the Conductor CLI. +# Usage: harness/e2e/run-functional.sh [scenario ...] (default: all) +set -uo pipefail +cd "$(dirname "$0")" +source ./server.sh +source ./processes.sh +SCENARIOS=("$@") +[ ${#SCENARIOS[@]} -eq 0 ] && SCENARIOS=(weights rotation starvation scaling warmstart cpu heap) +RUN="$PWD/out/functional-$(date +%Y%m%d-%H%M%S)" +mkdir -p "$RUN" + +(cd ../.. && ./gradlew -q :harness:installDist) || exit 1 +trap 'stop_processes; stop_server "$RUN/server"' EXIT +start_server "$RUN/server" || exit 1 + +overall=0 +for s in "${SCENARIOS[@]}"; do + # Worker settings per scenario: process ceiling, initial target, CPUs and memory the JVM sees, domains. + export FN_CPUS=2 FN_MEMORY=512m E2E_MAX_PROCESS_CONCURRENCY=64 E2E_INITIAL_PROCESS_CONCURRENCY=0 \ + E2E_MAX_WORKER_CONCURRENCY=64 E2E_TASK_DOMAINS="" E2E_MIN_PROCESS_CONCURRENCY=1 + case "$s" in + weights) E2E_MAX_PROCESS_CONCURRENCY=16; E2E_INITIAL_PROCESS_CONCURRENCY=16 ;; + rotation) E2E_MAX_PROCESS_CONCURRENCY=2; E2E_INITIAL_PROCESS_CONCURRENCY=2 ;; + starvation) E2E_MAX_PROCESS_CONCURRENCY=4; E2E_INITIAL_PROCESS_CONCURRENCY=4; E2E_TASK_DOMAINS="fn_d=partner_a" ;; + scaling) ;; + warmstart) E2E_INITIAL_PROCESS_CONCURRENCY=32; E2E_MIN_PROCESS_CONCURRENCY=16 ;; + cpu) FN_CPUS=1; E2E_INITIAL_PROCESS_CONCURRENCY=8 ;; + heap) ;; + esac + OUT="$RUN/$s"; export E2E_LOG_DIR_HOST="$OUT/logs"; mkdir -p "$E2E_LOG_DIR_HOST" + E2E_LOG_DIR="$E2E_LOG_DIR_HOST" CONDUCTOR_SERVER_URL=http://localhost:8080/api E2E_METRICS_PORT=9993 \ + start_jvm fn-worker "$OUT/worker.log" "$FN_CPUS" "$FN_MEMORY" io.orkes.conductor.harness.e2e.E2eFunctionalWorkerMain + E2E_SCENARIO="$s" E2E_OUT_DIR="$OUT" E2E_LOG_DIR="$E2E_LOG_DIR_HOST" CONDUCTOR_SERVER_URL=http://localhost:8080/api \ + E2E_WORKER_METRICS=http://localhost:9993 \ + java -cp "../build/install/harness/lib/*" io.orkes.conductor.harness.e2e.E2eFunctionalDriver > "$OUT/driver.log" 2>&1 + status=$? + [ $status -ne 0 ] && overall=1 + process_status fn-worker "$OUT/worker.log" > "$OUT/worker.txt" + stop_jvm fn-worker + sed -n '/==== scenario/,$p' "$OUT/driver.log" + echo "worker: $(cat "$OUT/worker.txt")" +done +echo "Output: $RUN" +exit $overall diff --git a/harness/e2e/run.sh b/harness/e2e/run.sh new file mode 100755 index 000000000..ad4796b53 --- /dev/null +++ b/harness/e2e/run.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Runs the adaptive concurrency end-to-end test against a local OSS Conductor started with the Conductor CLI. +# Usage: harness/e2e/run.sh [adaptive|fixed] +set -euo pipefail +cd "$(dirname "$0")" +source ./server.sh +source ./processes.sh +export E2E_MODE="${1:-adaptive}" +OUT="$PWD/out/$E2E_MODE-$(date +%Y%m%d-%H%M%S)" +export E2E_LOG_DIR_HOST="$OUT/logs" +mkdir -p "$E2E_LOG_DIR_HOST" + +(cd ../.. && ./gradlew -q :harness:installDist) +trap 'stop_processes; stop_server "$OUT/server"' EXIT +start_server "$OUT/server" +start_simulator "$OUT/simulator.log" +export CONDUCTOR_SERVER_URL=http://localhost:8080/api SIM_URL=http://localhost:18090 E2E_LOG_DIR="$E2E_LOG_DIR_HOST" +E2E_METRICS_PORT=9991 start_jvm worker-0 "$OUT/worker-0.log" 1 512m io.orkes.conductor.harness.e2e.E2eWorkerMain +E2E_METRICS_PORT=9992 start_jvm worker-1 "$OUT/worker-1.log" 1 512m io.orkes.conductor.harness.e2e.E2eWorkerMain + +# A failed check must not end the script before the output is collected. +set +e +E2E_OUT_DIR="$OUT" java -cp "../build/install/harness/lib/*" io.orkes.conductor.harness.e2e.E2eDriver | tee "$OUT/driver.log" +status=${PIPESTATUS[0]} +set -e + +for w in worker-0 worker-1; do + process_status "$w" "$OUT/$w.log" | tee -a "$OUT/workers.txt" +done +echo "Output: $OUT" +exit "$status" diff --git a/harness/e2e/server.sh b/harness/e2e/server.sh new file mode 100755 index 000000000..ab53d8054 --- /dev/null +++ b/harness/e2e/server.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Local OSS Conductor for the end-to-end tests, started with the Conductor CLI (`conductor server start`). +# The server keeps its SQLite database in the directory it starts from, so every run gets an empty one. +# Sourced by run.sh and run-functional.sh. The CLI's latest server runs unless E2E_SERVER_VERSION names a release. + +E2E_SERVER_VERSION="${E2E_SERVER_VERSION:-latest}" + +# start_server : stops any CLI-managed server, then starts a fresh one with its database in . +start_server() { + local dir="$1" + mkdir -p "$dir" + conductor server stop >/dev/null 2>&1 || true + (cd "$dir" && conductor server start --version "$E2E_SERVER_VERSION" > "$dir/start.log" 2>&1) || { + cat "$dir/start.log"; return 1; } + for _ in $(seq 1 90); do + curl -sf http://localhost:8080/health >/dev/null && return 0 + sleep 2 + done + echo "Conductor did not become healthy; see ~/.conductor-cli/server/conductor.log" >&2 + return 1 +} + +# stop_server : stops the server and keeps its log in . +stop_server() { + local dir="$1" + cp ~/.conductor-cli/server/conductor.log "$dir/conductor.log" 2>/dev/null || true + conductor server stop >/dev/null 2>&1 || true +} diff --git a/harness/src/main/java/io/orkes/conductor/harness/ControllerOverheadMain.java b/harness/src/main/java/io/orkes/conductor/harness/ControllerOverheadMain.java new file mode 100644 index 000000000..5dba1b2b7 --- /dev/null +++ b/harness/src/main/java/io/orkes/conductor/harness/ControllerOverheadMain.java @@ -0,0 +1,312 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.orkes.conductor.harness; + +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.conductor.client.automator.AdaptiveConcurrencyConfig; +import com.netflix.conductor.client.automator.AdaptiveConcurrencyController; +import com.netflix.conductor.client.automator.TaskRunnerConfigurer; +import com.netflix.conductor.client.events.taskrunner.PollStarted; +import com.netflix.conductor.client.http.ConductorClient; +import com.netflix.conductor.client.http.MetadataClient; +import com.netflix.conductor.client.http.TaskClient; +import com.netflix.conductor.client.http.WorkflowClient; +import com.netflix.conductor.client.worker.Worker; +import com.netflix.conductor.common.metadata.tasks.Task; +import com.netflix.conductor.common.metadata.tasks.TaskDef; +import com.netflix.conductor.common.metadata.tasks.TaskResult; +import com.netflix.conductor.common.metadata.tasks.TaskType; +import com.netflix.conductor.common.metadata.workflow.WorkflowDef; +import com.netflix.conductor.common.metadata.workflow.WorkflowTask; + +import io.orkes.conductor.client.ApiClient; + +/** + * Measures adaptive-controller overhead with many registered queues: a handful of busy workers fed by a workflow + * governor, plus hundreds of idle workers whose queues never receive tasks. Reports, per window, the latency of + * acquiring the controller lock (probed with {@code getHeldSlots()}), CPU time by thread group, poll request rate + * and the controller's process target and held slots. + * + *

Environment: {@code OVERHEAD_IDLE_QUEUES} (default 300), {@code OVERHEAD_BUSY_QUEUES} (5), + * {@code OVERHEAD_WORKFLOWS_PER_SEC} (10), {@code OVERHEAD_TASK_SLEEP_MS} (200), {@code OVERHEAD_POLL_INTERVAL_MS} + * (100), {@code OVERHEAD_STANDBY_POLL_TIMEOUT_MS} (0 = standby polls off, the heaviest waiter churn), + * {@code OVERHEAD_DURATION_SEC} (180), {@code OVERHEAD_REPORT_SEC} (30), and the usual + * {@code CONDUCTOR_SERVER_URL} and auth variables. + */ +public class ControllerOverheadMain { + + private static final Logger log = LoggerFactory.getLogger(ControllerOverheadMain.class); + + private static final String WORKFLOW_NAME = "overhead_controller_workflow"; + private static final String BUSY_PREFIX = "overhead_busy_"; + private static final String IDLE_PREFIX = "overhead_idle_"; + + public static void main(String[] args) throws Exception { + int idleQueues = envInt("OVERHEAD_IDLE_QUEUES", 300); + int busyQueues = envInt("OVERHEAD_BUSY_QUEUES", 5); + int workflowsPerSec = envInt("OVERHEAD_WORKFLOWS_PER_SEC", 10); + int taskSleepMs = envInt("OVERHEAD_TASK_SLEEP_MS", 200); + int pollIntervalMs = envInt("OVERHEAD_POLL_INTERVAL_MS", 100); + int standbyMs = envInt("OVERHEAD_STANDBY_POLL_TIMEOUT_MS", 0); + int durationSec = envInt("OVERHEAD_DURATION_SEC", 180); + int reportSec = envInt("OVERHEAD_REPORT_SEC", 30); + + ConductorClient client = ApiClient.builder() + .useEnvVariables(true) + .readTimeout(10_000) + .connectTimeout(10_000) + .writeTimeout(10_000) + .build(); + + registerMetadata(client, busyQueues, taskSleepMs); + + List workers = new ArrayList<>(); + for (int i = 0; i < busyQueues; i++) { + workers.add(new OverheadWorker(BUSY_PREFIX + i, pollIntervalMs, taskSleepMs)); + } + for (int i = 0; i < idleQueues; i++) { + workers.add(new OverheadWorker(IDLE_PREFIX + i, pollIntervalMs, 0)); + } + + var adaptiveConfig = new AdaptiveConcurrencyConfig.Builder() + .withMaxProcessConcurrency(envInt("OVERHEAD_MAX_PROCESS_CONCURRENCY", 256)) + .withDefaultMaxWorkerConcurrency(envInt("OVERHEAD_MAX_WORKER_CONCURRENCY", 64)) + .withStandbyPollTimeout(Duration.ofMillis(standbyMs)) + .build(); + AdaptiveConcurrencyController controller = new AdaptiveConcurrencyController(adaptiveConfig); + + AtomicLong pollsStarted = new AtomicLong(); + TaskRunnerConfigurer configurer = new TaskRunnerConfigurer.Builder(new TaskClient(client), workers) + .withAdaptiveConcurrency(controller) + .withListener(PollStarted.class, e -> pollsStarted.incrementAndGet()) + .build(); + + WorkflowGovernor governor = new WorkflowGovernor(new WorkflowClient(client), WORKFLOW_NAME, + workflowsPerSec, null); + + LockProbe lockProbe = new LockProbe(controller); + lockProbe.start(); + configurer.init(); + governor.start(); + log.info("[overhead] running: {} idle + {} busy queues, {} wf/s x {} tasks of {} ms, standby={} ms, {} s", + idleQueues, busyQueues, workflowsPerSec, busyQueues, taskSleepMs, standbyMs, durationSec); + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + governor.shutdown(); + configurer.shutdown(); + })); + + CpuByGroup cpu = new CpuByGroup(); + long endAt = System.nanoTime() + TimeUnit.SECONDS.toNanos(durationSec); + List allWindows = new ArrayList<>(); + cpu.snapshot(); // establish the baseline + long lastPolls = pollsStarted.get(); + while (System.nanoTime() < endAt) { + Thread.sleep(TimeUnit.SECONDS.toMillis(reportSec)); + long[] probe = lockProbe.drainSorted(); + allWindows.add(probe); + long polls = pollsStarted.get(); + double pollRate = (double) (polls - lastPolls) / reportSec; + lastPolls = polls; + Map cpuMs = cpu.snapshot(); + log.info("[overhead] target={} held={} polls/s={} lockProbe us p50={} p99={} max={} (n={}) " + + "cpu ms/s: pollers={} controller-timers={} exec={} other={} process={}", + controller.getTargetConcurrency(), controller.getHeldSlots(), + String.format("%.1f", pollRate), + percentileUs(probe, 0.50), percentileUs(probe, 0.99), percentileUs(probe, 1.0), probe.length, + perSec(cpuMs.get("pollers"), reportSec), perSec(cpuMs.get("controller"), reportSec), + perSec(cpuMs.get("exec"), reportSec), perSec(cpuMs.get("other"), reportSec), + perSec(cpuMs.get("process"), reportSec)); + } + + long[] all = allWindows.stream().flatMapToLong(Arrays::stream).sorted().toArray(); + log.info("[overhead] SUMMARY queues={} lock probe us: p50={} p90={} p99={} p99.9={} max={} (n={})", + idleQueues + busyQueues, percentileUs(all, 0.50), percentileUs(all, 0.90), + percentileUs(all, 0.99), percentileUs(all, 0.999), percentileUs(all, 1.0), all.length); + System.exit(0); + } + + /** Times the acquisition of the controller lock every 10 ms via a public getter that takes it. */ + private static final class LockProbe extends Thread { + private final AdaptiveConcurrencyController controller; + private final List samples = new ArrayList<>(); + + LockProbe(AdaptiveConcurrencyController controller) { + super("overhead-lock-probe"); + this.controller = controller; + setDaemon(true); + } + + @Override + public void run() { + while (true) { + long t0 = System.nanoTime(); + controller.getHeldSlots(); + long dt = System.nanoTime() - t0; + synchronized (samples) { + samples.add(dt); + } + try { + Thread.sleep(10); + } catch (InterruptedException e) { + return; + } + } + } + + long[] drainSorted() { + long[] out; + synchronized (samples) { + out = samples.stream().mapToLong(Long::longValue).toArray(); + samples.clear(); + } + Arrays.sort(out); + return out; + } + } + + /** CPU time deltas bucketed by thread group plus whole-process CPU. */ + private static final class CpuByGroup { + private final ThreadMXBean threads = ManagementFactory.getThreadMXBean(); + private final com.sun.management.OperatingSystemMXBean os = + (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); + private final Map lastCpuByThread = new HashMap<>(); + private long lastProcessCpu; + + Map snapshot() { + Map deltaMs = new LinkedHashMap<>( + Map.of("pollers", 0L, "controller", 0L, "exec", 0L, "other", 0L)); + for (long id : threads.getAllThreadIds()) { + ThreadInfo info = threads.getThreadInfo(id); + long cpu = threads.getThreadCpuTime(id); + if (info == null || cpu < 0) { + continue; + } + long delta = cpu - lastCpuByThread.getOrDefault(id, 0L); + lastCpuByThread.put(id, cpu); + String name = info.getThreadName(); + String group = name.startsWith("TaskRunner") ? "pollers" + : name.startsWith("adaptive-concurrency") ? "controller" + : name.startsWith("workflow-worker") ? "exec" + : "other"; + deltaMs.merge(group, TimeUnit.NANOSECONDS.toMillis(delta), Long::sum); + } + long processCpu = os.getProcessCpuTime(); + deltaMs.put("process", TimeUnit.NANOSECONDS.toMillis(processCpu - lastProcessCpu)); + lastProcessCpu = processCpu; + return deltaMs; + } + } + + private static String perSec(Long totalMs, int windowSec) { + return String.format("%.1f", totalMs == null ? 0 : (double) totalMs / windowSec); + } + + private static long percentileUs(long[] sorted, double q) { + if (sorted.length == 0) { + return -1; + } + int index = Math.min(sorted.length - 1, (int) Math.ceil(q * sorted.length) - 1); + return TimeUnit.NANOSECONDS.toMicros(sorted[Math.max(0, index)]); + } + + private static final class OverheadWorker implements Worker { + private static final AtomicInteger EXECUTIONS = new AtomicInteger(); + private final String taskName; + private final int pollIntervalMs; + private final int sleepMs; + + OverheadWorker(String taskName, int pollIntervalMs, int sleepMs) { + this.taskName = taskName; + this.pollIntervalMs = pollIntervalMs; + this.sleepMs = sleepMs; + } + + @Override + public String getTaskDefName() { + return taskName; + } + + @Override + public int getPollingInterval() { + return pollIntervalMs; + } + + @Override + public TaskResult execute(Task task) { + if (sleepMs > 0) { + try { + TimeUnit.MILLISECONDS.sleep(sleepMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + EXECUTIONS.incrementAndGet(); + TaskResult result = new TaskResult(task); + result.setStatus(TaskResult.Status.COMPLETED); + return result; + } + } + + private static void registerMetadata(ConductorClient client, int busyQueues, int taskSleepMs) { + MetadataClient metadataClient = new MetadataClient(client); + List taskDefs = new ArrayList<>(); + List workflowTasks = new ArrayList<>(); + for (int i = 0; i < busyQueues; i++) { + String name = BUSY_PREFIX + i; + TaskDef taskDef = new TaskDef(name); + taskDef.setDescription("Controller overhead busy task (" + taskSleepMs + " ms)"); + taskDef.setRetryCount(0); + taskDef.setTimeoutSeconds(300); + taskDef.setResponseTimeoutSeconds(120); + taskDefs.add(taskDef); + + WorkflowTask workflowTask = new WorkflowTask(); + workflowTask.setName(name); + workflowTask.setTaskReferenceName("ref_" + name); + workflowTask.setType(TaskType.SIMPLE.name()); + workflowTasks.add(workflowTask); + } + metadataClient.registerTaskDefs(taskDefs); + + WorkflowDef workflowDef = new WorkflowDef(); + workflowDef.setName(WORKFLOW_NAME); + workflowDef.setVersion(1); + workflowDef.setDescription("Controller overhead measurement workflow"); + workflowDef.setOwnerEmail("java-sdk-harness@conductor.io"); + workflowDef.setTasks(workflowTasks); + metadataClient.updateWorkflowDefs(List.of(workflowDef)); + log.info("[overhead] registered {} busy task defs and workflow {}", busyQueues, WORKFLOW_NAME); + } + + private static int envInt(String name, int defaultValue) { + String value = System.getenv(name); + return value == null || value.isBlank() ? defaultValue : Integer.parseInt(value.trim()); + } +} diff --git a/harness/src/main/java/io/orkes/conductor/harness/HarnessMain.java b/harness/src/main/java/io/orkes/conductor/harness/HarnessMain.java index 5cd12d3c6..8b6873fa5 100644 --- a/harness/src/main/java/io/orkes/conductor/harness/HarnessMain.java +++ b/harness/src/main/java/io/orkes/conductor/harness/HarnessMain.java @@ -12,6 +12,7 @@ */ package io.orkes.conductor.harness; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -21,6 +22,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.netflix.conductor.client.automator.AdaptiveConcurrencyConfig; +import com.netflix.conductor.client.automator.AdaptiveConcurrencyController; import com.netflix.conductor.client.automator.TaskRunnerConfigurer; import com.netflix.conductor.client.http.ConductorClient; import com.netflix.conductor.client.http.MetadataClient; @@ -57,6 +60,8 @@ public static void main(String[] args) throws Exception { int pollIntervalMs = envInt("HARNESS_POLL_INTERVAL_MS", 100); int metricsPort = envInt("HARNESS_METRICS_PORT", 9991); int probeRatePerSec = envInt("HARNESS_PROBE_RATE_PER_SEC", 0); + boolean adaptive = envBoolean("HARNESS_ADAPTIVE", false); + boolean virtualThreads = envBoolean("HARNESS_VIRTUAL_THREADS", false); List workers = new ArrayList<>(); for (String[] entry : SIMULATED_WORKERS) { @@ -84,10 +89,21 @@ public static void main(String[] args) throws Exception { TaskClient taskClient = new TaskClient(client); - TaskRunnerConfigurer configurer = - new TaskRunnerConfigurer.Builder(taskClient, workers) - .withTaskThreadCount(threadCounts) - .build(); + var configurerBuilder = new TaskRunnerConfigurer.Builder(taskClient, workers) + .withTaskThreadCount(threadCounts) + .withUseVirtualThreads(virtualThreads); + if (adaptive) { + var adaptiveConfig = new AdaptiveConcurrencyConfig.Builder() + .withMaxProcessConcurrency(envInt("HARNESS_ADAPTIVE_MAX_PROCESS_CONCURRENCY", 256)) + .withInitialProcessConcurrency(envInt("HARNESS_ADAPTIVE_INITIAL_PROCESS_CONCURRENCY", 0)) + .withMinProcessConcurrency(envInt("HARNESS_ADAPTIVE_MIN_PROCESS_CONCURRENCY", 1)) + .withStandbyPollTimeout(Duration.ofMillis(envInt("HARNESS_ADAPTIVE_STANDBY_POLL_TIMEOUT_MS", 1000))) + .withDefaultMaxWorkerConcurrency(envInt("HARNESS_ADAPTIVE_MAX_WORKER_CONCURRENCY", 64)) + .build(); + configurerBuilder.withAdaptiveConcurrency(new AdaptiveConcurrencyController(adaptiveConfig)); + log.info("Adaptive concurrency enabled with process ceiling {}", adaptiveConfig.getMaxProcessConcurrency()); + } + TaskRunnerConfigurer configurer = configurerBuilder.build(); if (!workersOnly) { try { @@ -168,6 +184,14 @@ private static void registerMetadata(ConductorClient client) { log.info("Registered workflow definition: {}", WORKFLOW_NAME); } + private static boolean envBoolean(String name, boolean defaultValue) { + String value = System.getenv(name); + if (value == null || value.isBlank()) { + return defaultValue; + } + return Boolean.parseBoolean(value.trim()); + } + private static int envInt(String name, int defaultValue) { String value = System.getenv(name); if (value == null || value.isBlank()) { diff --git a/harness/src/main/java/io/orkes/conductor/harness/e2e/DependencySimulator.java b/harness/src/main/java/io/orkes/conductor/harness/e2e/DependencySimulator.java new file mode 100644 index 000000000..a82be45c2 --- /dev/null +++ b/harness/src/main/java/io/orkes/conductor/harness/e2e/DependencySimulator.java @@ -0,0 +1,192 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.orkes.conductor.harness.e2e; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +/** + * A downstream service shared by every worker process. Each endpoint serves at most {@code capacity} requests at once; + * requests beyond that wait for a free server, so latency rises with load the way a real database or API does. + */ +public class DependencySimulator { + + private static final Logger log = LoggerFactory.getLogger(DependencySimulator.class); + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final Map endpoints = new LinkedHashMap<>(); + + public DependencySimulator(Map endpoints) { + this.endpoints.putAll(endpoints); + } + + public static void main(String[] args) throws IOException { + int port = E2eConfig.envInt("SIM_PORT", 8090); + Map endpoints = new LinkedHashMap<>(); + endpoints.put("profile", new Endpoint(E2eConfig.envInt("SIM_PROFILE_CAPACITY", 10), 100, 0.8)); + endpoints.put("persist", new Endpoint(E2eConfig.envInt("SIM_PERSIST_CAPACITY", 30), 20, 0.5)); + new DependencySimulator(endpoints).start(port); + } + + public void start(int port) throws IOException { + HttpServer server = HttpServer.create(new InetSocketAddress(port), 1024); + server.setExecutor(Executors.newVirtualThreadPerTaskExecutor()); + endpoints.forEach((name, endpoint) -> server.createContext("/" + name, exchange -> serve(exchange, endpoint))); + server.createContext("/stats", this::stats); + server.createContext("/control", this::control); + server.start(); + log.info("Dependency simulator listening on {} with endpoints {}", port, endpoints.keySet()); + } + + private void serve(HttpExchange exchange, Endpoint endpoint) throws IOException { + long startNanos = System.nanoTime(); + try { + endpoint.handle(); + long latencyMicros = (System.nanoTime() - startNanos) / 1_000; + respond(exchange, 200, "{\"latencyMicros\":" + latencyMicros + "}"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + respond(exchange, 503, "{}"); + } + } + + private void stats(HttpExchange exchange) throws IOException { + Map body = new LinkedHashMap<>(); + endpoints.forEach((name, endpoint) -> body.put(name, endpoint.snapshot())); + respond(exchange, 200, MAPPER.writeValueAsString(body)); + } + + private void control(HttpExchange exchange) throws IOException { + Map query = query(exchange.getRequestURI()); + Endpoint endpoint = endpoints.get(query.get("endpoint")); + if (endpoint == null || !query.containsKey("latencyMultiplier")) { + respond(exchange, 400, "{\"error\":\"endpoint and latencyMultiplier are required\"}"); + return; + } + endpoint.latencyMultiplier = Double.parseDouble(query.get("latencyMultiplier")); + log.info("Endpoint {} latency multiplier set to {}", query.get("endpoint"), endpoint.latencyMultiplier); + respond(exchange, 200, "{}"); + } + + private static Map query(URI uri) { + Map values = new HashMap<>(); + if (uri.getQuery() == null) { + return values; + } + for (String pair : uri.getQuery().split("&")) { + int separator = pair.indexOf('='); + if (separator > 0) { + values.put(pair.substring(0, separator), pair.substring(separator + 1)); + } + } + return values; + } + + private static void respond(HttpExchange exchange, int status, String body) throws IOException { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, bytes.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(bytes); + } + } + + /** A service with {@code capacity} servers and log-normal service time. */ + static class Endpoint { + + private final int capacity; + private final Semaphore servers; + private final double medianMillis; + private final double sigma; + private final AtomicInteger inFlight = new AtomicInteger(); + private final AtomicInteger maxInFlight = new AtomicInteger(); + private final AtomicLong requests = new AtomicLong(); + private final List windowLatencyMicros = Collections.synchronizedList(new ArrayList<>()); + private volatile double latencyMultiplier = 1.0; + + Endpoint(int capacity, double medianMillis, double sigma) { + this.capacity = capacity; + this.servers = new Semaphore(capacity, true); + this.medianMillis = medianMillis; + this.sigma = sigma; + } + + void handle() throws InterruptedException { + long startNanos = System.nanoTime(); + int current = inFlight.incrementAndGet(); + maxInFlight.accumulateAndGet(current, Math::max); + try { + servers.acquire(); + try { + double serviceMillis = medianMillis * latencyMultiplier + * Math.exp(sigma * ThreadLocalRandom.current().nextGaussian()); + Thread.sleep((long) serviceMillis, (int) ((serviceMillis % 1) * 1_000_000)); + } finally { + servers.release(); + } + } finally { + inFlight.decrementAndGet(); + requests.incrementAndGet(); + windowLatencyMicros.add((System.nanoTime() - startNanos) / 1_000); + } + } + + /** Returns counters and the latency distribution since the previous snapshot. */ + Map snapshot() { + List latencies; + synchronized (windowLatencyMicros) { + latencies = new ArrayList<>(windowLatencyMicros); + windowLatencyMicros.clear(); + } + Collections.sort(latencies); + Map values = new LinkedHashMap<>(); + values.put("capacity", capacity); + values.put("requests", requests.get()); + values.put("inFlight", inFlight.get()); + values.put("maxInFlight", maxInFlight.getAndSet(inFlight.get())); + values.put("latencyMultiplier", latencyMultiplier); + values.put("p50Millis", percentile(latencies, 0.50) / 1000.0); + values.put("p99Millis", percentile(latencies, 0.99) / 1000.0); + return values; + } + + private static long percentile(List sorted, double quantile) { + if (sorted.isEmpty()) { + return 0; + } + return sorted.get(Math.min(sorted.size() - 1, (int) Math.ceil(quantile * sorted.size()) - 1)); + } + } +} diff --git a/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eAnalysis.java b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eAnalysis.java new file mode 100644 index 000000000..a5e11e6c5 --- /dev/null +++ b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eAnalysis.java @@ -0,0 +1,418 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.orkes.conductor.harness.e2e; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.function.ToDoubleFunction; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.netflix.conductor.common.metadata.tasks.Task; +import com.netflix.conductor.common.run.Workflow; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +/** + * Turns the run's raw data into pass/fail checks and a report. Correctness comes from the server's task records and + * the workers' own execution logs; behavior comes from the one-second samples. + */ +class E2eAnalysis { + + private static final List LOAD_PHASES = List.of("warmup", "steady", "burst", "slowdown"); + + private final List started; + private final Map workflows; + private final List samples; + private final int workerCount; + private final Path logDir; + private final int startErrors; + private final boolean adaptive = "adaptive".equals(E2eConfig.env("E2E_MODE", "adaptive")); + private final int standbyMillis = E2eConfig.envInt("E2E_STANDBY_POLL_TIMEOUT_MS", 1000); + private final List> checks = new ArrayList<>(); + + E2eAnalysis(List started, Map workflows, + List samples, int workerCount, Path logDir, int startErrors) { + this.started = started; + this.workflows = workflows; + this.samples = samples; + this.workerCount = workerCount; + this.logDir = logDir; + this.startErrors = startErrors; + } + + /** One execution from a worker's log. */ + record Execution(String taskId, String taskType, long start, long end, String status, String worker) {} + + boolean writeReport(Path outDir) throws IOException { + Map report = new LinkedHashMap<>(); + report.put("mode", adaptive ? "adaptive" : "fixed"); + List executions = readExecutions(); + List tasks = workflows.values().stream() + .flatMap(workflow -> workflow.getTasks().stream()) + .filter(task -> E2eConfig.TASK_TYPES.contains(task.getTaskDefName())) + .toList(); + + checkCompletion(report); + checkExecutions(report, executions, tasks); + checkConcurrency(report, executions); + checkDomain(report, tasks); + checkSlotsReleased(report); + checkIdle(report); + report.put("queueWaitMillis", queueWaits(tasks)); + report.put("phases", phaseSummary()); + report.put("burstDrainSeconds", burstDrainSeconds()); + report.put("adjustmentReasons", reasons()); + report.put("checks", checks); + + ObjectMapper mapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT); + mapper.writeValue(outDir.resolve("report.json").toFile(), report); + boolean passed = checks.stream().allMatch(check -> Boolean.TRUE.equals(check.get("passed"))); + System.out.println(); + System.out.println("==== Adaptive concurrency end-to-end: " + report.get("mode") + " ===="); + Map summary = new LinkedHashMap<>(); + for (String key : List.of("workflowStatus", "queueWaitMillis", "phases", "burstDrainSeconds")) { + summary.put(key, report.get(key)); + } + System.out.println(mapper.writeValueAsString(summary)); + for (Map check : checks) { + System.out.printf("%-4s %-28s %s%n", Boolean.TRUE.equals(check.get("passed")) ? "PASS" : "FAIL", + check.get("name"), check.get("detail")); + } + System.out.println("Result: " + (passed ? "PASS" : "FAIL") + " (report: " + outDir.resolve("report.json") + + ")"); + return passed; + } + + private void check(String name, boolean passed, Object detail) { + Map check = new LinkedHashMap<>(); + check.put("name", name); + check.put("passed", passed); + check.put("detail", detail); + checks.add(check); + } + + private void checkCompletion(Map report) { + Map byStatus = workflows.values().stream() + .collect(Collectors.groupingBy(workflow -> workflow.getStatus().name(), TreeMap::new, + Collectors.counting())); + Map byName = started.stream() + .collect(Collectors.groupingBy(E2eDriver.StartedWorkflow::workflowName, TreeMap::new, + Collectors.counting())); + report.put("workflowsStarted", byName); + report.put("workflowStatus", byStatus); + List notCompleted = workflows.values().stream() + .filter(workflow -> workflow.getStatus() != Workflow.WorkflowStatus.COMPLETED) + .map(workflow -> workflow.getWorkflowId() + ":" + workflow.getStatus()) + .limit(10) + .toList(); + long completed = byStatus.getOrDefault("COMPLETED", 0L); + check("all workflows completed", completed == started.size() && startErrors == 0, + completed + "/" + started.size() + " completed, " + startErrors + " start errors" + + (notCompleted.isEmpty() ? "" : ", e.g. " + notCompleted)); + } + + private void checkExecutions(Map report, List executions, List tasks) { + Map terminalByTask = executions.stream() + .filter(execution -> !"IN_PROGRESS".equals(execution.status())) + .collect(Collectors.groupingBy(Execution::taskId, Collectors.counting())); + List duplicates = terminalByTask.entrySet().stream() + .filter(entry -> entry.getValue() > 1) + .map(Map.Entry::getKey) + .toList(); + long missing = tasks.stream() + .filter(task -> task.getStatus() == Task.Status.COMPLETED) + .filter(task -> !terminalByTask.containsKey(task.getTaskId())) + .count(); + long failures = executions.stream().filter(execution -> "EXCEPTION".equals(execution.status())).count(); + long retried = tasks.stream().filter(task -> task.getRetryCount() > 0).count(); + report.put("executions", Map.of("total", executions.size(), "injectedFailures", failures, + "serverRetries", retried)); + check("no duplicate executions", duplicates.isEmpty(), + duplicates.size() + " task ids executed to completion more than once" + + (duplicates.isEmpty() ? "" : ", e.g. " + duplicates.subList(0, Math.min(5, + duplicates.size())))); + check("every task execution logged", missing == 0, + missing + " completed server tasks have no worker execution record"); + } + + private void checkConcurrency(Map report, List executions) { + int maxProcess = E2eConfig.envInt("E2E_MAX_PROCESS_CONCURRENCY", 256); + int maxWorker = E2eConfig.envInt("E2E_MAX_WORKER_CONCURRENCY", 64); + Map fixed = E2eConfig.envIntMap("E2E_FIXED_THREADS", + "e2e_fetch_profile=8,e2e_score=3,e2e_persist=10,e2e_notify_partner=2"); + Map> peaks = new TreeMap<>(); + List violations = new ArrayList<>(); + executions.stream().collect(Collectors.groupingBy(Execution::worker)).forEach((worker, rows) -> { + Map peak = new TreeMap<>(); + peak.put("total", peakConcurrency(rows)); + rows.stream().collect(Collectors.groupingBy(Execution::taskType)) + .forEach((taskType, typed) -> peak.put(taskType, peakConcurrency(typed))); + peaks.put(worker, peak); + peak.forEach((key, value) -> { + int limit = adaptive ? ("total".equals(key) ? maxProcess : maxWorker) + : ("total".equals(key) ? Integer.MAX_VALUE : fixed.getOrDefault(key, 1)); + if (value > limit) { + violations.add(worker + "/" + key + "=" + value + ">" + limit); + } + }); + }); + report.put("peakConcurrency", peaks); + check("concurrency within ceilings", violations.isEmpty() && !peaks.isEmpty(), + violations.isEmpty() ? "peaks " + peaks : violations); + } + + private static int peakConcurrency(List rows) { + List events = new ArrayList<>(); + for (Execution row : rows) { + events.add(new long[] {row.start(), 1}); + events.add(new long[] {row.end(), -1}); + } + // Ends sort before starts at the same millisecond: a slot freed and reused is not an overlap. + events.sort((a, b) -> a[0] != b[0] ? Long.compare(a[0], b[0]) : Long.compare(a[1], b[1])); + int current = 0; + int peak = 0; + for (long[] event : events) { + current += (int) event[1]; + peak = Math.max(peak, current); + } + return peak; + } + + private void checkDomain(Map report, List tasks) { + List notify = tasks.stream() + .filter(task -> E2eConfig.NOTIFY_PARTNER.equals(task.getTaskDefName())) + .toList(); + long outsideDomain = notify.stream() + .filter(task -> !E2eConfig.PARTNER_DOMAIN.equals(task.getDomain())) + .count(); + List probeIds = started.stream().filter(E2eDriver.StartedWorkflow::probe) + .map(E2eDriver.StartedWorkflow::workflowId).toList(); + long probesRouted = probeIds.stream() + .map(workflows::get) + .filter(workflow -> workflow != null && workflow.getStatus() == Workflow.WorkflowStatus.COMPLETED) + .flatMap(workflow -> workflow.getTasks().stream()) + .filter(task -> E2eConfig.NOTIFY_PARTNER.equals(task.getTaskDefName())) + .filter(task -> E2eConfig.PARTNER_DOMAIN.equals(task.getDomain())) + .map(Task::getWorkflowInstanceId) + .distinct() + .count(); + check("domain routing kept alive", outsideDomain == 0 && probesRouted == probeIds.size(), + notify.size() + " partner tasks, " + outsideDomain + " fell back to the default queue; " + + probesRouted + "/" + probeIds.size() + " post-idle probes routed to the domain"); + } + + private void checkSlotsReleased(Map report) { + E2eSampler.Sample last = samples.get(samples.size() - 1); + List leaks = new ArrayList<>(); + // Each idle queue may hold one slot for its standby poll. + int standbySlots = adaptive && standbyMillis > 0 ? E2eConfig.TASK_TYPES.size() : 0; + for (int i = 0; i < workerCount; i++) { + double inflight = last.get("w" + i + ".inflight"); + double held = adaptive ? last.get("w" + i + ".processHeld") : 0; + if (inflight != 0 || held > standbySlots) { + leaks.add("w" + i + " inflight=" + inflight + " held=" + held); + } + } + check("slots released after drain", leaks.isEmpty(), leaks.isEmpty() + ? "no executions, at most " + standbySlots + " slots held by standby polls" : leaks); + } + + private void checkIdle(Map report) { + List idle = phase("idle"); + if (idle.size() < 40) { + check("idle polling backs off", false, "idle phase too short: " + idle.size() + "s"); + return; + } + // Skip the first 30 seconds: the backlog drains and backoff ramps up. + E2eSampler.Sample first = idle.get(30); + E2eSampler.Sample last = idle.get(idle.size() - 1); + double seconds = (last.epochMillis() - first.epochMillis()) / 1000.0; + Map pollRates = new TreeMap<>(); + double maxRate = 0; + for (int i = 0; i < workerCount; i++) { + for (String taskType : E2eConfig.TASK_TYPES) { + String key = "w" + i + "." + taskType + ".polls"; + double rate = (last.get(key) - first.get(key)) / seconds; + pollRates.put(key, round(rate)); + maxRate = Math.max(maxRate, rate); + } + } + report.put("idlePollsPerSecond", pollRates); + if (adaptive) { + // One standby poll per timeout, or one heartbeat per interval without standby polls. + double limit = standbyMillis > 0 ? 1.1 * 1000 / standbyMillis : 0.5; + check("idle polling backs off", maxRate <= limit, + "max " + round(maxRate) + " polls/s per queue per process (limit " + round(limit) + ")"); + double peak = samples.stream().mapToDouble(sample -> maxOverWorkers(sample, "processTarget")) + .filter(value -> !Double.isNaN(value)).max().orElse(Double.NaN); + double end = maxOverWorkers(last, "processTarget"); + // Decay never goes below the configured minimum. + int minimum = E2eConfig.envInt("E2E_MIN_PROCESS_CONCURRENCY", 1); + check("process target decays when idle", end <= Math.max(minimum, Math.ceil(peak / 2)), + "peak " + peak + ", end of idle " + end + " (minimum " + minimum + ")"); + } else { + check("idle polling backs off", true, "fixed mode reference: max " + round(maxRate) + " polls/s"); + } + } + + private Map> queueWaits(List tasks) { + Map phaseByWorkflow = started.stream() + .collect(Collectors.toMap(E2eDriver.StartedWorkflow::workflowId, E2eDriver.StartedWorkflow::phase)); + Map> result = new TreeMap<>(); + tasks.stream() + .filter(task -> task.getRetryCount() == 0 && task.getStartTime() > 0) + .collect(Collectors.groupingBy(task -> phaseByWorkflow.getOrDefault(task.getWorkflowInstanceId(), + "?") + "/" + task.getTaskDefName())) + .forEach((key, group) -> { + List waits = group.stream().map(task -> task.getStartTime() - task.getScheduledTime()) + .sorted().toList(); + Map stats = new LinkedHashMap<>(); + stats.put("n", waits.size()); + stats.put("p50", percentile(waits, 0.50)); + stats.put("p99", percentile(waits, 0.99)); + stats.put("max", waits.get(waits.size() - 1)); + result.put(key, stats); + }); + return result; + } + + private Map> phaseSummary() { + Map> result = new LinkedHashMap<>(); + for (String name : Stream.concat(LOAD_PHASES.stream(), Stream.of("idle")).toList()) { + List rows = phase(name); + if (rows.isEmpty()) { + continue; + } + Map summary = new LinkedHashMap<>(); + summary.put("seconds", rows.size()); + summary.put("profileMaxInFlight", max(rows, sample -> sample.get("sim.profile.maxInFlight"))); + summary.put("profileP99Millis", max(rows, sample -> sample.get("sim.profile.p99Millis"))); + summary.put("persistMaxInFlight", max(rows, sample -> sample.get("sim.persist.maxInFlight"))); + for (String queue : List.of(E2eConfig.FETCH_PROFILE, E2eConfig.SCORE, E2eConfig.PERSIST)) { + summary.put("maxDepth." + queue, max(rows, sample -> sample.get("depth." + queue))); + } + for (int i = 0; i < workerCount; i++) { + String prefix = "w" + i + "."; + summary.put(prefix + "cpuAvg", round(rows.stream().mapToDouble(s -> s.get(prefix + "cpu")) + .filter(v -> !Double.isNaN(v)).average().orElse(Double.NaN))); + summary.put(prefix + "cpuMax", max(rows, sample -> sample.get(prefix + "cpu"))); + summary.put(prefix + "heapMaxMb", max(rows, sample -> sample.get(prefix + "heapUsedMb"))); + summary.put(prefix + "inflightMax", max(rows, sample -> sample.get(prefix + "inflight"))); + if (adaptive) { + summary.put(prefix + "processTargetMax", max(rows, s -> s.get(prefix + "processTarget"))); + summary.put(prefix + "pausedSeconds", rows.stream().filter(s -> s.get(prefix + "paused") == 1) + .count()); + for (String taskType : E2eConfig.TASK_TYPES) { + summary.put(prefix + taskType + ".targetMax", + max(rows, s -> s.get(prefix + taskType + ".target"))); + } + } + E2eSampler.Sample first = rows.get(0); + E2eSampler.Sample last = rows.get(rows.size() - 1); + double seconds = Math.max(1, (last.epochMillis() - first.epochMillis()) / 1000.0); + for (String key : List.of("req.queueSizes", "req.poll", "req.update")) { + summary.put(prefix + key + "PerSec", + round((last.get(prefix + key) - first.get(prefix + key)) / seconds)); + } + } + result.put(name, summary); + } + return result; + } + + /** Seconds from the burst until the fetch and persist backlog first falls back below 20. */ + private Double burstDrainSeconds() { + List after = samples.stream() + .filter(sample -> List.of("burst", "slowdown", "idle", "probe", "drain").contains(sample.phase())) + .toList(); + if (after.isEmpty()) { + return null; + } + boolean backlogged = false; + for (E2eSampler.Sample sample : after) { + double backlog = sample.get("depth." + E2eConfig.FETCH_PROFILE) + sample.get("depth." + E2eConfig.PERSIST); + backlogged |= backlog >= 100; + if (backlogged && backlog < 20) { + return round((sample.epochMillis() - after.get(0).epochMillis()) / 1000.0); + } + } + return null; + } + + private Map reasons() { + E2eSampler.Sample last = samples.get(samples.size() - 1); + Map result = new TreeMap<>(); + last.values().forEach((key, value) -> { + if (key.contains(".reason.")) { + result.put(key, value); + } + }); + return result; + } + + private List phase(String name) { + return samples.stream().filter(sample -> name.equals(sample.phase())).toList(); + } + + private double maxOverWorkers(E2eSampler.Sample sample, String key) { + double max = Double.NaN; + for (int i = 0; i < workerCount; i++) { + double value = sample.get("w" + i + "." + key); + if (!Double.isNaN(value)) { + max = Double.isNaN(max) ? value : Math.max(max, value); + } + } + return max; + } + + private static double max(List rows, ToDoubleFunction value) { + return round(rows.stream().mapToDouble(value).filter(v -> !Double.isNaN(v)).max().orElse(Double.NaN)); + } + + private static long percentile(List sorted, double quantile) { + return sorted.get(Math.min(sorted.size() - 1, (int) Math.ceil(quantile * sorted.size()) - 1)); + } + + private static double round(double value) { + return Math.round(value * 100) / 100.0; + } + + private List readExecutions() throws IOException { + List executions = new ArrayList<>(); + if (!Files.isDirectory(logDir)) { + return Collections.emptyList(); + } + try (Stream files = Files.list(logDir)) { + for (Path file : files.filter(path -> path.getFileName().toString().startsWith("executions-")).toList()) { + List lines = Files.readAllLines(file); + for (String line : lines.subList(1, lines.size())) { + String[] f = line.split(","); + if (f.length == 9) { + executions.add(new Execution(f[0], f[1], Long.parseLong(f[5]), Long.parseLong(f[6]), f[7], + f[8])); + } + } + } + } + return executions; + } +} diff --git a/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eConfig.java b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eConfig.java new file mode 100644 index 000000000..d149c7b47 --- /dev/null +++ b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eConfig.java @@ -0,0 +1,78 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.orkes.conductor.harness.e2e; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Names and environment settings shared by the end-to-end simulator, workers and driver. */ +final class E2eConfig { + + static final String FETCH_PROFILE = "e2e_fetch_profile"; + static final String SCORE = "e2e_score"; + static final String PERSIST = "e2e_persist"; + static final String NOTIFY_PARTNER = "e2e_notify_partner"; + static final List TASK_TYPES = List.of(FETCH_PROFILE, SCORE, PERSIST, NOTIFY_PARTNER); + + static final String PARTNER_DOMAIN = "partner_a"; + + static final String ORDER_FULFILLMENT = "e2e_order_fulfillment"; + static final String RISK_REVIEW = "e2e_risk_review"; + static final String PARTNER_SYNC = "e2e_partner_sync"; + static final String NIGHTLY_RECONCILE = "e2e_nightly_reconcile"; + + /** Queue names as the server sizes them: task type, or {@code domain:taskType}. */ + static final List QUEUE_NAMES = + List.of(FETCH_PROFILE, SCORE, PERSIST, PARTNER_DOMAIN + ":" + NOTIFY_PARTNER, NOTIFY_PARTNER); + + private E2eConfig() {} + + static String env(String name, String defaultValue) { + String value = System.getenv(name); + return value == null || value.isBlank() ? defaultValue : value.trim(); + } + + static int envInt(String name, int defaultValue) { + return Integer.parseInt(env(name, Integer.toString(defaultValue))); + } + + static double envDouble(String name, double defaultValue) { + return Double.parseDouble(env(name, Double.toString(defaultValue))); + } + + static List envList(String name, String defaultValue) { + return Arrays.stream(env(name, defaultValue).split(",")).map(String::trim).filter(s -> !s.isEmpty()).toList(); + } + + /** Parses {@code a=x,b=y}. */ + static Map envStringMap(String name, String defaultValue) { + Map values = new LinkedHashMap<>(); + for (String pair : envList(name, defaultValue)) { + String[] parts = pair.split("="); + values.put(parts[0].trim(), parts[1].trim()); + } + return values; + } + + /** Parses {@code a=1,b=2}. */ + static Map envIntMap(String name, String defaultValue) { + Map values = new LinkedHashMap<>(); + for (String pair : envList(name, defaultValue)) { + String[] parts = pair.split("="); + values.put(parts[0].trim(), Integer.parseInt(parts[1].trim())); + } + return values; + } +} diff --git a/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eDriver.java b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eDriver.java new file mode 100644 index 000000000..134cf63d4 --- /dev/null +++ b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eDriver.java @@ -0,0 +1,339 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.orkes.conductor.harness.e2e; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Random; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.conductor.client.http.ConductorClient; +import com.netflix.conductor.client.http.MetadataClient; +import com.netflix.conductor.client.http.TaskClient; +import com.netflix.conductor.client.http.WorkflowClient; +import com.netflix.conductor.common.metadata.tasks.TaskDef; +import com.netflix.conductor.common.metadata.tasks.TaskType; +import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; +import com.netflix.conductor.common.metadata.workflow.WorkflowDef; +import com.netflix.conductor.common.metadata.workflow.WorkflowTask; +import com.netflix.conductor.common.run.Workflow; + +import io.orkes.conductor.client.ApiClient; + +/** + * Drives the end-to-end test: registers four workflows that share four tasks unevenly, starts them with Poisson + * arrivals through a sequence of phases, waits for the system to drain, then hands everything to + * {@link E2eAnalysis}. + */ +public class E2eDriver { + + private static final Logger log = LoggerFactory.getLogger(E2eDriver.class); + + private static final String OWNER = "java-sdk-e2e@conductor.io"; + + private final WorkflowClient workflowClient; + private final E2eSampler sampler; + private final String simulatorUrl; + private final String runId = "run-" + System.currentTimeMillis(); + private final Random random = new Random(42); + private final ExecutorService starters = Executors.newFixedThreadPool(16); + private final Queue started = new ConcurrentLinkedQueue<>(); + private final AtomicInteger startErrors = new AtomicInteger(); + private final HttpClient http = HttpClient.newHttpClient(); + + /** A workflow the driver started, with the phase it was started in. */ + record StartedWorkflow(String workflowId, String workflowName, String phase, boolean probe) {} + + E2eDriver(ConductorClient client, String simulatorUrl, List workerUrls) { + this.workflowClient = new WorkflowClient(client); + this.simulatorUrl = simulatorUrl; + this.sampler = new E2eSampler(new TaskClient(client), simulatorUrl, workerUrls); + } + + public static void main(String[] args) throws Exception { + ConductorClient client = ApiClient.builder().useEnvVariables(true).readTimeout(30_000).build(); + Path outDir = Path.of(E2eConfig.env("E2E_OUT_DIR", "build/e2e")); + Files.createDirectories(outDir); + E2eDriver driver = new E2eDriver(client, E2eConfig.env("SIM_URL", "http://localhost:8090"), + E2eConfig.envList("E2E_WORKER_METRICS", "http://localhost:9991,http://localhost:9992")); + driver.registerMetadata(new MetadataClient(client)); + boolean passed = driver.run(outDir); + System.exit(passed ? 0 : 1); + } + + boolean run(Path outDir) throws Exception { + double rate = E2eConfig.envDouble("E2E_RATE", 5); + sampler.start(); + sampler.setPhase("baseline"); + Thread.sleep(15_000); + + runPhase("warmup", E2eConfig.envInt("E2E_WARMUP_SECONDS", 60), rate / 2); + runPhase("steady", E2eConfig.envInt("E2E_STEADY_SECONDS", 120), rate); + sampler.setPhase("burst"); + int burstSize = E2eConfig.envInt("E2E_BURST_SIZE", 1000); + for (int i = 0; i < burstSize; i++) { + submit(E2eConfig.NIGHTLY_RECONCILE, "burst", false); + } + runPhase("burst", E2eConfig.envInt("E2E_BURST_SECONDS", 120), rate); + setLatencyMultiplier(E2eConfig.envDouble("E2E_SLOWDOWN_MULTIPLIER", 5)); + try { + runPhase("slowdown", E2eConfig.envInt("E2E_SLOWDOWN_SECONDS", 90), rate); + } finally { + setLatencyMultiplier(1); + } + runPhase("idle", E2eConfig.envInt("E2E_IDLE_SECONDS", 75), 0); + sampler.setPhase("probe"); + // After the idle period the domain must still be live, so these tasks must land in the partner domain. + for (int i = 0; i < E2eConfig.envInt("E2E_DOMAIN_PROBES", 10); i++) { + submit(E2eConfig.PARTNER_SYNC, "probe", true); + } + starters.shutdown(); + starters.awaitTermination(5, TimeUnit.MINUTES); + updateCounters(); + + sampler.setPhase("drain"); + waitForDrain(E2eConfig.envInt("E2E_DRAIN_TIMEOUT_SECONDS", 300)); + Map workflows = fetchWorkflows(); + sampler.setPhase("final"); + Thread.sleep(5_000); + sampler.stop(); + + sampler.writeCsv(outDir.resolve("timeseries.csv")); + E2eAnalysis analysis = new E2eAnalysis(List.copyOf(started), workflows, sampler.samples(), + sampler.workerCount(), Path.of(E2eConfig.env("E2E_LOG_DIR", "build/e2e/logs")), startErrors.get()); + return analysis.writeReport(outDir); + } + + private void runPhase(String phase, int seconds, double rate) throws InterruptedException { + sampler.setPhase(phase); + long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(seconds); + while (System.nanoTime() < end) { + if (rate <= 0) { + Thread.sleep(500); + } else { + // Poisson arrivals: exponential gaps between starts. + Thread.sleep((long) (-Math.log(1 - random.nextDouble()) / rate * 1000)); + submit(pickWorkflow(), phase, false); + } + updateCounters(); + } + } + + private String pickWorkflow() { + double roll = random.nextDouble(); + if (roll < 0.65) { + return E2eConfig.ORDER_FULFILLMENT; + } + return roll < 0.90 ? E2eConfig.RISK_REVIEW : E2eConfig.PARTNER_SYNC; + } + + private void submit(String workflowName, String phase, boolean probe) { + StartWorkflowRequest request = new StartWorkflowRequest(); + request.setName(workflowName); + request.setVersion(1); + request.setCorrelationId(runId); + request.setInput(input(workflowName)); + if (E2eConfig.PARTNER_SYNC.equals(workflowName)) { + // Falls back to the default queue, which nobody polls, if no worker polled the domain recently. + request.setTaskToDomain(Map.of(E2eConfig.NOTIFY_PARTNER, E2eConfig.PARTNER_DOMAIN + ",NO_DOMAIN")); + } + starters.submit(() -> { + try { + started.add(new StartedWorkflow(workflowClient.startWorkflow(request), workflowName, phase, probe)); + } catch (Exception e) { + startErrors.incrementAndGet(); + log.warn("Failed to start {}: {}", workflowName, e.getMessage()); + } + }); + } + + private synchronized Map input(String workflowName) { + Map input = new HashMap<>(); + input.put("customerId", "c-" + random.nextInt(100_000)); + switch (workflowName) { + case E2eConfig.ORDER_FULFILLMENT -> { + input.put("docKb", logNormal(32, 0.7, 4, 512)); + int items = logNormal(3, 0.8, 1, 50); + List> tasks = new ArrayList<>(); + Map tasksInput = new LinkedHashMap<>(); + for (int i = 0; i < items; i++) { + tasks.add(Map.of("name", E2eConfig.FETCH_PROFILE, "taskReferenceName", "item_" + i, "type", + TaskType.SIMPLE.name())); + tasksInput.put("item_" + i, Map.of("id", "sku-" + random.nextInt(10_000))); + } + input.put("dynamicTasks", tasks); + input.put("dynamicTasksInput", tasksInput); + } + case E2eConfig.RISK_REVIEW -> input.put("docKb", logNormal(128, 0.6, 16, 1024)); + case E2eConfig.PARTNER_SYNC -> input.put("partnerId", E2eConfig.PARTNER_DOMAIN); + default -> { + } + } + return input; + } + + private int logNormal(double median, double sigma, int min, int max) { + return (int) Math.max(min, Math.min(max, Math.round(median * Math.exp(sigma * random.nextGaussian())))); + } + + private void updateCounters() { + sampler.setCounters(Map.of("driver.started", (double) started.size(), "driver.startErrors", + (double) startErrors.get())); + } + + private void setLatencyMultiplier(double multiplier) throws Exception { + http.send(HttpRequest.newBuilder(URI.create( + simulatorUrl + "/control?endpoint=profile&latencyMultiplier=" + multiplier)).GET().build(), + HttpResponse.BodyHandlers.discarding()); + } + + /** Waits until every queue is empty and no worker is executing for 15 consecutive seconds. */ + private void waitForDrain(int timeoutSeconds) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); + int quietSeconds = 0; + while (System.nanoTime() < deadline && quietSeconds < 15) { + Thread.sleep(1_000); + E2eSampler.Sample latest = sampler.latest(); + boolean quiet = latest != null; + if (latest != null) { + for (String queue : E2eConfig.QUEUE_NAMES) { + quiet &= latest.get("depth." + queue) == 0; + } + for (int i = 0; i < sampler.workerCount(); i++) { + quiet &= latest.get("w" + i + ".inflight") == 0; + } + } + quietSeconds = quiet ? quietSeconds + 1 : 0; + } + log.info("Drain finished (quiet for {}s)", quietSeconds); + } + + /** Fetches every started workflow with its tasks, re-checking the ones still running for a short while. */ + private Map fetchWorkflows() throws Exception { + Map workflows = new HashMap<>(); + List pending = started.stream().map(StartedWorkflow::workflowId).toList(); + ExecutorService fetchers = Executors.newFixedThreadPool(16); + try { + for (int attempt = 0; attempt < 6 && !pending.isEmpty(); attempt++) { + if (attempt > 0) { + log.info("{} workflows not finished; re-checking in 10s", pending.size()); + Thread.sleep(10_000); + } + List> futures = new ArrayList<>(); + for (String id : pending) { + futures.add(fetchers.submit(() -> workflowClient.getWorkflow(id, true))); + } + List stillRunning = new ArrayList<>(); + for (Future future : futures) { + Workflow workflow = future.get(); + workflows.put(workflow.getWorkflowId(), workflow); + if (!workflow.getStatus().isTerminal()) { + stillRunning.add(workflow.getWorkflowId()); + } + } + pending = stillRunning; + } + } finally { + fetchers.shutdown(); + } + log.info("Fetched {} workflows", workflows.size()); + return workflows; + } + + void registerMetadata(MetadataClient metadataClient) { + List taskDefs = new ArrayList<>(); + for (String taskType : E2eConfig.TASK_TYPES) { + TaskDef taskDef = new TaskDef(taskType); + taskDef.setOwnerEmail(OWNER); + taskDef.setRetryCount(3); + taskDef.setRetryLogic(TaskDef.RetryLogic.FIXED); + taskDef.setRetryDelaySeconds(2); + taskDef.setTimeoutSeconds(900); + taskDef.setResponseTimeoutSeconds(120); + taskDefs.add(taskDef); + } + metadataClient.registerTaskDefs(taskDefs); + + WorkflowTask fork = task(TaskType.FORK_JOIN_DYNAMIC.name(), "line_items", Map.of( + "dynamicTasks", "${workflow.input.dynamicTasks}", + "dynamicTasksInput", "${workflow.input.dynamicTasksInput}")); + fork.setDynamicForkTasksParam("dynamicTasks"); + fork.setDynamicForkTasksInputParamName("dynamicTasksInput"); + WorkflowTask join = task(TaskType.JOIN.name(), "line_items_join", Map.of()); + join.setName("line_items_join"); + + List workflowDefs = List.of( + workflow(E2eConfig.ORDER_FULFILLMENT, + simple(E2eConfig.FETCH_PROFILE, "customer", Map.of("id", "${workflow.input.customerId}")), + fork, join, + simple(E2eConfig.SCORE, "score", Map.of("docKb", "${workflow.input.docKb}")), + simple(E2eConfig.PERSIST, "persist", Map.of("score", "${score.output.score}"))), + workflow(E2eConfig.RISK_REVIEW, + simple(E2eConfig.SCORE, "score_identity", Map.of("docKb", "${workflow.input.docKb}")), + simple(E2eConfig.SCORE, "score_transactions", Map.of("docKb", "${workflow.input.docKb}")), + simple(E2eConfig.PERSIST, "persist", Map.of())), + workflow(E2eConfig.PARTNER_SYNC, + simple(E2eConfig.FETCH_PROFILE, "customer", Map.of("id", "${workflow.input.customerId}")), + simple(E2eConfig.NOTIFY_PARTNER, "notify", Map.of("partnerId", + "${workflow.input.partnerId}"))), + workflow(E2eConfig.NIGHTLY_RECONCILE, + simple(E2eConfig.FETCH_PROFILE, "customer", Map.of("id", "${workflow.input.customerId}")), + simple(E2eConfig.PERSIST, "persist_ledger", Map.of()), + simple(E2eConfig.PERSIST, "persist_balance", Map.of()), + simple(E2eConfig.PERSIST, "persist_audit", Map.of()))); + metadataClient.updateWorkflowDefs(workflowDefs); + log.info("Registered {} task definitions and {} workflows", taskDefs.size(), workflowDefs.size()); + } + + private static WorkflowDef workflow(String name, WorkflowTask... tasks) { + WorkflowDef def = new WorkflowDef(); + def.setName(name); + def.setVersion(1); + def.setOwnerEmail(OWNER); + def.setTasks(List.of(tasks)); + return def; + } + + private static WorkflowTask simple(String taskType, String referenceName, Map input) { + WorkflowTask task = task(TaskType.SIMPLE.name(), referenceName, input); + task.setName(taskType); + return task; + } + + private static WorkflowTask task(String type, String referenceName, Map input) { + WorkflowTask task = new WorkflowTask(); + task.setName(referenceName); + task.setTaskReferenceName(referenceName); + task.setType(type); + task.setInputParameters(new HashMap<>(input)); + return task; + } +} diff --git a/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eFunctionalDriver.java b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eFunctionalDriver.java new file mode 100644 index 000000000..4afa5c04b --- /dev/null +++ b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eFunctionalDriver.java @@ -0,0 +1,802 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.orkes.conductor.harness.e2e; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.conductor.client.http.ConductorClient; +import com.netflix.conductor.client.http.MetadataClient; +import com.netflix.conductor.client.http.TaskClient; +import com.netflix.conductor.client.http.WorkflowClient; +import com.netflix.conductor.common.metadata.tasks.Task; +import com.netflix.conductor.common.metadata.tasks.TaskDef; +import com.netflix.conductor.common.metadata.tasks.TaskType; +import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest; +import com.netflix.conductor.common.metadata.workflow.WorkflowDef; +import com.netflix.conductor.common.metadata.workflow.WorkflowTask; +import com.netflix.conductor.common.run.Workflow; + +import io.orkes.conductor.client.ApiClient; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +/** + * Functional scenarios for adaptive concurrency against a real server and one worker process: depth-weighted + * shares, floor rotation, starvation, scaling up and down, CPU pressure and heap pressure. Each scenario loads the + * queues before the worker starts polling, so the controller sees a known backlog, and checks the outcome against + * the design's rules. + */ +public class E2eFunctionalDriver { + + private static final Logger log = LoggerFactory.getLogger(E2eFunctionalDriver.class); + + private static final String OWNER = "java-sdk-e2e@conductor.io"; + private static final List TASKS = List.of("fn_a", "fn_b", "fn_c", "fn_d"); + + private final String scenario; + private final WorkflowClient workflowClient; + private final MetadataClient metadataClient; + private final TaskClient taskClient; + private final Map domains; + private final E2eSampler sampler; + private final Path logDir; + private final int maxProcess = E2eConfig.envInt("E2E_MAX_PROCESS_CONCURRENCY", 256); + /** The SDK clamps each worker's maximum to the process ceiling. */ + private final int maxWorker = Math.min(maxProcess, E2eConfig.envInt("E2E_MAX_WORKER_CONCURRENCY", 64)); + private final ExecutorService starters = Executors.newFixedThreadPool(32); + private final ConcurrentLinkedQueue started = new ConcurrentLinkedQueue<>(); + private final List> checks = new ArrayList<>(); + private final Map observations = new LinkedHashMap<>(); + private final Random random = new Random(7); + private long goMillis; + + E2eFunctionalDriver(String scenario, ConductorClient client) { + this.scenario = scenario; + this.workflowClient = new WorkflowClient(client); + this.metadataClient = new MetadataClient(client); + this.taskClient = new TaskClient(client); + this.domains = E2eConfig.envStringMap("E2E_TASK_DOMAINS", ""); + this.logDir = Path.of(E2eConfig.env("E2E_LOG_DIR", "build/e2e/logs")); + List queueNames = TASKS.stream().map(this::queueName).toList(); + this.sampler = new E2eSampler(taskClient, queueNames, null, + List.of(E2eConfig.env("E2E_WORKER_METRICS", "http://localhost:9993"))); + } + + public static void main(String[] args) throws Exception { + ConductorClient client = ApiClient.builder().useEnvVariables(true).readTimeout(30_000).build(); + String scenario = E2eConfig.env("E2E_SCENARIO", "weights"); + Path outDir = Path.of(E2eConfig.env("E2E_OUT_DIR", "build/e2e/" + scenario)); + Files.createDirectories(outDir); + E2eFunctionalDriver driver = new E2eFunctionalDriver(scenario, client); + boolean passed = driver.run(outDir); + System.exit(passed ? 0 : 1); + } + + boolean run(Path outDir) throws Exception { + registerMetadata(); + sampler.start(); + sampler.setPhase("load"); + switch (scenario) { + case "weights" -> weights(); + case "rotation" -> rotation(); + case "starvation" -> starvation(); + case "scaling" -> scaling(); + case "warmstart" -> warmStart(); + case "cpu" -> cpu(); + case "heap" -> heap(); + default -> throw new IllegalArgumentException("Unknown scenario " + scenario); + } + sampler.setPhase("final"); + Thread.sleep(3_000); + sampler.stop(); + Map workflows = fetchWorkflows(); + commonChecks(workflows); + sampler.writeCsv(outDir.resolve("timeseries.csv")); + return writeReport(outDir); + } + + // ----- Scenarios ----- + + /** P = 16 and backlogs of 600/300/150/30: shares, executions and throughput must follow backlog. */ + private void weights() throws Exception { + Map backlog = Map.of("fn_a", 600, "fn_b", 300, "fn_c", 150, "fn_d", 30); + backlog.forEach((task, count) -> preload(task, count, Map.of("sleepMs", 1000))); + go(); + waitForDrain(600); + + List contended = after(3_000).stream() + .filter(s -> TASKS.stream().mapToDouble(t -> depth(s, t)).sum() > 0) + .toList(); + int sharesMatch = 0; + int executionsMatch = 0; + int fullSamples = 0; + int fullAssigned = 0; + int counted = 0; + for (E2eSampler.Sample s : contended) { + int target = (int) s.get("w0.processTarget"); + long[] backlogs = new long[TASKS.size()]; + long[] demands = new long[TASKS.size()]; + long totalDemand = 0; + for (int i = 0; i < TASKS.size(); i++) { + backlogs[i] = (long) (depth(s, TASKS.get(i)) + get(s, TASKS.get(i), "held")); + demands[i] = Math.min(maxWorker, backlogs[i]); + totalDemand += demands[i]; + } + int[] expected = expectedShares(target, backlogs, demands); + boolean shares = true; + boolean executions = true; + int assigned = 0; + for (int i = 0; i < TASKS.size(); i++) { + double actual = get(s, TASKS.get(i), "target"); + shares &= Math.abs(actual - expected[i]) <= 1; + executions &= Math.abs(get(s, TASKS.get(i), "inflight") - actual) <= 1; + assigned += (int) actual; + } + counted++; + sharesMatch += shares ? 1 : 0; + executionsMatch += executions ? 1 : 0; + if (totalDemand >= target) { + fullSamples++; + fullAssigned += assigned >= target ? 1 : 0; + } + } + check("shares follow backlog", counted > 0 && sharesMatch >= 0.9 * counted, + sharesMatch + "/" + counted + " samples within one slot of the water-filled shares"); + check("executions follow shares", counted > 0 && executionsMatch >= 0.85 * counted, + executionsMatch + "/" + counted + " samples with live executions within one of the share"); + check("no slot left unassigned", fullSamples > 0 && fullAssigned >= 0.95 * fullSamples, + fullAssigned + "/" + fullSamples + " samples with demand >= target assign the whole target"); + + // Throughput while all four queues are backlogged must split like the shares. + long contentionEnd = contended.stream().filter(s -> depth(s, "fn_d") == 0).findFirst() + .map(E2eSampler.Sample::epochMillis).orElse(Long.MAX_VALUE); + List window = executions().stream() + .filter(e -> e.start() >= goMillis + 5_000 && e.start() < contentionEnd) + .toList(); + int[] expected = expectedShares(maxProcess, new long[] {600, 300, 150, 30}, + new long[] {maxWorker, maxWorker, maxWorker, 30}); + Map starts = window.stream() + .collect(Collectors.groupingBy(E2eAnalysis.Execution::taskType, Collectors.counting())); + Map slotEquivalent = new LinkedHashMap<>(); + boolean proportional = window.size() > 50; + for (int i = 0; i < TASKS.size(); i++) { + double slots = maxProcess * (double) starts.getOrDefault(TASKS.get(i), 0L) / Math.max(1, window.size()); + slotEquivalent.put(TASKS.get(i), round(slots) + " (expected " + expected[i] + ")"); + proportional &= Math.abs(slots - expected[i]) <= 1; + } + check("throughput splits by backlog", proportional, + window.size() + " starts while all queues were backlogged; slot-equivalents " + slotEquivalent); + checkPeak(maxProcess); + } + + /** P = 2 with four backlogged queues: floors exceed the target, so queues must take turns. */ + private void rotation() throws Exception { + for (String task : TASKS) { + preload(task, 40, Map.of("sleepMs", 1000)); + } + go(); + waitForDrain(600); + checkProgress(15_000); + checkPeak(maxProcess); + Map finish = executions().stream().collect(Collectors.groupingBy( + E2eAnalysis.Execution::taskType, Collectors.reducing(0L, E2eAnalysis.Execution::end, Math::max))); + long first = Collections.min(finish.values()) - goMillis; + long last = Collections.max(finish.values()) - goMillis; + check("equal backlogs finish together", last <= first * 1.25, + "queues finished between " + first / 1000 + "s and " + last / 1000 + "s after start"); + } + + /** P = 4: three deep queues plus a trickle of domain-routed tasks that must neither wait nor fall back. */ + private void starvation() throws Exception { + for (String task : List.of("fn_a", "fn_b", "fn_c")) { + preload(task, 100, Map.of("sleepMs", 2000)); + } + go(); + // Routing to a domain needs a poll from it first, as in fixed mode; start the trickle once there is one. + while (taskClient.getPollData("fn_d").stream().noneMatch(p -> domains.get("fn_d").equals(p.getDomain()))) { + Thread.sleep(200); + } + long end = System.currentTimeMillis() + 90_000; + while (System.currentTimeMillis() < end) { + start("fn_d", Map.of("sleepMs", 200)); + Thread.sleep(3_000); + } + waitForDrain(600); + checkProgress(15_000); + List trickle = tasks("fn_d"); + long fellBack = trickle.stream().filter(t -> !domains.get("fn_d").equals(t.getDomain())).count(); + check("trickle stays in its domain", !trickle.isEmpty() && fellBack == 0, + trickle.size() + " trickle tasks, " + fellBack + " routed outside domain " + domains.get("fn_d")); + List waits = trickle.stream().filter(t -> t.getStartTime() > 0) + .map(t -> t.getStartTime() - t.getScheduledTime()).sorted().toList(); + long maxWait = waits.isEmpty() ? Long.MAX_VALUE : waits.get(waits.size() - 1); + check("trickle is not starved", waits.size() == trickle.size() && maxWait <= 5_000, + "trickle queue wait p50 " + (waits.isEmpty() ? "-" : waits.get(waits.size() / 2)) + " ms, max " + + maxWait + " ms (limit 5000)"); + checkPeak(maxProcess); + } + + /** A deep I/O backlog: the target grows by the step rule to the ceiling, then decays once the backlog clears. */ + private void scaling() throws Exception { + preload("fn_a", 1500, Map.of("sleepMs", 2000)); + go(); + waitForQueuesEmpty(600); + long drainedAt = System.currentTimeMillis(); + sampler.setPhase("idle"); + waitForIdle(30); + Thread.sleep(E2eConfig.envInt("E2E_IDLE_SECONDS", 240) * 1000L); + + List busy = after(0).stream().filter(s -> s.epochMillis() < drainedAt).toList(); + List badSteps = new ArrayList<>(); + double previous = Double.NaN; + long previousChange = goMillis; + int steps = 0; + for (E2eSampler.Sample s : busy) { + double target = s.get("w0.processTarget"); + if (!Double.isNaN(previous) && target > previous) { + steps++; + double limit = Math.min(16, Math.max(1, Math.ceil(0.2 * previous))); + long gap = s.epochMillis() - previousChange; + if (target - previous > limit || gap < 3_500) { + badSteps.add((int) previous + "->" + (int) target + " after " + gap + "ms"); + } + } + if (!Double.isNaN(previous) && target != previous) { + previousChange = s.epochMillis(); + } + previous = target; + } + double peak = busy.stream().mapToDouble(s -> s.get("w0.processTarget")).filter(v -> !Double.isNaN(v)).max() + .orElse(0); + check("growth follows the step rule", steps > 0 && badSteps.isEmpty(), + steps + " growth steps" + (badSteps.isEmpty() ? "" : ", violations " + badSteps)); + int ceiling = Math.min(maxProcess, maxWorker); + check("target reaches the ceiling", peak >= ceiling, "peak " + peak + " (ceiling " + ceiling + ")"); + long threadsShort = busy.stream().filter(s -> get(s, "fn_a", "threads") < get(s, "fn_a", "inflight")).count(); + long threadsOver = busy.stream() + .filter(s -> get(s, "fn_a", "threads") > Math.max(s.get("w0.processTarget"), get(s, "fn_a", "held")) + 1) + .count(); + check("threads track the target", threadsShort == 0 && threadsOver <= busy.size() / 20, + threadsShort + " samples with fewer threads than executions, " + threadsOver + + " with more threads than max(target, held) + 1, of " + busy.size()); + double usedAtPeak = busy.stream().filter(s -> s.get("w0.processTarget") >= ceiling) + .mapToDouble(s -> get(s, "fn_a", "inflight")).max().orElse(0); + check("capacity is used", usedAtPeak >= 0.9 * ceiling, "max executions at the ceiling " + usedAtPeak); + + List idle = samplesIn("idle"); + List badDecay = new ArrayList<>(); + previous = Double.NaN; + previousChange = drainedAt; + for (E2eSampler.Sample s : idle) { + double target = s.get("w0.processTarget"); + if (!Double.isNaN(previous) && target < previous) { + long gap = s.epochMillis() - previousChange; + if (target < Math.ceil(previous / 2) || gap < 25_000) { + badDecay.add((int) previous + "->" + (int) target + " after " + gap + "ms"); + } + previousChange = s.epochMillis(); + } + previous = target; + } + E2eSampler.Sample last = idle.get(idle.size() - 1); + // Idle, the target keeps one slot per queue for its standby poll. + int idleTarget = standbyEnabled() ? TASKS.size() : 1; + check("target decays when idle", last.get("w0.processTarget") <= idleTarget && badDecay.isEmpty(), + "end of idle target " + last.get("w0.processTarget") + " (expected at most " + idleTarget + ")" + + (badDecay.isEmpty() ? "" : ", violations " + badDecay)); + double threadsLeft = TASKS.stream().mapToDouble(t -> get(last, t, "threads")).sum(); + check("idle threads are released", threadsLeft <= TASKS.size(), + "execution threads at the end " + threadsLeft + " across " + TASKS.size() + " queues"); + observations.put("targetSeries", series(Stream.concat(busy.stream(), idle.stream()).toList(), + s -> s.get("w0.processTarget"))); + } + + private static boolean standbyEnabled() { + return E2eConfig.envInt("E2E_STANDBY_POLL_TIMEOUT_MS", 1000) > 0; + } + + /** A configured start and minimum: no ramp from one slot, decay stops at the minimum, bursts start warm. */ + private void warmStart() throws Exception { + int initial = E2eConfig.envInt("E2E_INITIAL_PROCESS_CONCURRENCY", 32); + int minimum = E2eConfig.envInt("E2E_MIN_PROCESS_CONCURRENCY", 16); + preload("fn_a", 1500, Map.of("sleepMs", 2000)); + go(); + waitForQueuesEmpty(600); + long drainedAt = System.currentTimeMillis(); + sampler.setPhase("idle"); + waitForIdle(30); + Thread.sleep(E2eConfig.envInt("E2E_IDLE_SECONDS", 150) * 1000L); + + List busy = after(0).stream().filter(s -> s.epochMillis() < drainedAt).toList(); + double first = busy.stream().mapToDouble(s -> s.get("w0.processTarget")).filter(v -> !Double.isNaN(v)) + .findFirst().orElse(Double.NaN); + check("starts at the configured target", first >= initial, "first target " + first + " (configured " + + initial + ")"); + long reachedCeiling = busy.stream().filter(s -> s.get("w0.processTarget") >= Math.min(maxProcess, maxWorker)) + .map(E2eSampler.Sample::epochMillis).findFirst().orElse(Long.MAX_VALUE); + check("reaches the ceiling", reachedCeiling != Long.MAX_VALUE, + "ceiling reached " + (reachedCeiling == Long.MAX_VALUE ? "never" : (reachedCeiling - goMillis) / 1000 + + "s after start")); + List idle = samplesIn("idle"); + double lowest = idle.stream().mapToDouble(s -> s.get("w0.processTarget")).filter(v -> !Double.isNaN(v)).min() + .orElse(Double.NaN); + double end = idle.get(idle.size() - 1).get("w0.processTarget"); + check("decay stops at the minimum", lowest >= minimum && end == minimum, + "lowest idle target " + lowest + ", end of idle " + end + " (minimum " + minimum + ")"); + double threadsLeft = TASKS.stream().mapToDouble(t -> get(idle.get(idle.size() - 1), t, "threads")).sum(); + check("idle threads are released", threadsLeft <= TASKS.size(), + "execution threads at the end of idle " + threadsLeft); + + // A second burst after idle starts with the minimum already available. + sampler.setPhase("burst"); + long burstAt = System.currentTimeMillis(); + preload("fn_a", 200, Map.of("sleepMs", 2000)); + waitForDrain(300); + long started = executions().stream() + .filter(e -> e.start() >= burstAt && e.start() <= burstAt + 3_000).count(); + check("burst after idle starts warm", started >= minimum, + started + " tasks started within 3s of the burst (minimum " + minimum + ")"); + observations.put("targetSeries", series(after(0), s -> s.get("w0.processTarget"))); + } + + /** One CPU, CPU-bound tasks, starting at P = 8: reductions of 25% at most every 15 s and no growth meanwhile. */ + private void cpu() throws Exception { + preload("fn_a", 300, Map.of("cpuMs", 250)); + go(); + waitForDrain(900); + List busy = after(0).stream().filter(s -> depth(s, "fn_a") > 0).toList(); + List reductions = new ArrayList<>(); + List bad = new ArrayList<>(); + double previous = Double.NaN; + long lastReduction = 0; + for (int i = 0; i < busy.size(); i++) { + E2eSampler.Sample s = busy.get(i); + double target = s.get("w0.processTarget"); + if (!Double.isNaN(previous) && target < previous) { + reductions.add(target); + double expected = previous * 0.75; + long gap = lastReduction == 0 ? Long.MAX_VALUE : s.epochMillis() - lastReduction; + if (Math.abs(target - expected) > 1 || gap < 14_000) { + bad.add((int) previous + "->" + (int) target + " after " + gap + "ms"); + } + lastReduction = s.epochMillis(); + } else if (!Double.isNaN(previous) && target > previous && i >= 3 + && busy.subList(i - 3, i).stream().allMatch(p -> p.get("w0.cpu") >= 0.9)) { + bad.add("growth " + (int) previous + "->" + (int) target + " while CPU was saturated"); + } + previous = target; + } + check("CPU pressure reduces the target", !reductions.isEmpty() && bad.isEmpty(), + reductions.size() + " reductions to " + reductions + (bad.isEmpty() ? "" : ", violations " + bad)); + double settled = busy.isEmpty() ? Double.NaN : busy.get(busy.size() - 1).get("w0.processTarget"); + check("target settles near the core count", settled <= 2, "target at the end of the backlog " + settled); + List runs = executions(); + double seconds = (runs.stream().mapToLong(E2eAnalysis.Execution::end).max().orElse(0) + - runs.stream().mapToLong(E2eAnalysis.Execution::start).min().orElse(0)) / 1000.0; + double throughput = runs.size() / Math.max(1, seconds); + check("CPU stays productive", throughput >= 0.7 * 4, round(throughput) + " tasks/s (one core, 250 ms " + + "of CPU each: ideal 4/s)"); + } + + /** Tasks that hold large payloads in a small heap: no OOM, and no polling while admission is paused. */ + private void heap() throws Exception { + for (int i = 0; i < 300; i++) { + int allocMb = (int) Math.min(30, Math.round(6 * Math.exp(0.5 * random.nextGaussian()))); + start("fn_a", Map.of("allocMb", allocMb, "sleepMs", 3000)); + } + awaitStarts(); + go(); + waitForDrain(900); + List run = after(0); + long down = run.stream().filter(s -> s.get("w0.up") != 1).count(); + check("worker survives heap pressure", down == 0, down + " samples with the worker unreachable"); + // Rule one: nothing polls without an open slot, so nothing polls while admission is paused. + List runs = executions(); + int pausedSamples = 0; + long startsWhilePaused = 0; + double pollsWhilePaused = 0; + for (int i = 1; i < run.size(); i++) { + E2eSampler.Sample before = run.get(i - 1); + E2eSampler.Sample now = run.get(i); + if (before.get("w0.paused") == 1 && now.get("w0.paused") == 1) { + pausedSamples++; + pollsWhilePaused += get(now, "fn_a", "polls") - get(before, "fn_a", "polls"); + startsWhilePaused += runs.stream() + .filter(e -> e.start() > before.epochMillis() && e.start() <= now.epochMillis()).count(); + } + } + check("no polling while paused", startsWhilePaused == 0 && pollsWhilePaused == 0, + pausedSamples + " paused seconds, " + (int) pollsWhilePaused + " polls, " + startsWhilePaused + + " task starts"); + observations.put("pausedSeconds", pausedSamples); + observations.put("peakTarget", run.stream().mapToDouble(s -> s.get("w0.processTarget")).filter(v -> !Double.isNaN(v)).max().orElse(0)); + observations.put("peakHeapMb", run.stream().mapToDouble(s -> s.get("w0.heapUsedMb")).max().orElse(0)); + observations.put("heapMaxMb", run.get(0).get("w0.heapMaxMb")); + observations.put("reasons", reasons(run.get(run.size() - 1))); + } + + // ----- Checks shared by scenarios ----- + + private void commonChecks(Map workflows) throws IOException { + long completed = workflows.values().stream() + .filter(w -> w.getStatus() == Workflow.WorkflowStatus.COMPLETED).count(); + check("all workflows completed", completed == started.size(), completed + "/" + started.size()); + Map terminal = executions().stream().filter(e -> !"IN_PROGRESS".equals(e.status())) + .collect(Collectors.groupingBy(E2eAnalysis.Execution::taskId, Collectors.counting())); + long duplicates = terminal.values().stream().filter(count -> count > 1).count(); + check("no duplicate executions", duplicates == 0, duplicates + " tasks executed more than once"); + E2eSampler.Sample last = sampler.latest(); + // Each idle queue may hold one slot for its standby poll. + int standbySlots = standbyEnabled() ? TASKS.size() : 0; + check("slots released", last.get("w0.processHeld") <= standbySlots && last.get("w0.inflight") == 0, + "held " + last.get("w0.processHeld") + " (standby at most " + standbySlots + "), executing " + + last.get("w0.inflight")); + } + + private void checkPeak(int limit) throws IOException { + List events = new ArrayList<>(); + for (E2eAnalysis.Execution e : executions()) { + events.add(new long[] {e.start(), 1}); + events.add(new long[] {e.end(), -1}); + } + events.sort((a, b) -> a[0] != b[0] ? Long.compare(a[0], b[0]) : Long.compare(a[1], b[1])); + int current = 0; + int peak = 0; + for (long[] event : events) { + current += (int) event[1]; + peak = Math.max(peak, current); + } + check("process concurrency within target", peak <= limit, "peak " + peak + " concurrent (limit " + limit + + ")"); + } + + /** Every queue with remaining backlog starts at least one task in every window. */ + private void checkProgress(long windowMillis) throws IOException { + Map> byType = executions().stream() + .collect(Collectors.groupingBy(E2eAnalysis.Execution::taskType)); + List gaps = new ArrayList<>(); + byType.forEach((type, runs) -> { + List startTimes = runs.stream().map(E2eAnalysis.Execution::start).sorted().toList(); + long previous = goMillis; + for (long start : startTimes) { + if (start - previous > windowMillis) { + gaps.add(type + " idle " + (start - previous) / 1000 + "s"); + } + previous = start; + } + }); + check("every backlogged queue progresses", gaps.isEmpty(), + gaps.isEmpty() ? "no queue went " + windowMillis / 1000 + "s without starting a task" : gaps); + } + + /** The spec's allocation: shares by backlog, water-filled against demand, floors, largest remainder. */ + static int[] expectedShares(int target, long[] backlogs, long[] demands) { + int n = backlogs.length; + int[] shares = new int[n]; + long total = 0; + for (long demand : demands) { + total += demand; + } + if (total <= target) { + for (int i = 0; i < n; i++) { + shares[i] = (int) demands[i]; + } + return shares; + } + boolean[] fixed = new boolean[n]; + double[] quotas = new double[n]; + long remaining = target; + boolean changed = true; + while (changed) { + changed = false; + long weight = 0; + for (int i = 0; i < n; i++) { + weight += fixed[i] ? 0 : backlogs[i]; + } + for (int i = 0; i < n; i++) { + if (fixed[i] || demands[i] == 0) { + continue; + } + quotas[i] = (double) remaining * backlogs[i] / weight; + if (quotas[i] >= demands[i]) { + fixed[i] = true; + shares[i] = (int) demands[i]; + remaining -= demands[i]; + changed = true; + } + } + } + long left = remaining; + List open = new ArrayList<>(); + for (int i = 0; i < n; i++) { + if (!fixed[i] && demands[i] > 0) { + shares[i] = (int) Math.floor(quotas[i]); + left -= shares[i]; + open.add(i); + } + } + for (int i : open) { + if (left > 0 && shares[i] == 0) { + shares[i] = 1; + left--; + } + } + open.sort((a, b) -> Double.compare(quotas[b] - Math.floor(quotas[b]), quotas[a] - Math.floor(quotas[a]))); + for (int i : open) { + if (left > 0 && shares[i] == Math.floor(quotas[i]) && shares[i] < demands[i]) { + shares[i]++; + left--; + } + } + for (int i = 0; i < n; i++) { + shares[i] = demands[i] > 0 ? Math.max(1, shares[i]) : 0; + } + return shares; + } + + // ----- Load ----- + + private void preload(String task, int count, Map input) { + for (int i = 0; i < count; i++) { + start(task, input); + } + awaitStarts(); + } + + private void start(String task, Map input) { + StartWorkflowRequest request = new StartWorkflowRequest(); + request.setName(task + "_wf"); + request.setVersion(1); + request.setInput(new HashMap<>(input)); + if (domains.containsKey(task)) { + request.setTaskToDomain(Map.of(task, domains.get(task) + ",NO_DOMAIN")); + } + starters.submit(() -> { + try { + started.add(workflowClient.startWorkflow(request)); + } catch (Exception e) { + log.error("Failed to start {}: {}", task, e.getMessage()); + } + }); + } + + private void awaitStarts() { + try { + starters.submit(() -> { }).get(); + // A no-op only proves one thread is free; wait until the backlog of starts is gone. + while (((java.util.concurrent.ThreadPoolExecutor) starters).getActiveCount() > 0 + || !((java.util.concurrent.ThreadPoolExecutor) starters).getQueue().isEmpty()) { + Thread.sleep(100); + } + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + private void go() throws Exception { + Thread.sleep(2_000); + log.info("Loaded {} workflows; depths {}", started.size(), sampler.latest() == null ? "-" + : TASKS.stream().collect(Collectors.toMap(t -> t, t -> depth(sampler.latest(), t)))); + goMillis = System.currentTimeMillis(); + sampler.setPhase("run"); + Files.writeString(logDir.resolve("go"), Long.toString(goMillis)); + } + + private void waitForDrain(int timeoutSeconds) throws InterruptedException { + waitForQueuesEmpty(timeoutSeconds); + waitForIdle(60); + } + + private void waitForQueuesEmpty(int timeoutSeconds) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutSeconds * 1000L; + while (System.currentTimeMillis() < deadline) { + Thread.sleep(1_000); + E2eSampler.Sample s = sampler.latest(); + if (s != null && s.epochMillis() > goMillis + 2_000 + && TASKS.stream().allMatch(t -> depth(s, t) == 0)) { + return; + } + } + log.warn("Queues not empty after {}s", timeoutSeconds); + } + + private void waitForIdle(int timeoutSeconds) throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutSeconds * 1000L; + while (System.currentTimeMillis() < deadline) { + Thread.sleep(1_000); + E2eSampler.Sample s = sampler.latest(); + if (s != null && s.get("w0.inflight") == 0 && TASKS.stream().allMatch(t -> depth(s, t) == 0)) { + return; + } + } + } + + private Map fetchWorkflows() throws Exception { + Map workflows = new HashMap<>(); + List> futures = new ArrayList<>(); + ExecutorService fetchers = Executors.newFixedThreadPool(16); + for (String id : started) { + futures.add(fetchers.submit(() -> workflowClient.getWorkflow(id, true))); + } + for (Future future : futures) { + Workflow workflow = future.get(); + workflows.put(workflow.getWorkflowId(), workflow); + } + fetchers.shutdown(); + starters.shutdown(); + this.workflows = workflows; + return workflows; + } + + private Map workflows = Map.of(); + + private List tasks(String taskType) { + try { + if (workflows.isEmpty()) { + fetchWorkflows(); + } + } catch (Exception e) { + throw new IllegalStateException(e); + } + return workflows.values().stream().flatMap(w -> w.getTasks().stream()) + .filter(t -> taskType.equals(t.getTaskDefName())).toList(); + } + + private void registerMetadata() { + List taskDefs = new ArrayList<>(); + List workflowDefs = new ArrayList<>(); + for (String task : TASKS) { + TaskDef taskDef = new TaskDef(task); + taskDef.setOwnerEmail(OWNER); + taskDef.setRetryCount(0); + taskDef.setTimeoutSeconds(3600); + taskDef.setResponseTimeoutSeconds(600); + taskDefs.add(taskDef); + + WorkflowTask workflowTask = new WorkflowTask(); + workflowTask.setName(task); + workflowTask.setTaskReferenceName(task); + workflowTask.setType(TaskType.SIMPLE.name()); + workflowTask.setInputParameters(new HashMap<>(Map.of( + "sleepMs", "${workflow.input.sleepMs}", + "cpuMs", "${workflow.input.cpuMs}", + "allocMb", "${workflow.input.allocMb}"))); + WorkflowDef def = new WorkflowDef(); + def.setName(task + "_wf"); + def.setVersion(1); + def.setOwnerEmail(OWNER); + def.setTasks(List.of(workflowTask)); + workflowDefs.add(def); + } + metadataClient.registerTaskDefs(taskDefs); + metadataClient.updateWorkflowDefs(workflowDefs); + } + + // ----- Helpers ----- + + private String queueName(String task) { + return domains.containsKey(task) ? domains.get(task) + ":" + task : task; + } + + private double depth(E2eSampler.Sample s, String task) { + double value = s.get("depth." + queueName(task)); + return Double.isNaN(value) ? 0 : value; + } + + private static double get(E2eSampler.Sample s, String task, String field) { + double value = s.get("w0." + task + "." + field); + return Double.isNaN(value) ? 0 : value; + } + + private List after(long offsetMillis) { + return sampler.samples().stream().filter(s -> s.epochMillis() >= goMillis + offsetMillis) + .filter(s -> !"final".equals(s.phase())).toList(); + } + + private List samplesIn(String phase) { + return sampler.samples().stream().filter(s -> phase.equals(s.phase())).toList(); + } + + private List executions() throws IOException { + List executions = new ArrayList<>(); + try (Stream files = Files.list(logDir)) { + for (Path file : files.filter(p -> p.getFileName().toString().startsWith("executions-")).toList()) { + List lines = Files.readAllLines(file); + for (String line : lines.subList(1, lines.size())) { + String[] f = line.split(","); + if (f.length == 9) { + executions.add(new E2eAnalysis.Execution(f[0], f[1], Long.parseLong(f[5]), + Long.parseLong(f[6]), f[7], f[8])); + } + } + } + } + return executions; + } + + private static Map reasons(E2eSampler.Sample s) { + Map result = new LinkedHashMap<>(); + s.values().forEach((key, value) -> { + if (key.contains(".reason.") && !key.endsWith("DEPTH_SAMPLE") && !key.endsWith("EMPTY_POLL")) { + result.merge(key.substring(key.indexOf(".reason.") + 8), value, Math::max); + } + }); + return result; + } + + /** Value changes only, as "seconds:value". */ + private List series(List samples, Function value) { + List points = new ArrayList<>(); + Double previous = null; + for (E2eSampler.Sample s : samples) { + Double v = value.apply(s); + if (!v.equals(previous)) { + points.add((s.epochMillis() - goMillis) / 1000 + "s:" + v.intValue()); + previous = v; + } + } + return points; + } + + private void check(String name, boolean passed, Object detail) { + Map check = new LinkedHashMap<>(); + check.put("name", name); + check.put("passed", passed); + check.put("detail", detail); + checks.add(check); + } + + private boolean writeReport(Path outDir) throws IOException { + Map report = new LinkedHashMap<>(); + report.put("scenario", scenario); + report.put("checks", checks); + report.put("observations", observations); + new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT) + .writeValue(outDir.resolve("report.json").toFile(), report); + boolean passed = checks.stream().allMatch(c -> Boolean.TRUE.equals(c.get("passed"))); + System.out.println("==== scenario " + scenario + " ===="); + for (Map c : checks) { + System.out.printf("%-4s %-34s %s%n", Boolean.TRUE.equals(c.get("passed")) ? "PASS" : "FAIL", c.get("name"), + c.get("detail")); + } + if (!observations.isEmpty()) { + System.out.println("observations " + observations); + } + System.out.println("Result " + scenario + ": " + (passed ? "PASS" : "FAIL")); + return passed; + } + + private static double round(double value) { + return Math.round(value * 100) / 100.0; + } +} diff --git a/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eFunctionalWorkerMain.java b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eFunctionalWorkerMain.java new file mode 100644 index 000000000..88349403a --- /dev/null +++ b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eFunctionalWorkerMain.java @@ -0,0 +1,191 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.orkes.conductor.harness.e2e; + +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadMXBean; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.conductor.client.automator.AdaptiveConcurrencyConfig; +import com.netflix.conductor.client.automator.AdaptiveConcurrencyController; +import com.netflix.conductor.client.automator.TaskRunnerConfigurer; +import com.netflix.conductor.client.http.ConductorClient; +import com.netflix.conductor.client.http.TaskClient; +import com.netflix.conductor.client.metrics.prometheus.PrometheusMetricsCollector; +import com.netflix.conductor.client.worker.Worker; +import com.netflix.conductor.common.metadata.tasks.Task; +import com.netflix.conductor.common.metadata.tasks.TaskResult; + +import io.orkes.conductor.client.ApiClient; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; +import io.micrometer.core.instrument.binder.system.ProcessorMetrics; + +/** + * Worker process for the functional scenarios. Each task type gets its own {@link TaskRunnerConfigurer} sharing one + * adaptive controller, so every queue's execution threads carry the task type in their name and can be counted. + * Polling starts once the driver has loaded the queues and created {@code E2E_LOG_DIR/go}. + */ +public class E2eFunctionalWorkerMain { + + private static final Logger log = LoggerFactory.getLogger(E2eFunctionalWorkerMain.class); + + public static void main(String[] args) throws Exception { + String identity = E2eConfig.env("HOSTNAME", "fn-worker"); + List taskTypes = E2eConfig.envList("E2E_TASKS", "fn_a,fn_b,fn_c,fn_d"); + Map domains = E2eConfig.envStringMap("E2E_TASK_DOMAINS", ""); + Path logDir = Path.of(E2eConfig.env("E2E_LOG_DIR", "build/e2e")); + + PrometheusMetricsCollector metricsCollector = new PrometheusMetricsCollector(); + var registry = metricsCollector.getRegistry(); + new ProcessorMetrics().bindTo(registry); + new JvmMemoryMetrics().bindTo(registry); + new JvmGcMetrics().bindTo(registry); + new JvmThreadMetrics().bindTo(registry); + ThreadMXBean threads = ManagementFactory.getThreadMXBean(); + for (String taskType : taskTypes) { + String prefix = threadPrefix(taskType); + Gauge.builder("e2e_pool_threads", threads, bean -> countThreads(bean, prefix)) + .tag("taskType", taskType) + .register(registry); + } + + ConductorClient client = ApiClient.builder() + .useEnvVariables(true) + .readTimeout(10_000) + .connectTimeout(10_000) + .writeTimeout(10_000) + .withMetricsCollector(metricsCollector) + .build(); + ExecutionLog executionLog = new ExecutionLog(logDir.resolve("executions-" + identity + ".csv"), identity, + taskTypes, registry); + + var config = new AdaptiveConcurrencyConfig.Builder() + .withMaxProcessConcurrency(E2eConfig.envInt("E2E_MAX_PROCESS_CONCURRENCY", 256)) + .withInitialProcessConcurrency(E2eConfig.envInt("E2E_INITIAL_PROCESS_CONCURRENCY", 0)) + .withMinProcessConcurrency(E2eConfig.envInt("E2E_MIN_PROCESS_CONCURRENCY", 1)) + .withStandbyPollTimeout(Duration.ofMillis(E2eConfig.envInt("E2E_STANDBY_POLL_TIMEOUT_MS", 1000))) + .withDefaultMaxWorkerConcurrency(E2eConfig.envInt("E2E_MAX_WORKER_CONCURRENCY", 64)) + .build(); + var controller = new AdaptiveConcurrencyController(config); + metricsCollector.startServer(E2eConfig.envInt("E2E_METRICS_PORT", 9991), "/metrics"); + + Path go = logDir.resolve("go"); + log.info("Worker {} waiting for {} (task types {}, domains {}, process ceiling {}, initial {}, worker ceiling {})", + identity, go, taskTypes, domains, config.getMaxProcessConcurrency(), + config.getInitialProcessConcurrency(), config.getDefaultMaxWorkerConcurrency()); + while (!Files.exists(go)) { + Thread.sleep(200); + } + + TaskClient taskClient = new TaskClient(client); + List configurers = new ArrayList<>(); + for (String taskType : taskTypes) { + Worker worker = new LoadWorker(taskType, identity, executionLog); + var builder = new TaskRunnerConfigurer.Builder(taskClient, List.of(worker)) + .withWorkerNamePrefix(threadPrefix(taskType) + "%d") + .withAdaptiveConcurrency(controller); + if (domains.containsKey(taskType)) { + builder.withTaskToDomain(Map.of(taskType, domains.get(taskType))); + } + TaskRunnerConfigurer configurer = builder.build(); + configurer.init(); + configurers.add(configurer); + } + log.info("Worker {} polling", identity); + Runtime.getRuntime().addShutdownHook(new Thread(() -> configurers.forEach(TaskRunnerConfigurer::shutdown))); + Thread.currentThread().join(); + } + + private static String threadPrefix(String taskType) { + return taskType + "-exec-"; + } + + private static double countThreads(ThreadMXBean bean, String prefix) { + int count = 0; + for (var info : bean.getThreadInfo(bean.getAllThreadIds())) { + if (info != null && info.getThreadName().startsWith(prefix)) { + count++; + } + } + return count; + } + + /** + * Does what the task input asks: sleep {@code sleepMs}, burn {@code cpuMs} of CPU, and hold {@code allocMb} of + * heap for the whole execution. + */ + static class LoadWorker extends E2eWorkers.InstrumentedWorker { + + LoadWorker(String taskType, String identity, ExecutionLog executionLog) { + super(taskType, identity, 100, executionLog); + } + + @Override + TaskResult doExecute(Task task) { + Map input = task.getInputData(); + int allocMb = (int) number(input, "allocMb"); + byte[][] payload = new byte[allocMb][]; + for (int i = 0; i < allocMb; i++) { + payload[i] = new byte[1024 * 1024]; + payload[i][i % 1024] = 1; + } + burnCpu(number(input, "cpuMs")); + long sleepMs = number(input, "sleepMs"); + if (sleepMs > 0) { + try { + Thread.sleep(sleepMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + TaskResult result = new TaskResult(task); + result.setStatus(TaskResult.Status.COMPLETED); + result.addOutputData("held", payload.length); + return result; + } + + /** Unset workflow inputs arrive as null values. */ + private static long number(Map input, String key) { + return input.get(key) instanceof Number value ? value.longValue() : 0; + } + + private static void burnCpu(long cpuMs) { + if (cpuMs <= 0) { + return; + } + ThreadMXBean bean = ManagementFactory.getThreadMXBean(); + long end = bean.getCurrentThreadCpuTime() + cpuMs * 1_000_000; + long x = 0; + while (bean.getCurrentThreadCpuTime() < end) { + for (int i = 0; i < 10_000; i++) { + x += i * 31L ^ x; + } + } + if (x == 42) { + log.trace("unlikely"); + } + } + } +} diff --git a/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eSampler.java b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eSampler.java new file mode 100644 index 000000000..830e245c3 --- /dev/null +++ b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eSampler.java @@ -0,0 +1,267 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.orkes.conductor.harness.e2e; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.conductor.client.http.TaskClient; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Samples the system once per second from three independent sources: server queue sizes, the dependency simulator, + * and each worker's Prometheus endpoint. Rows are kept in memory for the analysis and written to a CSV. + */ +class E2eSampler { + + private static final Logger log = LoggerFactory.getLogger(E2eSampler.class); + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final Pattern METRIC_LINE = Pattern.compile("^([a-zA-Z_:][a-zA-Z0-9_:]*)(\\{(.*)})?\\s+(\\S+)$"); + private static final Pattern LABEL = Pattern.compile("([a-zA-Z_][a-zA-Z0-9_]*)=\"([^\"]*)\""); + + private final TaskClient taskClient; + private final List queueNames; + private final String simulatorUrl; + private final List workerUrls; + private final HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(2)).build(); + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "e2e-sampler"); + t.setDaemon(true); + return t; + }); + private final List samples = Collections.synchronizedList(new ArrayList<>()); + private final long startMillis = System.currentTimeMillis(); + private volatile String phase = "setup"; + private volatile Map counters = Map.of(); + + E2eSampler(TaskClient taskClient, String simulatorUrl, List workerUrls) { + this(taskClient, E2eConfig.QUEUE_NAMES, simulatorUrl, workerUrls); + } + + /** + * @param simulatorUrl the dependency simulator, or null when the run has none + */ + E2eSampler(TaskClient taskClient, List queueNames, String simulatorUrl, List workerUrls) { + this.taskClient = taskClient; + this.queueNames = queueNames; + this.simulatorUrl = simulatorUrl; + this.workerUrls = workerUrls; + } + + /** One row: wall-clock time, phase, and named values. */ + record Sample(long epochMillis, double seconds, String phase, Map values) { + + double get(String name) { + return values.getOrDefault(name, Double.NaN); + } + } + + void start() { + scheduler.scheduleAtFixedRate(this::sample, 0, 1, TimeUnit.SECONDS); + } + + void stop() { + scheduler.shutdownNow(); + } + + void setPhase(String phase) { + this.phase = phase; + log.info("Phase {}", phase); + } + + /** Driver-side counters (workflows started, start errors) recorded with each sample. */ + void setCounters(Map counters) { + this.counters = counters; + } + + List samples() { + synchronized (samples) { + return new ArrayList<>(samples); + } + } + + Sample latest() { + synchronized (samples) { + return samples.isEmpty() ? null : samples.get(samples.size() - 1); + } + } + + int workerCount() { + return workerUrls.size(); + } + + private void sample() { + long now = System.currentTimeMillis(); + Map values = new LinkedHashMap<>(counters); + try { + Map sizes = taskClient.getQueueSizesForTasks(queueNames); + for (String queue : queueNames) { + values.put("depth." + queue, sizes.getOrDefault(queue, 0).doubleValue()); + } + } catch (Exception e) { + log.warn("Queue size sample failed: {}", e.getMessage()); + } + if (simulatorUrl != null) { + try { + JsonNode stats = MAPPER.readTree(get(simulatorUrl + "/stats")); + stats.fields().forEachRemaining(endpoint -> endpoint.getValue().fields().forEachRemaining( + field -> values.put("sim." + endpoint.getKey() + "." + field.getKey(), + field.getValue().asDouble()))); + } catch (Exception e) { + log.warn("Simulator sample failed: {}", e.getMessage()); + } + } + for (int i = 0; i < workerUrls.size(); i++) { + try { + sampleWorker("w" + i + ".", get(workerUrls.get(i) + "/metrics"), values); + values.put("w" + i + ".up", 1.0); + } catch (Exception e) { + values.put("w" + i + ".up", 0.0); + log.warn("Worker {} sample failed: {}", i, e.getMessage()); + } + } + samples.add(new Sample(now, (now - startMillis) / 1000.0, phase, values)); + } + + private static void sampleWorker(String prefix, String body, Map values) { + for (String line : body.split("\n")) { + if (line.startsWith("#")) { + continue; + } + Matcher matcher = METRIC_LINE.matcher(line); + if (!matcher.find()) { + continue; + } + String name = matcher.group(1); + Map labels = labels(matcher.group(3)); + double value; + try { + value = Double.parseDouble(matcher.group(4)); + } catch (NumberFormatException e) { + continue; + } + String taskType = labels.getOrDefault("taskType", labels.get("task_type")); + switch (name) { + case "adaptive_process_concurrency_target" -> values.put(prefix + "processTarget", value); + case "adaptive_process_held_slots" -> values.put(prefix + "processHeld", value); + case "adaptive_admission_paused" -> values.put(prefix + "paused", value); + case "process_cpu_usage" -> values.put(prefix + "cpu", value); + case "jvm_threads_live_threads" -> values.put(prefix + "threads", value); + case "e2e_process_inflight" -> values.put(prefix + "inflight", value); + case "e2e_process_inflight_max" -> values.put(prefix + "inflightMax", value); + case "adaptive_worker_concurrency_target" -> values.put(prefix + taskType + ".target", value); + case "adaptive_worker_held_slots" -> values.put(prefix + taskType + ".held", value); + case "adaptive_queue_depth" -> values.put(prefix + taskType + ".depth", value); + case "e2e_inflight" -> values.put(prefix + taskType + ".inflight", value); + case "e2e_inflight_max" -> values.put(prefix + taskType + ".inflightMax", value); + case "e2e_pool_threads" -> values.put(prefix + taskType + ".threads", value); + case "jvm_gc_live_data_size_bytes" -> values.put(prefix + "liveDataBytes", value); + case "jvm_gc_max_data_size_bytes" -> values.put(prefix + "maxDataBytes", value); + case "task_poll_total" -> values.merge(prefix + taskType + ".polls", value, Double::sum); + case "task_execution_started_total" -> values.merge(prefix + taskType + ".started", value, + Double::sum); + case "adaptive_concurrency_changes_total" -> values.merge( + prefix + "reason." + labels.get("reason"), value, Double::sum); + case "jvm_memory_used_bytes" -> { + if ("heap".equals(labels.get("area"))) { + values.merge(prefix + "heapUsedMb", value / 1_048_576, Double::sum); + } + } + case "jvm_memory_max_bytes" -> { + if ("heap".equals(labels.get("area")) && value > 0) { + values.merge(prefix + "heapMaxMb", value / 1_048_576, Double::sum); + } + } + case "jvm_gc_pause_seconds_sum" -> values.merge(prefix + "gcSeconds", value, Double::sum); + case "http_api_client_request_seconds_count" -> { + String uri = labels.getOrDefault("uri", ""); + String key = uri.contains("queue/sizes") ? "req.queueSizes" + : uri.contains("/poll") ? "req.poll" + : uri.startsWith("/tasks") ? "req.update" : "req.other"; + values.merge(prefix + key, value, Double::sum); + } + default -> { + } + } + } + } + + private static Map labels(String text) { + Map labels = new HashMap<>(); + if (text != null) { + Matcher matcher = LABEL.matcher(text); + while (matcher.find()) { + labels.put(matcher.group(1), matcher.group(2)); + } + } + return labels; + } + + private String get(String url) throws IOException, InterruptedException { + HttpResponse response = http.send( + HttpRequest.newBuilder(URI.create(url)).timeout(Duration.ofSeconds(2)).GET().build(), + HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 200) { + throw new IOException(url + " returned " + response.statusCode()); + } + return response.body(); + } + + void writeCsv(Path file) throws IOException { + List rows = samples(); + List columns = new ArrayList<>(); + for (Sample row : rows) { + for (String key : row.values().keySet()) { + if (!columns.contains(key)) { + columns.add(key); + } + } + } + StringBuilder out = new StringBuilder("epochMillis,seconds,phase"); + columns.forEach(column -> out.append(',').append(column)); + out.append('\n'); + for (Sample row : rows) { + out.append(row.epochMillis()).append(',').append(String.format("%.1f", row.seconds())).append(',') + .append(row.phase()); + for (String column : columns) { + Double value = row.values().get(column); + out.append(',').append(value == null ? "" : String.format("%.3f", value)); + } + out.append('\n'); + } + Files.writeString(file, out); + } +} diff --git a/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eWorkerMain.java b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eWorkerMain.java new file mode 100644 index 000000000..f6d49beaa --- /dev/null +++ b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eWorkerMain.java @@ -0,0 +1,109 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.orkes.conductor.harness.e2e; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.conductor.client.automator.AdaptiveConcurrencyConfig; +import com.netflix.conductor.client.automator.AdaptiveConcurrencyController; +import com.netflix.conductor.client.automator.TaskRunnerConfigurer; +import com.netflix.conductor.client.http.ConductorClient; +import com.netflix.conductor.client.http.TaskClient; +import com.netflix.conductor.client.metrics.prometheus.PrometheusMetricsCollector; +import com.netflix.conductor.client.worker.Worker; + +import io.orkes.conductor.client.ApiClient; + +import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; +import io.micrometer.core.instrument.binder.system.ProcessorMetrics; + +/** + * One worker process of the end-to-end test. {@code E2E_MODE=adaptive} uses the adaptive controller; + * {@code E2E_MODE=fixed} uses fixed thread counts from {@code E2E_FIXED_THREADS}. + */ +public class E2eWorkerMain { + + private static final Logger log = LoggerFactory.getLogger(E2eWorkerMain.class); + + public static void main(String[] args) throws Exception { + String mode = E2eConfig.env("E2E_MODE", "adaptive"); + String identity = E2eConfig.env("HOSTNAME", UUID.randomUUID().toString().substring(0, 8)); + int pollingInterval = E2eConfig.envInt("E2E_POLL_INTERVAL_MS", 100); + + PrometheusMetricsCollector metricsCollector = new PrometheusMetricsCollector(); + var registry = metricsCollector.getRegistry(); + new ProcessorMetrics().bindTo(registry); + new JvmMemoryMetrics().bindTo(registry); + new JvmGcMetrics().bindTo(registry); + new JvmThreadMetrics().bindTo(registry); + + ConductorClient client = ApiClient.builder() + .useEnvVariables(true) + .readTimeout(10_000) + .connectTimeout(10_000) + .writeTimeout(10_000) + .withMetricsCollector(metricsCollector) + .build(); + + ExecutionLog executionLog = new ExecutionLog( + Path.of(E2eConfig.env("E2E_LOG_DIR", "build/e2e"), "executions-" + identity + ".csv"), identity, + E2eConfig.TASK_TYPES, registry); + var downstream = new E2eWorkers.Downstream(E2eConfig.env("SIM_URL", "http://localhost:8090")); + List workers = List.of( + new E2eWorkers.FetchProfileWorker(identity, pollingInterval, executionLog, downstream, + E2eConfig.envDouble("E2E_FAILURE_RATE", 0.01)), + new E2eWorkers.ScoreWorker(identity, pollingInterval, executionLog, + E2eConfig.envInt("E2E_SCORE_PASSES", 40)), + new E2eWorkers.PersistWorker(identity, pollingInterval, executionLog, downstream, + E2eConfig.envInt("E2E_DB_POOL", 10)), + new E2eWorkers.NotifyPartnerWorker(identity, pollingInterval, executionLog, downstream, + E2eConfig.envInt("E2E_CALLBACK_SECONDS", 5))); + + var builder = new TaskRunnerConfigurer.Builder(new TaskClient(client), workers) + .withTaskToDomain(Map.of(E2eConfig.NOTIFY_PARTNER, E2eConfig.PARTNER_DOMAIN)) + .withUseVirtualThreads(Boolean.parseBoolean(E2eConfig.env("E2E_VIRTUAL_THREADS", "false"))) + .withUpdateV2(Boolean.parseBoolean(E2eConfig.env("E2E_UPDATE_V2", "false"))); + if ("adaptive".equals(mode)) { + var config = new AdaptiveConcurrencyConfig.Builder() + .withMaxProcessConcurrency(E2eConfig.envInt("E2E_MAX_PROCESS_CONCURRENCY", 256)) + .withInitialProcessConcurrency(E2eConfig.envInt("E2E_INITIAL_PROCESS_CONCURRENCY", 0)) + .withMinProcessConcurrency(E2eConfig.envInt("E2E_MIN_PROCESS_CONCURRENCY", 1)) + .withStandbyPollTimeout(Duration.ofMillis(E2eConfig.envInt("E2E_STANDBY_POLL_TIMEOUT_MS", 1000))) + .withDefaultMaxWorkerConcurrency(E2eConfig.envInt("E2E_MAX_WORKER_CONCURRENCY", 64)) + .build(); + builder.withAdaptiveConcurrency(new AdaptiveConcurrencyController(config)); + log.info("Worker {} running adaptive: process ceiling {}, worker ceiling {}", identity, + config.getMaxProcessConcurrency(), config.getDefaultMaxWorkerConcurrency()); + } else { + Map threads = E2eConfig.envIntMap("E2E_FIXED_THREADS", + "e2e_fetch_profile=8,e2e_score=3,e2e_persist=10,e2e_notify_partner=2"); + builder.withTaskThreadCount(threads); + log.info("Worker {} running fixed threads {}", identity, threads); + } + TaskRunnerConfigurer configurer = builder.build(); + + metricsCollector.startServer(E2eConfig.envInt("E2E_METRICS_PORT", 9991), "/metrics"); + configurer.init(); + Runtime.getRuntime().addShutdownHook(new Thread(configurer::shutdown)); + Thread.currentThread().join(); + } +} diff --git a/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eWorkers.java b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eWorkers.java new file mode 100644 index 000000000..5c314ae16 --- /dev/null +++ b/harness/src/main/java/io/orkes/conductor/harness/e2e/E2eWorkers.java @@ -0,0 +1,268 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.orkes.conductor.harness.e2e; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; + +import com.netflix.conductor.client.worker.Worker; +import com.netflix.conductor.common.metadata.tasks.Task; +import com.netflix.conductor.common.metadata.tasks.TaskResult; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** The four end-to-end workers. Each does real work: HTTP calls, JSON processing or a bounded connection pool. */ +final class E2eWorkers { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private E2eWorkers() {} + + /** Records each execution in the {@link ExecutionLog} around the worker's own logic. */ + abstract static class InstrumentedWorker implements Worker { + + private final String taskDefName; + private final String identity; + private final int pollingInterval; + private final ExecutionLog executionLog; + + InstrumentedWorker(String taskDefName, String identity, int pollingInterval, ExecutionLog executionLog) { + this.taskDefName = taskDefName; + this.identity = identity; + this.pollingInterval = pollingInterval; + this.executionLog = executionLog; + } + + @Override + public String getTaskDefName() { + return taskDefName; + } + + @Override + public String getIdentity() { + return identity; + } + + @Override + public int getPollingInterval() { + return pollingInterval; + } + + @Override + public final TaskResult execute(Task task) { + long startMillis = executionLog.start(task); + String status = "EXCEPTION"; + try { + TaskResult result = doExecute(task); + status = result.getStatus().name(); + return result; + } finally { + executionLog.finish(task, startMillis, status); + } + } + + abstract TaskResult doExecute(Task task); + } + + /** Calls a shared downstream API; a small fraction of calls fail transiently and are retried by the server. */ + static class FetchProfileWorker extends InstrumentedWorker { + + private final Downstream downstream; + private final double failureRate; + + FetchProfileWorker(String identity, int pollingInterval, ExecutionLog executionLog, Downstream downstream, + double failureRate) { + super(E2eConfig.FETCH_PROFILE, identity, pollingInterval, executionLog); + this.downstream = downstream; + this.failureRate = failureRate; + } + + @Override + TaskResult doExecute(Task task) { + if (ThreadLocalRandom.current().nextDouble() < failureRate) { + throw new IllegalStateException("Injected transient failure"); + } + long latencyMicros = downstream.call("profile"); + TaskResult result = new TaskResult(task); + result.setStatus(TaskResult.Status.COMPLETED); + result.addOutputData("customerId", task.getInputData().get("id")); + result.addOutputData("tier", ThreadLocalRandom.current().nextInt(1, 4)); + result.addOutputData("latencyMicros", latencyMicros); + return result; + } + } + + /** Builds, serializes and parses a JSON document whose size comes from the workflow input. */ + static class ScoreWorker extends InstrumentedWorker { + + private final int passes; + + ScoreWorker(String identity, int pollingInterval, ExecutionLog executionLog, int passes) { + super(E2eConfig.SCORE, identity, pollingInterval, executionLog); + this.passes = passes; + } + + @Override + TaskResult doExecute(Task task) { + int docKb = ((Number) task.getInputData().getOrDefault("docKb", 64)).intValue(); + long seed = task.getTaskId().hashCode(); + double score = 0; + List sample = new ArrayList<>(); + try { + for (int pass = 0; pass < passes; pass++) { + byte[] document = MAPPER.writeValueAsBytes(document(docKb, seed + pass)); + JsonNode parsed = MAPPER.readTree(document); + for (JsonNode record : parsed.get("records")) { + score += record.get("amount").asDouble() * (record.get("name").asText().hashCode() & 7); + if (sample.size() < 10) { + sample.add(record); + } + } + } + } catch (IOException e) { + throw new IllegalStateException(e); + } + TaskResult result = new TaskResult(task); + result.setStatus(TaskResult.Status.COMPLETED); + result.addOutputData("score", score); + result.addOutputData("sample", MAPPER.convertValue(sample, List.class)); + return result; + } + + private static ObjectNode document(int docKb, long seed) { + Random random = new Random(seed); + ObjectNode root = MAPPER.createObjectNode(); + ArrayNode records = root.putArray("records"); + int count = Math.max(1, docKb * 1024 / 160); + for (int i = 0; i < count; i++) { + ObjectNode record = records.addObject(); + record.put("id", i); + record.put("name", "item-" + Long.toHexString(random.nextLong())); + record.put("amount", random.nextDouble() * 1000); + record.putArray("tags").add("t" + random.nextInt(50)).add("t" + random.nextInt(50)); + } + return root; + } + } + + /** Writes through a fixed-size connection pool, like a JDBC pool in front of a database. */ + static class PersistWorker extends InstrumentedWorker { + + private final Downstream downstream; + private final Semaphore connections; + + PersistWorker(String identity, int pollingInterval, ExecutionLog executionLog, Downstream downstream, + int poolSize) { + super(E2eConfig.PERSIST, identity, pollingInterval, executionLog); + this.downstream = downstream; + this.connections = new Semaphore(poolSize, true); + } + + @Override + TaskResult doExecute(Task task) { + try { + if (!connections.tryAcquire(30, TimeUnit.SECONDS)) { + throw new IllegalStateException("Timed out waiting for a connection"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + try { + downstream.call("persist"); + } finally { + connections.release(); + } + TaskResult result = new TaskResult(task); + result.setStatus(TaskResult.Status.COMPLETED); + result.addOutputData("persisted", true); + return result; + } + } + + /** Long-running pattern: reports IN_PROGRESS with a callback until the partner acknowledges, then completes. */ + static class NotifyPartnerWorker extends InstrumentedWorker { + + private final Downstream downstream; + private final int callbackAfterSeconds; + + NotifyPartnerWorker(String identity, int pollingInterval, ExecutionLog executionLog, Downstream downstream, + int callbackAfterSeconds) { + super(E2eConfig.NOTIFY_PARTNER, identity, pollingInterval, executionLog); + this.downstream = downstream; + this.callbackAfterSeconds = callbackAfterSeconds; + } + + @Override + TaskResult doExecute(Task task) { + downstream.call("profile"); + TaskResult result = new TaskResult(task); + Map output = task.getOutputData(); + if (output == null || !output.containsKey("sentAt")) { + result.setStatus(TaskResult.Status.IN_PROGRESS); + result.setCallbackAfterSeconds(callbackAfterSeconds); + result.addOutputData("sentAt", System.currentTimeMillis()); + } else { + result.setStatus(TaskResult.Status.COMPLETED); + result.addOutputData("sentAt", output.get("sentAt")); + result.addOutputData("ackedAt", System.currentTimeMillis()); + } + return result; + } + } + + /** HTTP client for the {@link DependencySimulator}. */ + static class Downstream { + + private final HttpClient client = + HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); + private final String baseUrl; + + Downstream(String baseUrl) { + this.baseUrl = baseUrl; + } + + long call(String endpoint) { + HttpRequest request = HttpRequest.newBuilder(URI.create(baseUrl + "/" + endpoint)) + .timeout(Duration.ofSeconds(30)) + .GET() + .build(); + try { + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 200) { + throw new IllegalStateException("Downstream " + endpoint + " returned " + response.statusCode()); + } + return MAPPER.readTree(response.body()).get("latencyMicros").asLong(); + } catch (IOException e) { + throw new IllegalStateException("Downstream " + endpoint + " failed", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + } +} diff --git a/harness/src/main/java/io/orkes/conductor/harness/e2e/ExecutionLog.java b/harness/src/main/java/io/orkes/conductor/harness/e2e/ExecutionLog.java new file mode 100644 index 000000000..625972b6c --- /dev/null +++ b/harness/src/main/java/io/orkes/conductor/harness/e2e/ExecutionLog.java @@ -0,0 +1,116 @@ +/* + * Copyright 2026 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package io.orkes.conductor.harness.e2e; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.conductor.common.metadata.tasks.Task; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; + +/** + * Records every execution independently of the SDK: live and peak concurrency as gauges, and one CSV line per + * execution so the driver can detect duplicate executions and recompute concurrency after the run. + */ +class ExecutionLog { + + static final String HEADER = "taskId,taskType,workflowId,retryCount,pollCount,startMillis,endMillis,status,worker"; + + private static final Logger log = LoggerFactory.getLogger(ExecutionLog.class); + + private final String worker; + private final BlockingQueue lines = new LinkedBlockingQueue<>(); + private final Map inFlight = new ConcurrentHashMap<>(); + private final Map maxInFlight = new ConcurrentHashMap<>(); + private final AtomicInteger totalInFlight = new AtomicInteger(); + private final AtomicInteger totalMaxInFlight = new AtomicInteger(); + + ExecutionLog(Path file, String worker, List taskTypes, MeterRegistry registry) { + this.worker = worker; + for (String taskType : taskTypes) { + AtomicInteger current = inFlight.computeIfAbsent(taskType, k -> new AtomicInteger()); + AtomicInteger max = maxInFlight.computeIfAbsent(taskType, k -> new AtomicInteger()); + Gauge.builder("e2e_inflight", current, AtomicInteger::doubleValue).tag("taskType", taskType) + .register(registry); + Gauge.builder("e2e_inflight_max", max, AtomicInteger::doubleValue).tag("taskType", taskType) + .register(registry); + } + Gauge.builder("e2e_process_inflight", totalInFlight, AtomicInteger::doubleValue).register(registry); + Gauge.builder("e2e_process_inflight_max", totalMaxInFlight, AtomicInteger::doubleValue).register(registry); + Thread writer = new Thread(() -> write(file), "e2e-execution-log"); + writer.setDaemon(true); + writer.start(); + } + + long start(Task task) { + int current = inFlight.get(task.getTaskDefName()).incrementAndGet(); + maxInFlight.get(task.getTaskDefName()).accumulateAndGet(current, Math::max); + totalMaxInFlight.accumulateAndGet(totalInFlight.incrementAndGet(), Math::max); + return System.currentTimeMillis(); + } + + void finish(Task task, long startMillis, String status) { + long endMillis = System.currentTimeMillis(); + inFlight.get(task.getTaskDefName()).decrementAndGet(); + totalInFlight.decrementAndGet(); + lines.add(String.join(",", task.getTaskId(), task.getTaskDefName(), task.getWorkflowInstanceId(), + Integer.toString(task.getRetryCount()), Integer.toString(task.getPollCount()), + Long.toString(startMillis), Long.toString(endMillis), status, worker)); + } + + private void write(Path file) { + try { + Files.createDirectories(file.getParent()); + try (BufferedWriter out = Files.newBufferedWriter(file, StandardCharsets.UTF_8)) { + out.write(HEADER); + out.newLine(); + List batch = new ArrayList<>(); + while (true) { + String line = lines.poll(1, TimeUnit.SECONDS); + if (line != null) { + batch.add(line); + lines.drainTo(batch); + for (String entry : batch) { + out.write(entry); + out.newLine(); + } + batch.clear(); + } + out.flush(); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (IOException e) { + log.error("Execution log writer failed", e); + throw new UncheckedIOException(e); + } + } +} diff --git a/harness/src/main/resources/logback.xml b/harness/src/main/resources/logback.xml index 0aab35aab..e2b82e27c 100644 --- a/harness/src/main/resources/logback.xml +++ b/harness/src/main/resources/logback.xml @@ -5,6 +5,9 @@ + +