deletedOrNotFound() {
+ return deletedOrNotFound;
+ }
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/BatchFileDeleter.java b/paimon-common/src/main/java/org/apache/paimon/fs/BatchFileDeleter.java
new file mode 100644
index 000000000000..bfc0996ea81f
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/fs/BatchFileDeleter.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fs;
+
+import org.apache.paimon.annotation.Public;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Deletes files in one provider request without falling back to individual deletes.
+ *
+ * A successful invocation confirms every requested file as deleted or not found. A failure or
+ * timeout only means that the complete batch was not confirmed; the provider may already have
+ * deleted some files. If a caller retries, it must retry the same complete batch. Implementations
+ * must validate the complete request before accessing storage.
+ *
+ * @since 2.1
+ */
+@Public
+public interface BatchFileDeleter {
+
+ /** Maximum number of files accepted by one {@link #delete(List)} invocation. */
+ int maxBatchSize();
+
+ /**
+ * Deletes one non-empty batch.
+ *
+ * @return files confirmed deleted or not found
+ * @throws IOException if any requested file cannot be confirmed
+ */
+ BatchDeleteResult delete(List files) throws IOException;
+}
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java
index 2b0dcec3f760..98d386c702a4 100644
--- a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java
+++ b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java
@@ -217,6 +217,18 @@ default FileStatus[] listDirectories(Path path) throws IOException {
*/
boolean exists(Path path) throws IOException;
+ /**
+ * Returns a strict batch-delete capability for the provider serving the given path.
+ *
+ * An empty result is the only signal that callers may use individual deletes instead. The
+ * default performs no storage access and preserves compatibility with existing providers.
+ *
+ * @since 2.1
+ */
+ default Optional batchFileDeleter(Path path) throws IOException {
+ return Optional.empty();
+ }
+
/**
* Delete a file.
*
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java
index 587c1f2d4423..24d8868d1eec 100644
--- a/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java
+++ b/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java
@@ -24,6 +24,9 @@
import java.io.IOException;
import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Supplier;
/**
* A {@link FileIO} for plugin jar. {@link FileIO} is serializable, so plugin FileIO should be
@@ -78,6 +81,28 @@ public boolean exists(Path path) throws IOException {
return wrap(() -> fileIO(path).exists(path));
}
+ @Override
+ public Optional batchFileDeleter(Path path) throws IOException {
+ Optional capability = wrap(() -> fileIO(path).batchFileDeleter(path));
+ if (!capability.isPresent()) {
+ return Optional.empty();
+ }
+
+ BatchFileDeleter delegate = capability.get();
+ return Optional.of(
+ new BatchFileDeleter() {
+ @Override
+ public int maxBatchSize() {
+ return wrapUnchecked(delegate::maxBatchSize);
+ }
+
+ @Override
+ public BatchDeleteResult delete(List files) throws IOException {
+ return wrap(() -> delegate.delete(files));
+ }
+ });
+ }
+
@Override
public boolean delete(Path path, boolean recursive) throws IOException {
return wrap(() -> fileIO(path).delete(path, recursive));
@@ -132,6 +157,16 @@ private T wrap(Func func) throws IOException {
}
}
+ private T wrapUnchecked(Supplier supplier) {
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(pluginClassLoader());
+ return supplier.get();
+ } finally {
+ Thread.currentThread().setContextClassLoader(cl);
+ }
+ }
+
/** Apply function with wrapping classloader. */
@FunctionalInterface
protected interface Func {
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java
index 5568ba896cb3..f2475d985534 100644
--- a/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java
+++ b/paimon-common/src/main/java/org/apache/paimon/fs/ResolvingFileIO.java
@@ -26,10 +26,14 @@
import java.io.IOException;
import java.io.Serializable;
+import java.net.URI;
import java.time.Duration;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Supplier;
import static org.apache.paimon.options.CatalogOptions.RESOLVING_FILE_IO_ENABLED;
@@ -94,6 +98,30 @@ public boolean exists(Path path) throws IOException {
return wrap(() -> fileIO(path).exists(path));
}
+ @Override
+ public Optional batchFileDeleter(Path path) throws IOException {
+ Optional capability = wrap(() -> fileIO(path).batchFileDeleter(path));
+ if (!capability.isPresent()) {
+ return Optional.empty();
+ }
+
+ URI provider = path.toUri();
+ BatchFileDeleter delegate = capability.get();
+ return Optional.of(
+ new BatchFileDeleter() {
+ @Override
+ public int maxBatchSize() {
+ return wrapUnchecked(delegate::maxBatchSize);
+ }
+
+ @Override
+ public BatchDeleteResult delete(List files) throws IOException {
+ validateProvider(files, provider);
+ return wrap(() -> delegate.delete(files));
+ }
+ });
+ }
+
@Override
public boolean delete(Path path, boolean recursive) throws IOException {
return wrap(() -> fileIO(path).delete(path, recursive));
@@ -149,6 +177,30 @@ private T wrap(Func func) throws IOException {
}
}
+ private T wrapUnchecked(Supplier supplier) {
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(ResolvingFileIO.class.getClassLoader());
+ return supplier.get();
+ } finally {
+ Thread.currentThread().setContextClassLoader(cl);
+ }
+ }
+
+ private static void validateProvider(List files, URI provider) {
+ if (files == null) {
+ throw new IllegalArgumentException("Batch delete files must not be null.");
+ }
+ for (Path file : files) {
+ if (file == null
+ || !Objects.equals(provider.getScheme(), file.toUri().getScheme())
+ || !Objects.equals(provider.getAuthority(), file.toUri().getAuthority())) {
+ throw new IllegalArgumentException(
+ "Batch delete files must use the capability provider's scheme and authority.");
+ }
+ }
+ }
+
/** Apply function with wrapping classloader. */
@FunctionalInterface
protected interface Func {
diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java
index 65eeaa3ebfd7..e21e237769ec 100644
--- a/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java
+++ b/paimon-common/src/main/java/org/apache/paimon/fs/cache/CachingFileIO.java
@@ -20,6 +20,7 @@
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.data.BlobDescriptor;
+import org.apache.paimon.fs.BatchFileDeleter;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.FileStatus;
import org.apache.paimon.fs.Path;
@@ -40,6 +41,7 @@
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
@@ -165,6 +167,11 @@ public boolean exists(Path path) throws IOException {
return delegate.exists(path);
}
+ @Override
+ public Optional batchFileDeleter(Path path) throws IOException {
+ return delegate.batchFileDeleter(path);
+ }
+
@Override
public boolean delete(Path path, boolean recursive) throws IOException {
return delegate.delete(path, recursive);
diff --git a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java
index fb210dda435f..982cad5818f0 100644
--- a/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java
+++ b/paimon-common/src/main/java/org/apache/paimon/rest/RESTTokenFileIO.java
@@ -21,6 +21,8 @@
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.data.BlobDescriptor;
+import org.apache.paimon.fs.BatchDeleteResult;
+import org.apache.paimon.fs.BatchFileDeleter;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.FileStatus;
import org.apache.paimon.fs.Path;
@@ -47,7 +49,9 @@
import java.io.IOException;
import java.io.UncheckedIOException;
import java.time.Duration;
+import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@@ -145,6 +149,33 @@ public boolean exists(Path path) throws IOException {
return fileIO().exists(path);
}
+ @Override
+ public Optional batchFileDeleter(Path path) throws IOException {
+ Optional capability = fileIO().batchFileDeleter(path);
+ if (!capability.isPresent()) {
+ return Optional.empty();
+ }
+
+ int maxBatchSize = capability.get().maxBatchSize();
+ return Optional.of(
+ new BatchFileDeleter() {
+ @Override
+ public int maxBatchSize() {
+ return maxBatchSize;
+ }
+
+ @Override
+ public BatchDeleteResult delete(List files) throws IOException {
+ Optional current = fileIO().batchFileDeleter(path);
+ if (!current.isPresent()) {
+ throw new IOException(
+ "Batch delete capability is unavailable after refreshing credentials.");
+ }
+ return current.get().delete(files);
+ }
+ });
+ }
+
@Override
public boolean delete(Path path, boolean recursive) throws IOException {
return fileIO().delete(path, recursive);
diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/FileIOBatchDeleteContractTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/FileIOBatchDeleteContractTest.java
new file mode 100644
index 000000000000..8f3d5cd2815d
--- /dev/null
+++ b/paimon-common/src/test/java/org/apache/paimon/fs/FileIOBatchDeleteContractTest.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fs;
+
+import org.apache.paimon.catalog.CatalogContext;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import javax.tools.JavaCompiler;
+import javax.tools.ToolProvider;
+
+import java.io.IOException;
+import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Public contract and binary compatibility tests for strict batch delete. */
+class FileIOBatchDeleteContractTest {
+
+ private static final Path FIRST = new Path("oss://bucket/table/a.parquet");
+ private static final Path SECOND = new Path("oss://bucket/table/b.parquet");
+
+ @TempDir java.nio.file.Path tempDir;
+
+ @Test
+ void testLegacyImplementationUsesDefaultUnsupportedWithoutStorageAccess() throws Exception {
+ LegacyFileIO legacy = new LegacyFileIO();
+
+ Optional capability = legacy.batchFileDeleter(FIRST);
+
+ assertThat(capability).isEmpty();
+ assertThat(legacy.storageCalls).hasValue(0);
+ }
+
+ @Test
+ void testBatchDeleteResultDefensivelyCopiesAndDoesNotExposeMutableState() {
+ List callerOwned = new ArrayList<>(Arrays.asList(FIRST, SECOND));
+
+ BatchDeleteResult result = new BatchDeleteResult(callerOwned);
+ callerOwned.clear();
+
+ assertThat(result.deletedOrNotFound()).containsExactly(FIRST, SECOND);
+ assertThatThrownBy(() -> result.deletedOrNotFound().add(FIRST))
+ .isInstanceOf(UnsupportedOperationException.class);
+ assertThatThrownBy(() -> result.deletedOrNotFound().set(0, SECOND))
+ .isInstanceOf(UnsupportedOperationException.class);
+ assertThat(result.deletedOrNotFound()).containsExactly(FIRST, SECOND);
+ }
+
+ @Test
+ void testProviderCompiledAgainstOldInterfaceLoadsAndUsesNewDefaultMethod() throws Exception {
+ java.nio.file.Path sources = Files.createDirectories(tempDir.resolve("sources"));
+ java.nio.file.Path oldApiClasses = Files.createDirectories(tempDir.resolve("old-api"));
+ java.nio.file.Path providerClasses = Files.createDirectories(tempDir.resolve("provider"));
+ java.nio.file.Path oldInterface =
+ writeSource(
+ sources,
+ "org/apache/paimon/fs/FileIO.java",
+ "package org.apache.paimon.fs;\n"
+ + "public interface FileIO extends java.io.Serializable {}\n");
+ java.nio.file.Path oldProvider =
+ writeSource(
+ sources,
+ "fixture/LegacyProvider.java",
+ "package fixture;\n"
+ + "public final class LegacyProvider "
+ + "implements org.apache.paimon.fs.FileIO {\n"
+ + " public LegacyProvider() {}\n"
+ + "}\n");
+ JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
+ assertThat(compiler).as("Maven tests must run on a JDK").isNotNull();
+ assertThat(
+ compiler.run(
+ null,
+ null,
+ null,
+ "-d",
+ oldApiClasses.toString(),
+ oldInterface.toString()))
+ .isZero();
+ assertThat(
+ compiler.run(
+ null,
+ null,
+ null,
+ "-classpath",
+ oldApiClasses.toString(),
+ "-d",
+ providerClasses.toString(),
+ oldProvider.toString()))
+ .isZero();
+
+ // Parent-first loading replaces the compile-time interface with the current FileIO while
+ // retaining provider bytecode compiled without the new method.
+ try (URLClassLoader loader =
+ new URLClassLoader(
+ new java.net.URL[] {providerClasses.toUri().toURL()},
+ FileIO.class.getClassLoader())) {
+ Class providerClass = Class.forName("fixture.LegacyProvider", true, loader);
+ assertThat(providerClass.getInterfaces()).containsExactly(FileIO.class);
+ FileIO provider = (FileIO) providerClass.getDeclaredConstructor().newInstance();
+
+ assertThat(FileIO.class.getMethod("batchFileDeleter", Path.class).isDefault()).isTrue();
+ assertThat(provider.batchFileDeleter(FIRST)).isEmpty();
+ }
+ }
+
+ private static java.nio.file.Path writeSource(
+ java.nio.file.Path root, String relative, String source) throws IOException {
+ java.nio.file.Path file = root.resolve(relative);
+ Files.createDirectories(file.getParent());
+ Files.write(file, source.getBytes(StandardCharsets.UTF_8));
+ return file;
+ }
+
+ /**
+ * This fixture intentionally does not override batchFileDeleter. Every observable storage
+ * method fails, so even a harmless-looking capability probe has causal evidence.
+ */
+ private static class LegacyFileIO implements FileIO {
+
+ private final AtomicInteger storageCalls = new AtomicInteger();
+
+ @Override
+ public boolean isObjectStore() {
+ return true;
+ }
+
+ @Override
+ public void configure(CatalogContext context) {}
+
+ @Override
+ public SeekableInputStream newInputStream(Path path) {
+ return unexpectedStorageCall("newInputStream");
+ }
+
+ @Override
+ public PositionOutputStream newOutputStream(Path path, boolean overwrite) {
+ return unexpectedStorageCall("newOutputStream");
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path path) {
+ return unexpectedStorageCall("getFileStatus");
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path path) {
+ return unexpectedStorageCall("listStatus");
+ }
+
+ @Override
+ public boolean exists(Path path) {
+ return unexpectedStorageCall("exists");
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive) {
+ return unexpectedStorageCall("delete");
+ }
+
+ @Override
+ public boolean mkdirs(Path path) {
+ return unexpectedStorageCall("mkdirs");
+ }
+
+ @Override
+ public boolean rename(Path src, Path dst) {
+ return unexpectedStorageCall("rename");
+ }
+
+ private T unexpectedStorageCall(String operation) {
+ storageCalls.incrementAndGet();
+ throw new AssertionError("Default capability accessed storage through " + operation);
+ }
+ }
+}
diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/FileIOBatchDeleteForwardingTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/FileIOBatchDeleteForwardingTest.java
new file mode 100644
index 000000000000..abb5f25cf555
--- /dev/null
+++ b/paimon-common/src/test/java/org/apache/paimon/fs/FileIOBatchDeleteForwardingTest.java
@@ -0,0 +1,843 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.fs;
+
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.cache.CachingFileIO;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.rest.RESTApi;
+import org.apache.paimon.rest.RESTTokenFileIO;
+import org.apache.paimon.rest.responses.GetTableTokenResponse;
+import org.apache.paimon.utils.FileType;
+import org.apache.paimon.utils.InstantiationUtil;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/** Contract tests for forwarding strict batch-delete capabilities through FileIO wrappers. */
+class FileIOBatchDeleteForwardingTest {
+
+ private static final Path FIRST = new Path("oss://bucket/table/a.parquet");
+ private static final Path SECOND = new Path("oss://bucket/table/b.parquet");
+ private static final List FILES = Arrays.asList(FIRST, SECOND);
+
+ @Test
+ void testPluginForwardsSupportedCapabilityUnderPluginClassLoader() throws Exception {
+ FileIO delegate = mock(FileIO.class);
+ ClassLoader pluginClassLoader = new ClassLoader() {};
+ ClassLoader original = Thread.currentThread().getContextClassLoader();
+ BatchDeleteResult expected = result(FILES);
+ BatchFileDeleter inner =
+ new BatchFileDeleter() {
+ @Override
+ public int maxBatchSize() {
+ assertThat(Thread.currentThread().getContextClassLoader())
+ .isSameAs(pluginClassLoader);
+ return 1000;
+ }
+
+ @Override
+ public BatchDeleteResult delete(List files) {
+ assertThat(Thread.currentThread().getContextClassLoader())
+ .isSameAs(pluginClassLoader);
+ assertThat(files).containsExactlyElementsOf(FILES);
+ return expected;
+ }
+ };
+ when(delegate.batchFileDeleter(FIRST))
+ .thenAnswer(
+ ignored -> {
+ assertThat(Thread.currentThread().getContextClassLoader())
+ .isSameAs(pluginClassLoader);
+ return Optional.of(inner);
+ });
+ TestPluginFileIO plugin = new TestPluginFileIO(delegate, pluginClassLoader);
+
+ // A broken TCCL restore can poison later ServiceLoader tests in the same worker, so the
+ // fixture restores the caller loader independently of the production finally block.
+ try {
+ BatchFileDeleter forwarded =
+ plugin.batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+
+ assertThat(forwarded.maxBatchSize()).isEqualTo(1000);
+ assertThat(forwarded.delete(FILES)).isSameAs(expected);
+ assertThat(Thread.currentThread().getContextClassLoader()).isSameAs(original);
+ verify(delegate).batchFileDeleter(FIRST);
+ verify(delegate, never()).delete(any(), anyBoolean());
+ } finally {
+ Thread.currentThread().setContextClassLoader(original);
+ }
+ }
+
+ @Test
+ void testPluginForwardsUnsupportedCapability() throws Exception {
+ FileIO delegate = mock(FileIO.class);
+ ClassLoader pluginClassLoader = new ClassLoader() {};
+ TestPluginFileIO plugin = new TestPluginFileIO(delegate, pluginClassLoader);
+ when(delegate.batchFileDeleter(FIRST)).thenReturn(Optional.empty());
+ ClassLoader previous = Thread.currentThread().getContextClassLoader();
+
+ try {
+ assertThat(plugin.batchFileDeleter(FIRST)).isEmpty();
+ assertThat(Thread.currentThread().getContextClassLoader()).isSameAs(previous);
+ verify(delegate, never()).delete(any(), anyBoolean());
+ } finally {
+ Thread.currentThread().setContextClassLoader(previous);
+ }
+ }
+
+ @Test
+ void testPluginPropagatesDiscoveryFailureAndRestoresCallerClassLoader() throws Exception {
+ FileIO delegate = mock(FileIO.class);
+ ClassLoader pluginClassLoader = new ClassLoader() {};
+ ClassLoader previous = Thread.currentThread().getContextClassLoader();
+ ClassLoader callerClassLoader = new ClassLoader(previous) {};
+ IOException failure = new IOException("plugin discovery failed");
+ when(delegate.batchFileDeleter(FIRST))
+ .thenAnswer(
+ ignored -> {
+ assertThat(Thread.currentThread().getContextClassLoader())
+ .isSameAs(pluginClassLoader);
+ throw failure;
+ });
+ TestPluginFileIO plugin = new TestPluginFileIO(delegate, pluginClassLoader);
+
+ Thread.currentThread().setContextClassLoader(callerClassLoader);
+ try {
+ assertThatThrownBy(() -> plugin.batchFileDeleter(FIRST)).isSameAs(failure);
+ assertThat(Thread.currentThread().getContextClassLoader()).isSameAs(callerClassLoader);
+ verify(delegate, never()).delete(any(), anyBoolean());
+ } finally {
+ Thread.currentThread().setContextClassLoader(previous);
+ }
+ }
+
+ @Test
+ void testPluginPropagatesDeleteFailureAndRestoresCallerClassLoader() throws Exception {
+ FileIO delegate = mock(FileIO.class);
+ ClassLoader pluginClassLoader = new ClassLoader() {};
+ ClassLoader previous = Thread.currentThread().getContextClassLoader();
+ ClassLoader callerClassLoader = new ClassLoader(previous) {};
+ IOException failure = new IOException("plugin batch failed");
+ when(delegate.batchFileDeleter(FIRST))
+ .thenReturn(
+ Optional.of(
+ new BatchFileDeleter() {
+ @Override
+ public int maxBatchSize() {
+ assertThat(Thread.currentThread().getContextClassLoader())
+ .isSameAs(pluginClassLoader);
+ return 1000;
+ }
+
+ @Override
+ public BatchDeleteResult delete(List files)
+ throws IOException {
+ assertThat(Thread.currentThread().getContextClassLoader())
+ .isSameAs(pluginClassLoader);
+ throw failure;
+ }
+ }));
+ TestPluginFileIO plugin = new TestPluginFileIO(delegate, pluginClassLoader);
+
+ Thread.currentThread().setContextClassLoader(callerClassLoader);
+ try {
+ BatchFileDeleter forwarded =
+ plugin.batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+ assertThat(forwarded.maxBatchSize()).isEqualTo(1000);
+ assertThatThrownBy(() -> forwarded.delete(FILES)).isSameAs(failure);
+ assertThat(Thread.currentThread().getContextClassLoader()).isSameAs(callerClassLoader);
+ verify(delegate, never()).delete(any(), anyBoolean());
+ } finally {
+ Thread.currentThread().setContextClassLoader(previous);
+ }
+ }
+
+ @Test
+ void testPluginPropagatesMaxBatchSizeFailureAndRestoresCallerClassLoader() throws Exception {
+ FileIO delegate = mock(FileIO.class);
+ ClassLoader pluginClassLoader = new ClassLoader() {};
+ ClassLoader previous = Thread.currentThread().getContextClassLoader();
+ ClassLoader callerClassLoader = new ClassLoader(previous) {};
+ RuntimeException failure = new RuntimeException("plugin max batch size failed");
+ when(delegate.batchFileDeleter(FIRST))
+ .thenReturn(
+ Optional.of(
+ new BatchFileDeleter() {
+ @Override
+ public int maxBatchSize() {
+ assertThat(Thread.currentThread().getContextClassLoader())
+ .isSameAs(pluginClassLoader);
+ throw failure;
+ }
+
+ @Override
+ public BatchDeleteResult delete(List files) {
+ throw new AssertionError("delete must not be called");
+ }
+ }));
+ TestPluginFileIO plugin = new TestPluginFileIO(delegate, pluginClassLoader);
+
+ Thread.currentThread().setContextClassLoader(callerClassLoader);
+ try {
+ BatchFileDeleter forwarded =
+ plugin.batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+ assertThatThrownBy(forwarded::maxBatchSize).isSameAs(failure);
+ assertThat(Thread.currentThread().getContextClassLoader()).isSameAs(callerClassLoader);
+ verify(delegate, never()).delete(any(), anyBoolean());
+ } finally {
+ Thread.currentThread().setContextClassLoader(previous);
+ }
+ }
+
+ @Test
+ void testResolvingForwardsSupportedAndUnsupportedCapabilities() throws Exception {
+ FileIO supportedDelegate = mock(FileIO.class);
+ BatchDeleteResult expected = result(FILES);
+ BatchFileDeleter inner = deleter(1000, files -> expected);
+ when(supportedDelegate.batchFileDeleter(FIRST)).thenReturn(Optional.of(inner));
+ ResolvingFileIO supported = resolving(supportedDelegate);
+
+ BatchFileDeleter forwarded =
+ supported.batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+ assertThat(forwarded.maxBatchSize()).isEqualTo(1000);
+ assertThat(forwarded.delete(FILES)).isSameAs(expected);
+
+ FileIO unsupportedDelegate = mock(FileIO.class);
+ when(unsupportedDelegate.batchFileDeleter(FIRST)).thenReturn(Optional.empty());
+ assertThat(resolving(unsupportedDelegate).batchFileDeleter(FIRST)).isEmpty();
+ }
+
+ @Test
+ void testResolvingRejectsMixedAuthorityBeforeProviderInvocation() throws Exception {
+ FileIO delegate = mock(FileIO.class);
+ AtomicInteger providerCalls = new AtomicInteger();
+ when(delegate.batchFileDeleter(FIRST))
+ .thenReturn(
+ Optional.of(
+ deleter(
+ 1000,
+ files -> {
+ providerCalls.incrementAndGet();
+ return result(files);
+ })));
+ BatchFileDeleter forwarded =
+ resolving(delegate).batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+
+ assertThatThrownBy(
+ () ->
+ forwarded.delete(
+ Arrays.asList(
+ FIRST,
+ new Path("oss://other-bucket/table/b.parquet"))))
+ .isInstanceOfAny(IllegalArgumentException.class, IOException.class);
+ assertThat(providerCalls).hasValue(0);
+ verify(delegate, never()).delete(any(), anyBoolean());
+ }
+
+ @Test
+ void testResolvingRejectsMixedSchemeBeforeProviderInvocation() throws Exception {
+ FileIO delegate = mock(FileIO.class);
+ AtomicInteger providerCalls = new AtomicInteger();
+ when(delegate.batchFileDeleter(FIRST))
+ .thenReturn(
+ Optional.of(
+ deleter(
+ 1000,
+ files -> {
+ providerCalls.incrementAndGet();
+ return result(files);
+ })));
+ BatchFileDeleter forwarded =
+ resolving(delegate).batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+
+ assertThatThrownBy(
+ () ->
+ forwarded.delete(
+ Arrays.asList(
+ FIRST, new Path("s3://bucket/table/b.parquet"))))
+ .isInstanceOfAny(IllegalArgumentException.class, IOException.class);
+ assertThat(providerCalls).hasValue(0);
+ verify(delegate, never()).delete(any(), anyBoolean());
+ }
+
+ @Test
+ void testResolvingPropagatesProviderFailureWithoutFallback() throws Exception {
+ FileIO delegate = mock(FileIO.class);
+ IOException failure = new IOException("resolved batch failed");
+ when(delegate.batchFileDeleter(FIRST))
+ .thenReturn(Optional.of(deleter(1000, files -> raise(failure))));
+ BatchFileDeleter forwarded =
+ resolving(delegate).batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+
+ assertThatThrownBy(() -> forwarded.delete(FILES)).isSameAs(failure);
+ verify(delegate, never()).delete(any(), anyBoolean());
+ }
+
+ @Test
+ void testResolvingPropagatesDiscoveryFailureWithoutFallback() throws Exception {
+ FileIO delegate = mock(FileIO.class);
+ IOException failure = new IOException("resolved discovery failed");
+ when(delegate.batchFileDeleter(FIRST)).thenThrow(failure);
+
+ assertThatThrownBy(() -> resolving(delegate).batchFileDeleter(FIRST)).isSameAs(failure);
+ verify(delegate, never()).delete(any(), anyBoolean());
+ }
+
+ @Test
+ void testCachingForwardsSupportedAndUnsupportedCapabilities() throws Exception {
+ FileIO supportedDelegate = mock(FileIO.class);
+ BatchDeleteResult expected = result(FILES);
+ when(supportedDelegate.batchFileDeleter(FIRST))
+ .thenReturn(Optional.of(deleter(1000, files -> expected)));
+ CachingFileIO supported = caching(supportedDelegate);
+
+ BatchFileDeleter forwarded =
+ supported.batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+ assertThat(forwarded.maxBatchSize()).isEqualTo(1000);
+ assertThat(forwarded.delete(FILES)).isSameAs(expected);
+ verify(supportedDelegate, times(1)).batchFileDeleter(FIRST);
+
+ FileIO unsupportedDelegate = mock(FileIO.class);
+ when(unsupportedDelegate.batchFileDeleter(FIRST)).thenReturn(Optional.empty());
+ assertThat(caching(unsupportedDelegate).batchFileDeleter(FIRST)).isEmpty();
+ }
+
+ @Test
+ void testCachingPropagatesProviderFailureWithoutFallback() throws Exception {
+ FileIO delegate = mock(FileIO.class);
+ IOException failure = new IOException("cached batch failed");
+ when(delegate.batchFileDeleter(FIRST))
+ .thenReturn(Optional.of(deleter(1000, files -> raise(failure))));
+ BatchFileDeleter forwarded =
+ caching(delegate).batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+
+ assertThatThrownBy(() -> forwarded.delete(FILES)).isSameAs(failure);
+ verify(delegate, never()).delete(any(), anyBoolean());
+ }
+
+ @Test
+ void testCachingPropagatesDiscoveryFailureWithoutFallback() throws Exception {
+ FileIO delegate = mock(FileIO.class);
+ IOException failure = new IOException("cached discovery failed");
+ when(delegate.batchFileDeleter(FIRST)).thenThrow(failure);
+
+ assertThatThrownBy(() -> caching(delegate).batchFileDeleter(FIRST)).isSameAs(failure);
+ verify(delegate, times(1)).batchFileDeleter(FIRST);
+ verify(delegate, never()).delete(any(), anyBoolean());
+ }
+
+ @Test
+ void testRestTokenForwardsSupportedAndUnsupportedCapabilities() throws Exception {
+ FileIO supportedDelegate = mock(FileIO.class);
+ BatchDeleteResult expected = result(FILES);
+ when(supportedDelegate.batchFileDeleter(FIRST))
+ .thenReturn(Optional.of(deleter(1000, files -> expected)));
+ RESTTokenFileIO supported = restFileIO(supportedDelegate);
+
+ BatchFileDeleter forwarded =
+ supported.batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+ assertThat(forwarded.maxBatchSize()).isEqualTo(1000);
+ assertThat(forwarded.delete(FILES)).isSameAs(expected);
+
+ FileIO unsupportedDelegate = mock(FileIO.class);
+ when(unsupportedDelegate.batchFileDeleter(FIRST)).thenReturn(Optional.empty());
+ assertThat(restFileIO(unsupportedDelegate).batchFileDeleter(FIRST)).isEmpty();
+ }
+
+ @Test
+ void testRestTokenRefreshDoesNotInvokeStaleDeleter() throws Exception {
+ FileIO staleDelegate = mock(FileIO.class);
+ FileIO currentDelegate = mock(FileIO.class);
+ AtomicInteger staleCalls = new AtomicInteger();
+ AtomicInteger currentCalls = new AtomicInteger();
+ when(staleDelegate.batchFileDeleter(FIRST))
+ .thenReturn(
+ Optional.of(
+ deleter(
+ 1000,
+ files -> {
+ staleCalls.incrementAndGet();
+ return result(files);
+ })));
+ BatchDeleteResult expected = result(FILES);
+ when(currentDelegate.batchFileDeleter(FIRST))
+ .thenReturn(
+ Optional.of(
+ deleter(
+ 1000,
+ files -> {
+ currentCalls.incrementAndGet();
+ return expected;
+ })));
+ FileIOLoader loader =
+ loader(staleDelegate, staleDelegate, currentDelegate, currentDelegate);
+ RESTApi api = mock(RESTApi.class);
+ Identifier identifier = Identifier.create("db", "table");
+ when(api.loadTableToken(identifier)).thenReturn(token(0L), token(Long.MAX_VALUE));
+ RESTTokenFileIO rest =
+ new RESTTokenFileIO(
+ CatalogContext.create(new Options(), loader, null), api, identifier, FIRST);
+
+ BatchFileDeleter forwarded = rest.batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+ assertThat(forwarded.delete(FILES)).isSameAs(expected);
+
+ assertThat(staleCalls).hasValue(0);
+ assertThat(currentCalls).hasValue(1);
+ verify(api, times(2)).loadTableToken(identifier);
+ verify(staleDelegate, never()).delete(any(), anyBoolean());
+ verify(currentDelegate, never()).delete(any(), anyBoolean());
+ }
+
+ @Test
+ void testRestTokenCurrentUnsupportedIsHardFailureWithoutUsingStaleCapability()
+ throws Exception {
+ FileIO staleDelegate = mock(FileIO.class);
+ FileIO currentDelegate = mock(FileIO.class);
+ AtomicInteger staleCalls = new AtomicInteger();
+ when(staleDelegate.batchFileDeleter(FIRST))
+ .thenReturn(
+ Optional.of(
+ deleter(
+ 1000,
+ files -> {
+ staleCalls.incrementAndGet();
+ return result(files);
+ })));
+ when(currentDelegate.batchFileDeleter(FIRST)).thenReturn(Optional.empty());
+ RestRefreshFixture fixture = refreshingRest(staleDelegate, currentDelegate);
+
+ BatchFileDeleter forwarded =
+ fixture.fileIO.batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+ assertThat(forwarded.maxBatchSize()).isEqualTo(1000);
+ assertThatThrownBy(() -> forwarded.delete(FILES)).isInstanceOf(IOException.class);
+
+ assertThat(staleCalls).hasValue(0);
+ verify(fixture.api, times(2)).loadTableToken(fixture.identifier);
+ verify(staleDelegate, never()).delete(any(), anyBoolean());
+ verify(currentDelegate, never()).delete(any(), anyBoolean());
+ }
+
+ @Test
+ void testRestTokenCurrentDiscoveryFailureIsHardFailureWithoutUsingStaleCapability()
+ throws Exception {
+ FileIO staleDelegate = mock(FileIO.class);
+ FileIO currentDelegate = mock(FileIO.class);
+ AtomicInteger staleCalls = new AtomicInteger();
+ when(staleDelegate.batchFileDeleter(FIRST))
+ .thenReturn(
+ Optional.of(
+ deleter(
+ 1000,
+ files -> {
+ staleCalls.incrementAndGet();
+ return result(files);
+ })));
+ IOException failure = new IOException("refreshed capability discovery failed");
+ when(currentDelegate.batchFileDeleter(FIRST)).thenThrow(failure);
+ RestRefreshFixture fixture = refreshingRest(staleDelegate, currentDelegate);
+
+ BatchFileDeleter forwarded =
+ fixture.fileIO.batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+ assertThatThrownBy(() -> forwarded.delete(FILES)).isSameAs(failure);
+
+ assertThat(staleCalls).hasValue(0);
+ verify(fixture.api, times(2)).loadTableToken(fixture.identifier);
+ verify(staleDelegate, never()).delete(any(), anyBoolean());
+ verify(currentDelegate, never()).delete(any(), anyBoolean());
+ }
+
+ @Test
+ void testRestTokenMaxBatchSizeIsDiscoverySnapshotButDeleteUsesCurrentCapability()
+ throws Exception {
+ FileIO staleDelegate = mock(FileIO.class);
+ FileIO currentDelegate = mock(FileIO.class);
+ AtomicInteger staleCalls = new AtomicInteger();
+ AtomicInteger currentCalls = new AtomicInteger();
+ when(staleDelegate.batchFileDeleter(FIRST))
+ .thenReturn(
+ Optional.of(
+ deleter(
+ 1000,
+ files -> {
+ staleCalls.incrementAndGet();
+ return result(files);
+ })));
+ BatchDeleteResult expected = result(FILES);
+ when(currentDelegate.batchFileDeleter(FIRST))
+ .thenReturn(
+ Optional.of(
+ deleter(
+ 7,
+ files -> {
+ currentCalls.incrementAndGet();
+ return expected;
+ })));
+ RestRefreshFixture fixture = refreshingRest(staleDelegate, currentDelegate);
+
+ BatchFileDeleter forwarded =
+ fixture.fileIO.batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+
+ // The scheduler-facing limit is a discovery snapshot. It does not authorize use of the
+ // captured deleter: invocation still refreshes and lets the current provider validate.
+ assertThat(forwarded.maxBatchSize()).isEqualTo(1000);
+ assertThat(forwarded.delete(FILES)).isSameAs(expected);
+ assertThat(staleCalls).hasValue(0);
+ assertThat(currentCalls).hasValue(1);
+ }
+
+ @Test
+ void testRestTokenPropagatesProviderFailureWithoutFallback() throws Exception {
+ FileIO delegate = mock(FileIO.class);
+ IOException failure = new IOException("REST batch failed");
+ when(delegate.batchFileDeleter(FIRST))
+ .thenReturn(Optional.of(deleter(1000, files -> raise(failure))));
+ BatchFileDeleter forwarded =
+ restFileIO(delegate).batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+
+ assertThatThrownBy(() -> forwarded.delete(FILES)).isSameAs(failure);
+ verify(delegate, never()).delete(any(), anyBoolean());
+ }
+
+ @Test
+ void testRestTokenPropagatesDiscoveryFailureWithoutFallback() throws Exception {
+ FileIO delegate = mock(FileIO.class);
+ IOException failure = new IOException("REST discovery failed");
+ when(delegate.batchFileDeleter(FIRST)).thenThrow(failure);
+
+ assertThatThrownBy(() -> restFileIO(delegate).batchFileDeleter(FIRST)).isSameAs(failure);
+ verify(delegate, never()).delete(any(), anyBoolean());
+ }
+
+ @Test
+ void testPluginSerializationForcesCapabilityRediscovery() throws Exception {
+ FileIO staleDelegate = mock(FileIO.class);
+ FileIO currentDelegate = mock(FileIO.class);
+ AtomicInteger staleCalls = new AtomicInteger();
+ AtomicInteger currentCalls = new AtomicInteger();
+ when(staleDelegate.batchFileDeleter(FIRST))
+ .thenReturn(
+ Optional.of(
+ deleter(
+ 1000,
+ files -> {
+ staleCalls.incrementAndGet();
+ return result(files);
+ })));
+ BatchDeleteResult expected = result(FILES);
+ when(currentDelegate.batchFileDeleter(FIRST))
+ .thenReturn(
+ Optional.of(
+ deleter(
+ 1000,
+ files -> {
+ currentCalls.incrementAndGet();
+ return expected;
+ })));
+ SerializablePluginFileIO.reset(staleDelegate);
+ try {
+ SerializablePluginFileIO original = new SerializablePluginFileIO();
+ assertThat(original.batchFileDeleter(FIRST)).isPresent();
+
+ SerializablePluginFileIO restored = InstantiationUtil.clone(original);
+ SerializablePluginFileIO.activeDelegate.set(currentDelegate);
+ BatchFileDeleter rediscovered =
+ restored.batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+
+ assertThat(rediscovered.delete(FILES)).isSameAs(expected);
+ assertThat(SerializablePluginFileIO.discoveryCalls.get()).hasValue(2);
+ assertThat(staleCalls).hasValue(0);
+ assertThat(currentCalls).hasValue(1);
+ } finally {
+ SerializablePluginFileIO.clear();
+ }
+ }
+
+ @Test
+ void testRestCachingResolvingPluginChainPreservesRefreshAndStrictFailure() throws Exception {
+ FileIO staleProvider = mock(FileIO.class);
+ FileIO currentProvider = mock(FileIO.class);
+ when(staleProvider.exists(any())).thenReturn(true);
+ when(currentProvider.exists(any())).thenReturn(true);
+ AtomicInteger staleCalls = new AtomicInteger();
+ AtomicInteger currentCalls = new AtomicInteger();
+ ClassLoader stalePluginClassLoader = new ClassLoader() {};
+ ClassLoader currentPluginClassLoader = new ClassLoader() {};
+ when(staleProvider.batchFileDeleter(FIRST))
+ .thenAnswer(
+ ignored -> {
+ assertThat(Thread.currentThread().getContextClassLoader())
+ .isSameAs(stalePluginClassLoader);
+ return Optional.of(
+ deleter(
+ 1000,
+ files -> {
+ staleCalls.incrementAndGet();
+ return result(files);
+ }));
+ });
+ BatchDeleteResult expected = result(FILES);
+ when(currentProvider.batchFileDeleter(FIRST))
+ .thenAnswer(
+ ignored -> {
+ assertThat(Thread.currentThread().getContextClassLoader())
+ .isSameAs(currentPluginClassLoader);
+ return Optional.of(
+ deleter(
+ 1000,
+ files -> {
+ assertThat(
+ Thread.currentThread()
+ .getContextClassLoader())
+ .isSameAs(currentPluginClassLoader);
+ currentCalls.incrementAndGet();
+ return expected;
+ }));
+ });
+ FileIO staleChain =
+ frozenResolving(new TestPluginFileIO(staleProvider, stalePluginClassLoader));
+ FileIO currentChain =
+ frozenResolving(new TestPluginFileIO(currentProvider, currentPluginClassLoader));
+ FileIOLoader outerLoader = loader(staleChain, staleChain, currentChain, currentChain);
+ RESTApi api = mock(RESTApi.class);
+ Identifier identifier = Identifier.create("db", "table");
+ when(api.loadTableToken(identifier)).thenReturn(token(0L), token(Long.MAX_VALUE));
+ RESTTokenFileIO rest =
+ new RESTTokenFileIO(
+ CatalogContext.create(new Options(), outerLoader, null),
+ api,
+ identifier,
+ FIRST);
+ CachingFileIO chainRoot = caching(rest);
+ ClassLoader previous = Thread.currentThread().getContextClassLoader();
+ ClassLoader callerClassLoader = new ClassLoader(previous) {};
+
+ Thread.currentThread().setContextClassLoader(callerClassLoader);
+ try {
+ BatchFileDeleter chain =
+ chainRoot.batchFileDeleter(FIRST).orElseThrow(AssertionError::new);
+ assertThatThrownBy(
+ () ->
+ chain.delete(
+ Arrays.asList(
+ FIRST,
+ new Path(
+ "oss://other-bucket/table/b.parquet"))))
+ .isInstanceOfAny(IllegalArgumentException.class, IOException.class);
+ assertThat(staleCalls).hasValue(0);
+ assertThat(currentCalls).hasValue(0);
+
+ assertThat(chain.delete(FILES)).isSameAs(expected);
+ assertThat(staleCalls).hasValue(0);
+ assertThat(currentCalls).hasValue(1);
+ assertThat(Thread.currentThread().getContextClassLoader()).isSameAs(callerClassLoader);
+ verify(staleProvider, never()).delete(any(), anyBoolean());
+ verify(currentProvider, never()).delete(any(), anyBoolean());
+ } finally {
+ Thread.currentThread().setContextClassLoader(previous);
+ }
+ }
+
+ private static ResolvingFileIO resolving(FileIO delegate) throws IOException {
+ FileIOLoader loader = loader(delegate, delegate);
+ ResolvingFileIO resolving = new ResolvingFileIO();
+ resolving.configure(CatalogContext.create(new Options(), loader, null));
+ return resolving;
+ }
+
+ private static ResolvingFileIO frozenResolving(FileIO delegate) throws IOException {
+ FileIOLoader loader = loader(delegate, delegate);
+ FrozenResolvingFileIO resolving = new FrozenResolvingFileIO();
+ resolving.configure(CatalogContext.create(new Options(), loader, null));
+ return resolving;
+ }
+
+ private static CachingFileIO caching(FileIO delegate) {
+ return new CachingFileIO(
+ delegate,
+ mock(org.apache.paimon.fs.cache.LocalCacheManager.class),
+ EnumSet.of(FileType.DATA));
+ }
+
+ private static RESTTokenFileIO restFileIO(FileIO delegate) {
+ FileIOLoader loader = loader(delegate, delegate);
+ RESTApi api = mock(RESTApi.class);
+ Identifier identifier = Identifier.create("db", "table");
+ when(api.loadTableToken(identifier)).thenReturn(token(Long.MAX_VALUE));
+ return new RESTTokenFileIO(
+ CatalogContext.create(new Options(), loader, null), api, identifier, FIRST);
+ }
+
+ private static RestRefreshFixture refreshingRest(FileIO staleDelegate, FileIO currentDelegate) {
+ FileIOLoader loader =
+ loader(staleDelegate, staleDelegate, currentDelegate, currentDelegate);
+ RESTApi api = mock(RESTApi.class);
+ Identifier identifier = Identifier.create("db", "table");
+ when(api.loadTableToken(identifier)).thenReturn(token(0L), token(Long.MAX_VALUE));
+ return new RestRefreshFixture(
+ new RESTTokenFileIO(
+ CatalogContext.create(new Options(), loader, null), api, identifier, FIRST),
+ api,
+ identifier);
+ }
+
+ private static FileIOLoader loader(FileIO first, FileIO... remaining) {
+ FileIOLoader loader = mock(FileIOLoader.class);
+ when(loader.getScheme()).thenReturn("oss");
+ when(loader.load(any())).thenReturn(first, remaining);
+ return loader;
+ }
+
+ private static GetTableTokenResponse token(long expiresAtMillis) {
+ return new GetTableTokenResponse(
+ Collections.singletonMap("token", UUID.randomUUID().toString()), expiresAtMillis);
+ }
+
+ private static BatchDeleteResult result(List files) {
+ return new BatchDeleteResult(files);
+ }
+
+ private static BatchDeleteResult raise(IOException failure) throws IOException {
+ throw failure;
+ }
+
+ private static BatchFileDeleter deleter(int maxBatchSize, DeleteAction action) {
+ return new BatchFileDeleter() {
+ @Override
+ public int maxBatchSize() {
+ return maxBatchSize;
+ }
+
+ @Override
+ public BatchDeleteResult delete(List files) throws IOException {
+ return action.delete(files);
+ }
+ };
+ }
+
+ @FunctionalInterface
+ private interface DeleteAction {
+ BatchDeleteResult delete(List files) throws IOException;
+ }
+
+ private static class RestRefreshFixture {
+
+ private final RESTTokenFileIO fileIO;
+ private final RESTApi api;
+ private final Identifier identifier;
+
+ private RestRefreshFixture(RESTTokenFileIO fileIO, RESTApi api, Identifier identifier) {
+ this.fileIO = fileIO;
+ this.api = api;
+ this.identifier = identifier;
+ }
+ }
+
+ private static class FrozenResolvingFileIO extends ResolvingFileIO {
+
+ private boolean initialized;
+
+ @Override
+ public void configure(CatalogContext context) {
+ if (!initialized) {
+ super.configure(context);
+ initialized = true;
+ }
+ }
+ }
+
+ private static class SerializablePluginFileIO extends PluginFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final ThreadLocal activeDelegate = new ThreadLocal<>();
+ private static final ThreadLocal discoveryCalls = new ThreadLocal<>();
+
+ private static void reset(FileIO delegate) {
+ activeDelegate.set(delegate);
+ discoveryCalls.set(new AtomicInteger());
+ }
+
+ private static void clear() {
+ activeDelegate.remove();
+ discoveryCalls.remove();
+ }
+
+ @Override
+ public boolean isObjectStore() {
+ return true;
+ }
+
+ @Override
+ protected FileIO createFileIO(Path path) {
+ discoveryCalls.get().incrementAndGet();
+ return activeDelegate.get();
+ }
+
+ @Override
+ protected ClassLoader pluginClassLoader() {
+ return SerializablePluginFileIO.class.getClassLoader();
+ }
+ }
+
+ private static class TestPluginFileIO extends PluginFileIO {
+
+ private final FileIO delegate;
+ private final ClassLoader classLoader;
+
+ private TestPluginFileIO(FileIO delegate, ClassLoader classLoader) {
+ this.delegate = delegate;
+ this.classLoader = classLoader;
+ }
+
+ @Override
+ public boolean isObjectStore() {
+ return true;
+ }
+
+ @Override
+ protected FileIO createFileIO(Path path) {
+ return delegate;
+ }
+
+ @Override
+ protected ClassLoader pluginClassLoader() {
+ return classLoader;
+ }
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java
index 73c11d774166..f9d08a5c1bfb 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java
@@ -78,6 +78,14 @@ public BatchTableCommit newCommit() {
CoreOptions options = new CoreOptions(table.options());
boolean formatTablePartitionOnlyValueInPath = options.formatTablePartitionOnlyValueInPath();
String syncHiveUri = options.formatTableCommitSyncPartitionHiveUri();
+ int cleanupThreadNum =
+ table.partitionManager() != null && !table.partitionKeys().isEmpty()
+ ? options.formatTableCommitCleanupThreadNum()
+ : 1;
+ int publishThreadNum =
+ table.partitionManager() != null && !table.partitionKeys().isEmpty()
+ ? options.formatTableCommitPublishThreadNum()
+ : 1;
return new FormatTableCommit(
table.location(),
table.partitionKeys(),
@@ -90,7 +98,9 @@ public BatchTableCommit newCommit() {
syncHiveUri,
table.catalogContext(),
table.partitionManager(),
- options.dynamicPartitionOverwrite());
+ options.dynamicPartitionOverwrite(),
+ cleanupThreadNum,
+ publishThreadNum);
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java
index be2105930a57..b4a7a1c350f6 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java
@@ -18,11 +18,14 @@
package org.apache.paimon.table.format;
+import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.CatalogFactory;
import org.apache.paimon.catalog.DelegateCatalog;
import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.BatchDeleteResult;
+import org.apache.paimon.fs.BatchFileDeleter;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.FileStatus;
import org.apache.paimon.fs.Path;
@@ -38,6 +41,7 @@
import org.apache.paimon.table.sink.TableCommit;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.PartitionPathUtils;
+import org.apache.paimon.utils.ThreadPoolUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,14 +51,23 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Method;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Deque;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
import java.util.stream.Collectors;
import static org.apache.paimon.table.format.FormatBatchWriteBuilder.validateStaticPartition;
@@ -64,6 +77,18 @@ public class FormatTableCommit implements BatchTableCommit {
private static final Logger LOG = LoggerFactory.getLogger(FormatTableCommit.class);
+ private static final int MAX_CLEANUP_THREAD_NUM = 64;
+
+ private static final int MAX_PUBLISH_THREAD_NUM = 64;
+
+ private static final ExecutorService CLEANUP_EXECUTOR =
+ ThreadPoolUtils.createCachedThreadPool(
+ MAX_CLEANUP_THREAD_NUM, "FORMAT-TABLE-COMMIT-CLEANUP-THREAD-POOL");
+
+ private static final ExecutorService PUBLISH_EXECUTOR =
+ ThreadPoolUtils.createCachedThreadPool(
+ MAX_PUBLISH_THREAD_NUM, "FORMAT-TABLE-COMMIT-PUBLISH-THREAD-POOL");
+
private String location;
private final boolean formatTablePartitionOnlyValueInPath;
private final String defaultPartName;
@@ -75,6 +100,10 @@ public class FormatTableCommit implements BatchTableCommit {
private Identifier tableIdentifier;
@Nullable private final FormatTablePartitionManager partitionManager;
private final boolean dynamicPartitionOverwrite;
+ private final int cleanupThreadNum;
+ private final ExecutorService cleanupExecutor;
+ private final int publishThreadNum;
+ private final ExecutorService publishExecutor;
public FormatTableCommit(
String location,
@@ -89,6 +118,190 @@ public FormatTableCommit(
CatalogContext catalogContext,
@Nullable FormatTablePartitionManager partitionManager,
boolean dynamicPartitionOverwrite) {
+ this(
+ location,
+ partitionKeys,
+ fileIO,
+ formatTablePartitionOnlyValueInPath,
+ defaultPartName,
+ overwrite,
+ tableIdentifier,
+ staticPartitions,
+ syncHiveUri,
+ catalogContext,
+ partitionManager,
+ dynamicPartitionOverwrite,
+ 1,
+ 1,
+ CLEANUP_EXECUTOR,
+ PUBLISH_EXECUTOR);
+ }
+
+ FormatTableCommit(
+ String location,
+ List partitionKeys,
+ FileIO fileIO,
+ boolean formatTablePartitionOnlyValueInPath,
+ String defaultPartName,
+ boolean overwrite,
+ Identifier tableIdentifier,
+ @Nullable Map staticPartitions,
+ @Nullable String syncHiveUri,
+ CatalogContext catalogContext,
+ @Nullable FormatTablePartitionManager partitionManager,
+ boolean dynamicPartitionOverwrite,
+ int cleanupThreadNum) {
+ this(
+ location,
+ partitionKeys,
+ fileIO,
+ formatTablePartitionOnlyValueInPath,
+ defaultPartName,
+ overwrite,
+ tableIdentifier,
+ staticPartitions,
+ syncHiveUri,
+ catalogContext,
+ partitionManager,
+ dynamicPartitionOverwrite,
+ cleanupThreadNum,
+ 1,
+ CLEANUP_EXECUTOR,
+ PUBLISH_EXECUTOR);
+ }
+
+ FormatTableCommit(
+ String location,
+ List partitionKeys,
+ FileIO fileIO,
+ boolean formatTablePartitionOnlyValueInPath,
+ String defaultPartName,
+ boolean overwrite,
+ Identifier tableIdentifier,
+ @Nullable Map staticPartitions,
+ @Nullable String syncHiveUri,
+ CatalogContext catalogContext,
+ @Nullable FormatTablePartitionManager partitionManager,
+ boolean dynamicPartitionOverwrite,
+ int cleanupThreadNum,
+ int publishThreadNum) {
+ this(
+ location,
+ partitionKeys,
+ fileIO,
+ formatTablePartitionOnlyValueInPath,
+ defaultPartName,
+ overwrite,
+ tableIdentifier,
+ staticPartitions,
+ syncHiveUri,
+ catalogContext,
+ partitionManager,
+ dynamicPartitionOverwrite,
+ cleanupThreadNum,
+ publishThreadNum,
+ CLEANUP_EXECUTOR,
+ PUBLISH_EXECUTOR);
+ }
+
+ FormatTableCommit(
+ String location,
+ List partitionKeys,
+ FileIO fileIO,
+ boolean formatTablePartitionOnlyValueInPath,
+ String defaultPartName,
+ boolean overwrite,
+ Identifier tableIdentifier,
+ @Nullable Map staticPartitions,
+ @Nullable String syncHiveUri,
+ CatalogContext catalogContext,
+ @Nullable FormatTablePartitionManager partitionManager,
+ boolean dynamicPartitionOverwrite,
+ int cleanupThreadNum,
+ ExecutorService cleanupExecutor) {
+ this(
+ location,
+ partitionKeys,
+ fileIO,
+ formatTablePartitionOnlyValueInPath,
+ defaultPartName,
+ overwrite,
+ tableIdentifier,
+ staticPartitions,
+ syncHiveUri,
+ catalogContext,
+ partitionManager,
+ dynamicPartitionOverwrite,
+ cleanupThreadNum,
+ 1,
+ cleanupExecutor,
+ PUBLISH_EXECUTOR);
+ }
+
+ FormatTableCommit(
+ String location,
+ List partitionKeys,
+ FileIO fileIO,
+ boolean formatTablePartitionOnlyValueInPath,
+ String defaultPartName,
+ boolean overwrite,
+ Identifier tableIdentifier,
+ @Nullable Map staticPartitions,
+ @Nullable String syncHiveUri,
+ CatalogContext catalogContext,
+ @Nullable FormatTablePartitionManager partitionManager,
+ boolean dynamicPartitionOverwrite,
+ int cleanupThreadNum,
+ int publishThreadNum,
+ ExecutorService publishExecutor) {
+ this(
+ location,
+ partitionKeys,
+ fileIO,
+ formatTablePartitionOnlyValueInPath,
+ defaultPartName,
+ overwrite,
+ tableIdentifier,
+ staticPartitions,
+ syncHiveUri,
+ catalogContext,
+ partitionManager,
+ dynamicPartitionOverwrite,
+ cleanupThreadNum,
+ publishThreadNum,
+ CLEANUP_EXECUTOR,
+ publishExecutor);
+ }
+
+ private FormatTableCommit(
+ String location,
+ List partitionKeys,
+ FileIO fileIO,
+ boolean formatTablePartitionOnlyValueInPath,
+ String defaultPartName,
+ boolean overwrite,
+ Identifier tableIdentifier,
+ @Nullable Map staticPartitions,
+ @Nullable String syncHiveUri,
+ CatalogContext catalogContext,
+ @Nullable FormatTablePartitionManager partitionManager,
+ boolean dynamicPartitionOverwrite,
+ int cleanupThreadNum,
+ int publishThreadNum,
+ ExecutorService cleanupExecutor,
+ ExecutorService publishExecutor) {
+ if (cleanupThreadNum < 1 || cleanupThreadNum > MAX_CLEANUP_THREAD_NUM) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Format Table cleanup thread number must be between 1 and %s, but was %s.",
+ MAX_CLEANUP_THREAD_NUM, cleanupThreadNum));
+ }
+ if (publishThreadNum < 1 || publishThreadNum > MAX_PUBLISH_THREAD_NUM) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Format Table publish thread number must be between 1 and %s, but was %s.",
+ MAX_PUBLISH_THREAD_NUM, publishThreadNum));
+ }
this.location = location;
this.fileIO = fileIO;
this.formatTablePartitionOnlyValueInPath = formatTablePartitionOnlyValueInPath;
@@ -100,6 +313,10 @@ public FormatTableCommit(
this.tableIdentifier = tableIdentifier;
this.partitionManager = partitionManager;
this.dynamicPartitionOverwrite = dynamicPartitionOverwrite;
+ this.cleanupThreadNum = cleanupThreadNum;
+ this.cleanupExecutor = cleanupExecutor;
+ this.publishThreadNum = publishThreadNum;
+ this.publishExecutor = publishExecutor;
if (syncHiveUri != null) {
try {
Options options = new Options();
@@ -151,32 +368,37 @@ public void commit(List commitMessages) {
// A static partition may name only the leading keys, in which case the path
// is a prefix and the partition directories of the remaining keys sit below.
clearedPartitionPaths.addAll(
- deletePreviousDataFile(
- partitionPath, partitionKeys.size() - staticPartitions.size()));
+ deletePreviousDataFiles(
+ Collections.singletonList(partitionPath),
+ partitionKeys.size() - staticPartitions.size(),
+ cleanupThreadNum));
}
if (!fileIO.exists(partitionPath)) {
fileIO.mkdirs(partitionPath);
}
} else if (overwrite) {
if (replacesOnlyWrittenPartitions()) {
- Set partitionPaths = new HashSet<>();
+ Set partitionPaths = new LinkedHashSet<>();
for (TwoPhaseCommitMessage message : messages) {
partitionPaths.add(message.getCommitter().targetPath().getParent());
}
- for (Path p : partitionPaths) {
- // The parent of a written file is a complete partition directory - the
- // table directory itself when the table is unpartitioned - so there is no
- // partition level below it to descend, and it is a partition this commit
- // writes anyway.
- deletePreviousDataFile(p, 0);
+ // The parent of a written file is a complete partition directory - the table
+ // directory itself when the table is unpartitioned - so there is no partition
+ // level below it to descend. Collect every selected directory before deleting
+ // so many small partitions can share the same cleanup concurrency window.
+ if (partitionManager != null && cleanupThreadNum > 1) {
+ deletePreviousDynamicDataFiles(
+ new ArrayList<>(partitionPaths), cleanupThreadNum);
+ } else {
+ deletePreviousDataFiles(
+ new ArrayList<>(partitionPaths), 0, cleanupThreadNum);
}
} else {
// Overwriting without naming a partition replaces the table, so what has to go
// is everything the table holds rather than the files this commit happens to
// write: a statement whose query returns nothing still empties the table.
- for (Path dataDirectory : tableDataDirectories()) {
- clearedPartitionPaths.addAll(deletePreviousDataFile(dataDirectory, 0));
- }
+ clearedPartitionPaths.addAll(
+ deletePreviousDataFiles(tableDataDirectories(), 0, cleanupThreadNum));
}
}
@@ -187,9 +409,9 @@ public void commit(List commitMessages) {
boolean reportsStatistics = registersPartitions && partitionManager != null;
Map, PartitionStatistics> statisticsByPartition =
new LinkedHashMap<>();
+ publishMessages(messages);
for (TwoPhaseCommitMessage message : messages) {
TwoPhaseOutputStream.Committer committer = message.getCommitter();
- committer.commit(this.fileIO);
if (registersPartitions) {
// Extracted once: registration and statistics must key on the same spec.
Map spec =
@@ -244,10 +466,209 @@ public void commit(List commitMessages) {
}
}
- } catch (Exception e) {
- this.abort(commitMessages);
- throw new RuntimeException(e);
+ } catch (Throwable failure) {
+ // Cleanup restores the caller's interrupt before failing. Clear it only while aborting
+ // staging output, then restore it; an abort failure is secondary to the commit failure
+ // that made abort necessary.
+ boolean interrupted = Thread.interrupted();
+ try {
+ this.abort(commitMessages);
+ } catch (Throwable abortFailure) {
+ if (failure != abortFailure) {
+ failure.addSuppressed(abortFailure);
+ }
+ } finally {
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ if (failure instanceof Error) {
+ throw (Error) failure;
+ }
+ throw new RuntimeException(failure);
+ }
+ }
+
+ private void publishMessages(List messages) throws Throwable {
+ if (publishThreadNum == 1 || messages.size() <= 1) {
+ for (TwoPhaseCommitMessage message : messages) {
+ message.getCommitter().commit(fileIO);
+ }
+ return;
+ }
+
+ Map> messagesByPartition = new LinkedHashMap<>();
+ for (int index = 0; index < messages.size(); index++) {
+ TwoPhaseCommitMessage message = messages.get(index);
+ Path partition = message.getCommitter().targetPath().getParent();
+ messagesByPartition
+ .computeIfAbsent(partition, ignored -> new ArrayDeque<>())
+ .addLast(new IndexedPublishMessage(index, partition, message));
+ }
+
+ if (messagesByPartition.size() <= 1) {
+ for (TwoPhaseCommitMessage message : messages) {
+ message.getCommitter().commit(fileIO);
+ }
+ return;
+ }
+
+ // Force lazy filesystem and security binding on the caller before any worker can use it.
+ fileIO.exists(new Path(location));
+ ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
+ CompletionService completions =
+ new ExecutorCompletionService<>(publishExecutor);
+ PublishSubmissionState submissionState = new PublishSubmissionState();
+ Deque readyPartitions = new ArrayDeque<>(messagesByPartition.keySet());
+ Map publishFailures = new TreeMap<>();
+ List coordinatorFailures = new ArrayList<>();
+ InterruptedException interruption = null;
+ int inFlight = 0;
+
+ while ((!submissionState.isStopped() && !readyPartitions.isEmpty()) || inFlight > 0) {
+ while (!submissionState.isStopped()
+ && !readyPartitions.isEmpty()
+ && inFlight < publishThreadNum) {
+ Path partition = readyPartitions.removeFirst();
+ IndexedPublishMessage indexedMessage =
+ messagesByPartition.get(partition).removeFirst();
+ try {
+ boolean submitted =
+ submissionState.submitIfRunning(
+ () ->
+ completions.submit(
+ () ->
+ publishMessage(
+ indexedMessage,
+ contextClassLoader,
+ submissionState)));
+ if (!submitted) {
+ break;
+ }
+ inFlight++;
+ } catch (Throwable failure) {
+ submissionState.stop();
+ publishFailures.put(indexedMessage.index, failure);
+ }
+ }
+
+ if (inFlight == 0) {
+ break;
+ }
+
+ Future completed = null;
+ while (completed == null) {
+ try {
+ completed = completions.take();
+ } catch (InterruptedException e) {
+ submissionState.stop();
+ if (interruption == null) {
+ interruption = e;
+ } else {
+ interruption.addSuppressed(e);
+ }
+ }
+ }
+ inFlight--;
+
+ PublishResult result = null;
+ while (result == null) {
+ try {
+ result = completed.get();
+ } catch (InterruptedException e) {
+ submissionState.stop();
+ if (interruption == null) {
+ interruption = e;
+ } else {
+ interruption.addSuppressed(e);
+ }
+ } catch (ExecutionException e) {
+ submissionState.stop();
+ coordinatorFailures.add(e.getCause() == null ? e : e.getCause());
+ break;
+ } catch (Throwable failure) {
+ submissionState.stop();
+ coordinatorFailures.add(failure);
+ break;
+ }
+ }
+
+ if (Thread.currentThread().isInterrupted()) {
+ submissionState.stop();
+ Thread.interrupted();
+ InterruptedException pendingInterruption =
+ new InterruptedException(
+ "Interrupted while publishing format table files.");
+ if (interruption == null) {
+ interruption = pendingInterruption;
+ } else {
+ interruption.addSuppressed(pendingInterruption);
+ }
+ }
+
+ if (result != null) {
+ if (result.failure != null) {
+ submissionState.stop();
+ publishFailures.put(result.index, result.failure);
+ } else if (!submissionState.isStopped()
+ && !messagesByPartition.get(result.partition).isEmpty()) {
+ readyPartitions.addLast(result.partition);
+ }
+ }
+ }
+
+ Throwable primary = aggregateFailures(publishFailures.values());
+ for (Throwable failure : coordinatorFailures) {
+ if (primary == null) {
+ primary = failure;
+ } else if (primary != failure) {
+ primary.addSuppressed(failure);
+ }
+ }
+ if (interruption != null) {
+ if (primary == null) {
+ primary = interruption;
+ } else if (primary != interruption) {
+ primary.addSuppressed(interruption);
+ }
+ Thread.currentThread().interrupt();
+ }
+ if (primary != null) {
+ throw primary;
+ }
+ }
+
+ private PublishResult publishMessage(
+ IndexedPublishMessage indexedMessage,
+ ClassLoader contextClassLoader,
+ PublishSubmissionState submissionState) {
+ Thread currentThread = Thread.currentThread();
+ ClassLoader originalClassLoader = null;
+ boolean originalClassLoaderCaptured = false;
+ Throwable failure = null;
+ try {
+ originalClassLoader = currentThread.getContextClassLoader();
+ originalClassLoaderCaptured = true;
+ currentThread.setContextClassLoader(contextClassLoader);
+ indexedMessage.message.getCommitter().commit(fileIO);
+ } catch (Throwable publishFailure) {
+ failure = publishFailure;
+ submissionState.stop();
+ } finally {
+ if (originalClassLoaderCaptured) {
+ try {
+ currentThread.setContextClassLoader(originalClassLoader);
+ } catch (Throwable restoreFailure) {
+ if (failure == null) {
+ failure = restoreFailure;
+ } else if (failure != restoreFailure) {
+ failure.addSuppressed(restoreFailure);
+ }
+ submissionState.stop();
+ }
+ }
}
+ return new PublishResult(indexedMessage.index, indexedMessage.partition, failure);
}
/**
@@ -426,8 +847,9 @@ private static Path buildPartitionPath(
@Override
public void abort(List commitMessages) {
- try {
- for (CommitMessage commitMessage : commitMessages) {
+ Throwable primary = null;
+ for (CommitMessage commitMessage : commitMessages) {
+ try {
if (commitMessage instanceof TwoPhaseCommitMessage) {
TwoPhaseCommitMessage twoPhaseCommitMessage =
(TwoPhaseCommitMessage) commitMessage;
@@ -437,9 +859,19 @@ public void abort(List commitMessages) {
"Unsupported commit message type: "
+ commitMessage.getClass().getName());
}
+ } catch (Throwable failure) {
+ if (primary == null) {
+ primary = failure;
+ } else if (primary != failure) {
+ primary.addSuppressed(failure);
+ }
}
- } catch (Exception e) {
- throw new RuntimeException(e);
+ }
+ if (primary instanceof Error) {
+ throw (Error) primary;
+ }
+ if (primary != null) {
+ throw new RuntimeException(primary);
}
}
@@ -506,42 +938,464 @@ private List, Path>> partitionsInTheFileSyste
*/
private Set deletePreviousDataFile(Path partitionPath, int partitionLevels)
throws IOException {
+ return deletePreviousDataFiles(
+ Collections.singletonList(partitionPath), partitionLevels, 1);
+ }
+
+ private Set deletePreviousDataFiles(
+ List partitionPaths, int partitionLevels, int threadNum) throws IOException {
+ PreviousDataFiles dataFiles = new PreviousDataFiles(partitionPaths, partitionLevels);
+ return deletePreviousDataFiles(dataFiles, threadNum);
+ }
+
+ private Set deletePreviousDataFiles(PreviousDataFiles dataFiles, int threadNum)
+ throws IOException {
Set clearedPartitionPaths = new HashSet<>();
- if (fileIO.exists(partitionPath)) {
- // Committed data files only: what sits under a staging directory is another writer's
- // uncommitted output, whatever its name looks like.
- for (FileStatus file :
- FormatTableScan.listDataFiles(
- fileIO,
- partitionPath,
- partitionLevels,
- formatTablePartitionOnlyValueInPath,
- defaultPartName)) {
- boolean deleted;
+ if (threadNum == 1) {
+ FileStatus file;
+ while ((file = dataFiles.next()) != null) {
+ if (deleteDataFile(file)) {
+ clearedPartitionPaths.add(file.getPath().getParent());
+ }
+ }
+ } else {
+ clearedPartitionPaths.addAll(deleteDataFilesConcurrently(dataFiles, threadNum));
+ }
+ return clearedPartitionPaths;
+ }
+
+ private void deletePreviousDynamicDataFiles(List partitionPaths, int threadNum)
+ throws IOException {
+ PreviousDataFiles dataFiles = new PreviousDataFiles(partitionPaths, 0);
+ FileStatus firstFile = dataFiles.next();
+ if (firstFile == null) {
+ return;
+ }
+
+ Optional capability = fileIO.batchFileDeleter(firstFile.getPath());
+ if (capability == null) {
+ throw new IOException(
+ String.format(
+ "Batch delete capability lookup returned null for table %s.",
+ tableIdentifier.getFullName()));
+ }
+
+ dataFiles.pushBack(firstFile);
+ if (!capability.isPresent()) {
+ deletePreviousDataFiles(dataFiles, threadNum);
+ return;
+ }
+
+ deleteDataFilesInBatches(dataFiles, capability.get());
+ }
+
+ private void deleteDataFilesInBatches(PreviousDataFiles dataFiles, BatchFileDeleter fileDeleter)
+ throws IOException {
+ int maxBatchSize = fileDeleter.maxBatchSize();
+ if (maxBatchSize <= 0) {
+ throw new IOException(
+ String.format(
+ "Batch delete size for table %s must be greater than 0, but was %s.",
+ tableIdentifier.getFullName(), maxBatchSize));
+ }
+
+ while (true) {
+ List batch = new ArrayList<>();
+ while (batch.size() < maxBatchSize) {
+ FileStatus file = dataFiles.next();
+ if (file == null) {
+ break;
+ }
+ batch.add(file.getPath());
+ }
+ if (batch.isEmpty()) {
+ return;
+ }
+
+ List request = Collections.unmodifiableList(batch);
+ BatchDeleteResult result = fileDeleter.delete(request);
+ if (Thread.currentThread().isInterrupted()) {
+ InterruptedException interruption =
+ new InterruptedException(
+ "Interrupted while deleting old Format Table data files in batches.");
+ throw new IOException(interruption.getMessage(), interruption);
+ }
+ if (result == null) {
+ throw new IOException(
+ String.format(
+ "Batch delete for table %s returned no result.",
+ tableIdentifier.getFullName()));
+ }
+
+ List confirmed = result.deletedOrNotFound();
+ if (confirmed == null || !request.equals(confirmed)) {
+ throw new IOException(
+ String.format(
+ "Batch delete result for table %s did not exactly match the %s requested files in order (result size: %s).",
+ tableIdentifier.getFullName(),
+ request.size(),
+ confirmed == null ? "null" : confirmed.size()));
+ }
+ if (batch.size() < maxBatchSize) {
+ return;
+ }
+ }
+ }
+
+ /** Deletes one listed data file and reports whether this commit removed it. */
+ private boolean deleteDataFile(FileStatus file) throws IOException {
+ boolean deleted;
+ try {
+ deleted = fileIO.delete(file.getPath(), false);
+ } catch (FileNotFoundException ignore) {
+ return false;
+ }
+ if (deleted) {
+ return true;
+ }
+ if (fileIO.exists(file.getPath())) {
+ // A refusal is not a concurrent-delete race: the file is still readable, and going on
+ // would report the partition as holding nothing while its rows are still there.
+ throw new IOException(
+ String.format(
+ "Failed to delete data file %s of table %s.",
+ file.getPath(), tableIdentifier.getFullName()));
+ }
+ return false;
+ }
+
+ /**
+ * Keeps at most {@code threadNum} deletes in flight. A listing or deletion failure stops new
+ * submissions and drains accepted work; file failures are then reported in discovery order. An
+ * interrupted caller is restored only after the drain completes.
+ */
+ private Set deleteDataFilesConcurrently(PreviousDataFiles dataFiles, int threadNum)
+ throws IOException {
+ CompletionService completions =
+ new ExecutorCompletionService<>(cleanupExecutor);
+ CleanupSubmissionState submissionState = new CleanupSubmissionState();
+ Set clearedPartitionPaths = new HashSet<>();
+ Map fileFailures = new TreeMap<>();
+ List coordinatorFailures = new ArrayList<>();
+ ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
+ InterruptedException interruption = null;
+ boolean exhausted = false;
+ int nextIndex = 0;
+ int inFlight = 0;
+
+ while ((!submissionState.isStopped() && !exhausted) || inFlight > 0) {
+ while (!submissionState.isStopped() && !exhausted && inFlight < threadNum) {
+ FileStatus file;
try {
- deleted = fileIO.delete(file.getPath(), false);
- } catch (FileNotFoundException ignore) {
- continue;
- } catch (IOException e) {
- throw new RuntimeException(e);
+ file = dataFiles.next();
+ } catch (Throwable failure) {
+ submissionState.stop();
+ coordinatorFailures.add(failure);
+ break;
}
- if (deleted) {
- // Only what this commit removed: a file another writer deleted first would
- // have every concurrent writer report the whole subtree.
- clearedPartitionPaths.add(file.getPath().getParent());
- } else if (fileIO.exists(file.getPath())) {
- // A refusal is not that race: the file is still readable, and going on would
- // report the partition as holding nothing while its rows are still there.
- throw new IOException(
- String.format(
- "Failed to delete data file %s of table %s.",
- file.getPath(), tableIdentifier.getFullName()));
+ if (file == null) {
+ exhausted = true;
+ break;
}
+
+ int index = nextIndex;
+ try {
+ boolean submitted =
+ submissionState.submitIfRunning(
+ () ->
+ completions.submit(
+ () ->
+ cleanupDataFile(
+ index,
+ file,
+ contextClassLoader,
+ submissionState)));
+ if (!submitted) {
+ break;
+ }
+ nextIndex++;
+ inFlight++;
+ } catch (Throwable failure) {
+ submissionState.stop();
+ coordinatorFailures.add(failure);
+ }
+ }
+
+ if (inFlight == 0) {
+ break;
+ }
+
+ Future completed = null;
+ while (completed == null) {
+ try {
+ completed = completions.take();
+ } catch (InterruptedException e) {
+ submissionState.stop();
+ if (interruption == null) {
+ interruption = e;
+ } else {
+ interruption.addSuppressed(e);
+ }
+ }
+ }
+ inFlight--;
+
+ CleanupResult result = null;
+ while (result == null) {
+ try {
+ result = completed.get();
+ } catch (InterruptedException e) {
+ submissionState.stop();
+ if (interruption == null) {
+ interruption = e;
+ } else {
+ interruption.addSuppressed(e);
+ }
+ } catch (ExecutionException e) {
+ submissionState.stop();
+ coordinatorFailures.add(e.getCause() == null ? e : e.getCause());
+ break;
+ }
+ }
+ if (result != null) {
+ if (result.failure != null) {
+ submissionState.stop();
+ fileFailures.put(result.index, result.failure);
+ } else if (result.clearedPath != null) {
+ clearedPartitionPaths.add(result.clearedPath);
+ }
+ }
+ }
+
+ Throwable primary = aggregateFailures(fileFailures.values());
+ for (Throwable failure : coordinatorFailures) {
+ if (primary == null) {
+ primary = failure;
+ } else if (primary != failure) {
+ primary.addSuppressed(failure);
+ }
+ }
+ if (interruption != null) {
+ if (primary == null) {
+ primary = interruption;
+ } else {
+ primary.addSuppressed(interruption);
}
+ Thread.currentThread().interrupt();
+ }
+ if (primary != null) {
+ rethrowCleanupFailure(primary);
}
return clearedPartitionPaths;
}
+ private CleanupResult cleanupDataFile(
+ int index,
+ FileStatus file,
+ ClassLoader contextClassLoader,
+ CleanupSubmissionState submissionState) {
+ Thread currentThread = Thread.currentThread();
+ ClassLoader originalClassLoader = null;
+ boolean originalClassLoaderCaptured = false;
+ Path clearedPath = null;
+ Throwable failure = null;
+ try {
+ originalClassLoader = currentThread.getContextClassLoader();
+ originalClassLoaderCaptured = true;
+ currentThread.setContextClassLoader(contextClassLoader);
+ if (deleteDataFile(file)) {
+ clearedPath = file.getPath().getParent();
+ }
+ } catch (Throwable cleanupFailure) {
+ failure = cleanupFailure;
+ submissionState.stop();
+ } finally {
+ if (originalClassLoaderCaptured) {
+ try {
+ currentThread.setContextClassLoader(originalClassLoader);
+ } catch (Throwable restoreFailure) {
+ if (failure == null) {
+ failure = restoreFailure;
+ } else if (failure != restoreFailure) {
+ failure.addSuppressed(restoreFailure);
+ }
+ submissionState.stop();
+ }
+ }
+ }
+ return new CleanupResult(index, clearedPath, failure);
+ }
+
+ private static void rethrowCleanupFailure(Throwable failure) throws IOException {
+ if (failure instanceof IOException) {
+ throw (IOException) failure;
+ }
+ if (failure instanceof RuntimeException) {
+ throw (RuntimeException) failure;
+ }
+ if (failure instanceof Error) {
+ throw (Error) failure;
+ }
+ throw new IOException("Interrupted while cleaning old Format Table data files.", failure);
+ }
+
+ @Nullable
+ private static Throwable aggregateFailures(Iterable failures) {
+ Throwable primary = null;
+ for (Throwable failure : failures) {
+ if (failure == null) {
+ continue;
+ }
+ if (primary == null) {
+ primary = failure;
+ } else if (primary != failure) {
+ primary.addSuppressed(failure);
+ }
+ }
+ return primary;
+ }
+
+ private final class PreviousDataFiles {
+
+ private final List partitionPaths;
+ private final int partitionLevels;
+ private List currentFiles = Collections.emptyList();
+ @Nullable private FileStatus pushedBack;
+ private int nextPartition;
+ private int nextFile;
+
+ private PreviousDataFiles(List partitionPaths, int partitionLevels) {
+ this.partitionPaths = partitionPaths;
+ this.partitionLevels = partitionLevels;
+ }
+
+ @Nullable
+ private FileStatus next() throws IOException {
+ if (pushedBack != null) {
+ FileStatus file = pushedBack;
+ pushedBack = null;
+ return file;
+ }
+ while (nextFile >= currentFiles.size()) {
+ if (nextPartition >= partitionPaths.size()) {
+ return null;
+ }
+ Path partitionPath = partitionPaths.get(nextPartition++);
+ if (!fileIO.exists(partitionPath)) {
+ continue;
+ }
+ // Committed data files only: what sits under a staging directory is another
+ // writer's uncommitted output, whatever its name looks like.
+ currentFiles =
+ FormatTableScan.listDataFiles(
+ fileIO,
+ partitionPath,
+ partitionLevels,
+ formatTablePartitionOnlyValueInPath,
+ defaultPartName);
+ nextFile = 0;
+ }
+ return currentFiles.get(nextFile++);
+ }
+
+ private void pushBack(FileStatus file) {
+ if (pushedBack != null) {
+ throw new IllegalStateException("Only one old data file can be pushed back.");
+ }
+ pushedBack = file;
+ }
+ }
+
+ private static final class IndexedPublishMessage {
+
+ private final int index;
+ private final Path partition;
+ private final TwoPhaseCommitMessage message;
+
+ private IndexedPublishMessage(int index, Path partition, TwoPhaseCommitMessage message) {
+ this.index = index;
+ this.partition = partition;
+ this.message = message;
+ }
+ }
+
+ private static final class PublishResult {
+
+ private final int index;
+ private final Path partition;
+ @Nullable private final Throwable failure;
+
+ private PublishResult(int index, Path partition, @Nullable Throwable failure) {
+ this.index = index;
+ this.partition = partition;
+ this.failure = failure;
+ }
+ }
+
+ private static final class PublishSubmissionState {
+
+ private volatile boolean stopped;
+
+ private synchronized boolean submitIfRunning(Runnable submission) {
+ if (stopped) {
+ return false;
+ }
+ submission.run();
+ return true;
+ }
+
+ private void stop() {
+ stopped = true;
+ synchronized (this) {
+ // Wait for any accepted submission to leave the monitor.
+ }
+ }
+
+ private boolean isStopped() {
+ return stopped;
+ }
+ }
+
+ @VisibleForTesting
+ static final class CleanupSubmissionState {
+
+ private volatile boolean stopped;
+
+ synchronized boolean submitIfRunning(Runnable submission) {
+ if (stopped) {
+ return false;
+ }
+ submission.run();
+ return true;
+ }
+
+ void stop() {
+ // Publish before acquiring the monitor so another submitIfRunning call observes the
+ // stop, even if it enters the monitor before this call does.
+ stopped = true;
+ synchronized (this) {
+ // Wait for any accepted submission to leave the monitor.
+ }
+ }
+
+ boolean isStopped() {
+ return stopped;
+ }
+ }
+
+ private static final class CleanupResult {
+
+ private final int index;
+ @Nullable private final Path clearedPath;
+ @Nullable private final Throwable failure;
+
+ private CleanupResult(int index, @Nullable Path clearedPath, @Nullable Throwable failure) {
+ this.index = index;
+ this.clearedPath = clearedPath;
+ this.failure = failure;
+ }
+ }
+
@Override
public void truncateTable() {
// Data files only. The partition directories stay, and so do their catalog registrations:
diff --git a/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
index ac71ff2552aa..5f7fd1518929 100644
--- a/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/CoreOptionsTest.java
@@ -288,4 +288,78 @@ public void testLocalKvDbBlockSize() {
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("local-kv-db.block-size");
}
+
+ @Test
+ public void testFormatTableCommitCleanupThreadNumDefaultsTo64AndAcceptsBounds() {
+ Options conf = new Options();
+ assertThat(new CoreOptions(conf).formatTableCommitCleanupThreadNum()).isEqualTo(64);
+
+ conf.set(CoreOptions.FORMAT_TABLE_COMMIT_CLEANUP_THREAD_NUM, 1);
+ assertThat(new CoreOptions(conf).formatTableCommitCleanupThreadNum()).isEqualTo(1);
+
+ conf.set(CoreOptions.FORMAT_TABLE_COMMIT_CLEANUP_THREAD_NUM, 64);
+ assertThat(new CoreOptions(conf).formatTableCommitCleanupThreadNum()).isEqualTo(64);
+ }
+
+ @Test
+ public void testFormatTableCommitCleanupThreadNumRejectsValuesOutsideSupportedRange() {
+ Options conf = new Options();
+ conf.set(CoreOptions.FORMAT_TABLE_COMMIT_CLEANUP_THREAD_NUM, 0);
+ assertThatThrownBy(() -> new CoreOptions(conf).formatTableCommitCleanupThreadNum())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("format-table.commit.cleanup-thread-num")
+ .hasMessageContaining("1")
+ .hasMessageContaining("64");
+
+ conf.set(CoreOptions.FORMAT_TABLE_COMMIT_CLEANUP_THREAD_NUM, -1);
+ assertThatThrownBy(() -> new CoreOptions(conf).formatTableCommitCleanupThreadNum())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("format-table.commit.cleanup-thread-num")
+ .hasMessageContaining("1")
+ .hasMessageContaining("64");
+
+ conf.set(CoreOptions.FORMAT_TABLE_COMMIT_CLEANUP_THREAD_NUM, 65);
+ assertThatThrownBy(() -> new CoreOptions(conf).formatTableCommitCleanupThreadNum())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("format-table.commit.cleanup-thread-num")
+ .hasMessageContaining("1")
+ .hasMessageContaining("64");
+ }
+
+ @Test
+ public void testFormatTableCommitPublishThreadNumDefaultsTo64AndAcceptsBounds() {
+ Options conf = new Options();
+ assertThat(new CoreOptions(conf).formatTableCommitPublishThreadNum()).isEqualTo(64);
+
+ conf.set(CoreOptions.FORMAT_TABLE_COMMIT_PUBLISH_THREAD_NUM, 1);
+ assertThat(new CoreOptions(conf).formatTableCommitPublishThreadNum()).isEqualTo(1);
+
+ conf.set(CoreOptions.FORMAT_TABLE_COMMIT_PUBLISH_THREAD_NUM, 64);
+ assertThat(new CoreOptions(conf).formatTableCommitPublishThreadNum()).isEqualTo(64);
+ }
+
+ @Test
+ public void testFormatTableCommitPublishThreadNumRejectsValuesOutsideSupportedRange() {
+ Options conf = new Options();
+ conf.set(CoreOptions.FORMAT_TABLE_COMMIT_PUBLISH_THREAD_NUM, 0);
+ assertThatThrownBy(() -> new CoreOptions(conf).formatTableCommitPublishThreadNum())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("format-table.commit.publish-thread-num")
+ .hasMessageContaining("1")
+ .hasMessageContaining("64");
+
+ conf.set(CoreOptions.FORMAT_TABLE_COMMIT_PUBLISH_THREAD_NUM, -1);
+ assertThatThrownBy(() -> new CoreOptions(conf).formatTableCommitPublishThreadNum())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("format-table.commit.publish-thread-num")
+ .hasMessageContaining("1")
+ .hasMessageContaining("64");
+
+ conf.set(CoreOptions.FORMAT_TABLE_COMMIT_PUBLISH_THREAD_NUM, 65);
+ assertThatThrownBy(() -> new CoreOptions(conf).formatTableCommitPublishThreadNum())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("format-table.commit.publish-thread-num")
+ .hasMessageContaining("1")
+ .hasMessageContaining("64");
+ }
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitBatchDeleteTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitBatchDeleteTest.java
new file mode 100644
index 000000000000..019ffc78f244
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitBatchDeleteTest.java
@@ -0,0 +1,1666 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.BatchDeleteResult;
+import org.apache.paimon.fs.BatchFileDeleter;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.FileStatus;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.TwoPhaseOutputStream;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.partition.Partition;
+import org.apache.paimon.partition.PartitionStatistics;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.mockito.ArgumentCaptor;
+
+import javax.annotation.Nullable;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.AbstractExecutorService;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.apache.paimon.CoreOptions.PARTITION_DEFAULT_NAME;
+import static org.apache.paimon.table.format.FormatTableCommitTestUtils.failureTree;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.catchThrowable;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/** Strict batch-delete consumer tests for {@link FormatTableCommit}. */
+class FormatTableCommitBatchDeleteTest {
+
+ private static final int OSS_BATCH_SIZE = 1000;
+ private static final Identifier TABLE =
+ Identifier.create("batch_delete_db", "batch_delete_table");
+
+ @TempDir java.nio.file.Path tempDir;
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ @Test
+ void testChunks9384FilesSequentiallyAndWaitsForFinalBatchBeforePublishing() throws Exception {
+ CountDownLatch finalBatchStartedOrCommitReturned = new CountDownLatch(1);
+ Path tablePath = new Path(new Path(tempDir.toUri()), "large-batch");
+ Path firstPartition = new Path(tablePath, "part=p0");
+ Path secondPartition = new Path(tablePath, "part=p1");
+ SuccessfulBatchFileIO fileIO = new SuccessfulBatchFileIO(finalBatchStartedOrCommitReturned);
+ List oldFiles = new ArrayList<>();
+ oldFiles.addAll(fileIO.addOldFiles(firstPartition, 9000));
+ oldFiles.addAll(fileIO.addOldFiles(secondPartition, 384));
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ TrackingCommitter firstCommitter =
+ new TrackingCommitter(new Path(firstPartition, "data-new.csv"), null);
+ TrackingCommitter secondCommitter =
+ new TrackingCommitter(new Path(secondPartition, "data-new.csv"), null);
+ TrackingCommitMessage firstMessage = new TrackingCommitMessage(firstCommitter, 7, 123);
+ TrackingCommitMessage secondMessage = new TrackingCommitMessage(secondCommitter, 11, 456);
+ List messages = Arrays.asList(firstMessage, secondMessage);
+ CountingExecutorService publishExecutor =
+ new CountingExecutorService(Executors.newFixedThreadPool(2));
+ FormatTableCommit commit =
+ new FormatTableCommit(
+ tablePath.toString(),
+ Collections.singletonList("part"),
+ fileIO,
+ false,
+ PARTITION_DEFAULT_NAME.defaultValue(),
+ true,
+ TABLE,
+ null,
+ null,
+ null,
+ partitionManager,
+ true,
+ 64,
+ 2,
+ publishExecutor);
+
+ ExecutorService callerExecutor = Executors.newSingleThreadExecutor();
+ Future result =
+ callerExecutor.submit(
+ () -> {
+ try {
+ commit.commit(messages);
+ } finally {
+ fileIO.signalCommitReturned();
+ }
+ });
+ try {
+ assertThat(finalBatchStartedOrCommitReturned.await(30, TimeUnit.SECONDS)).isTrue();
+ assertThat(fileIO.finalBatchStarted()).isTrue();
+ assertThat(result.isDone()).isFalse();
+ assertThat(fileIO.batchCalls()).isEqualTo(10);
+ assertThat(fileIO.maxBatchSizeCalls()).isOne();
+ assertThat(fileIO.maxConcurrentBatchCalls()).isOne();
+ assertThat(publishExecutor.acceptedTasks()).isZero();
+ assertThat(firstCommitter.commitCalls()).isZero();
+ assertThat(secondCommitter.commitCalls()).isZero();
+ assertThat(firstCommitter.cleanCalls()).isZero();
+ assertThat(secondCommitter.cleanCalls()).isZero();
+ assertThat(firstMessage.statisticsAccessCalls()).isZero();
+ assertThat(secondMessage.statisticsAccessCalls()).isZero();
+ verify(partitionManager, never())
+ .createPartitions(anyList(), eq(true), any(), anyBoolean());
+
+ fileIO.releaseFinalBatch();
+ result.get(30, TimeUnit.SECONDS);
+ } finally {
+ fileIO.releaseFinalBatch();
+ callerExecutor.shutdownNow();
+ publishExecutor.shutdownNow();
+ }
+
+ List> expectedBatches = chunks(oldFiles, OSS_BATCH_SIZE);
+ assertThat(fileIO.discoveryPaths()).containsExactly(oldFiles.get(0));
+ assertThat(fileIO.pathExistedAtDiscovery()).containsExactly(true);
+ List> actualBatches = fileIO.batchInputs();
+ assertThat(actualBatches).containsExactlyElementsOf(expectedBatches);
+ assertThat(actualBatches.subList(0, 9))
+ .allSatisfy(batch -> assertThat(batch).hasSize(1000));
+ assertThat(actualBatches.get(9)).hasSize(384);
+ Set> identities = Collections.newSetFromMap(new IdentityHashMap<>());
+ for (List batch : actualBatches) {
+ assertThat(identities.add(batch)).as("each batch is a fresh list").isTrue();
+ }
+ assertThat(fileIO.singleDeleteCalls()).isZero();
+ assertThat(fileIO.listStatus(firstPartition)).isEmpty();
+ assertThat(fileIO.listStatus(secondPartition)).isEmpty();
+ assertThat(publishExecutor.acceptedTasks()).isEqualTo(2);
+ assertThat(firstCommitter.commitCalls()).isOne();
+ assertThat(secondCommitter.commitCalls()).isOne();
+ assertThat(firstCommitter.cleanCalls()).isOne();
+ assertThat(secondCommitter.cleanCalls()).isOne();
+ assertThat(firstCommitter.discardCalls()).isZero();
+ assertThat(secondCommitter.discardCalls()).isZero();
+ assertThat(firstMessage.statisticsAccessCalls()).isEqualTo(2);
+ assertThat(secondMessage.statisticsAccessCalls()).isEqualTo(2);
+
+ ArgumentCaptor>> specs =
+ ArgumentCaptor.forClass((Class) List.class);
+ ArgumentCaptor> statistics =
+ ArgumentCaptor.forClass((Class) List.class);
+ verify(partitionManager)
+ .createPartitions(specs.capture(), eq(true), statistics.capture(), eq(true));
+ assertThat(specs.getValue())
+ .containsExactlyInAnyOrder(
+ Collections.singletonMap("part", "p0"),
+ Collections.singletonMap("part", "p1"));
+ assertThat(statistics.getValue())
+ .hasSize(2)
+ .anySatisfy(
+ stat -> {
+ assertThat(stat.spec())
+ .isEqualTo(Collections.singletonMap("part", "p0"));
+ assertThat(stat.recordCount()).isEqualTo(7);
+ assertThat(stat.fileSizeInBytes()).isEqualTo(123);
+ assertThat(stat.fileCount()).isOne();
+ })
+ .anySatisfy(
+ stat -> {
+ assertThat(stat.spec())
+ .isEqualTo(Collections.singletonMap("part", "p1"));
+ assertThat(stat.recordCount()).isEqualTo(11);
+ assertThat(stat.fileSizeInBytes()).isEqualTo(456);
+ assertThat(stat.fileCount()).isOne();
+ });
+ }
+
+ @Test
+ void testUnsupportedCapabilityPushesFirstFileBackIntoSingleDeleteCleanup() throws Exception {
+ UnsupportedBatchFileIO fileIO = new UnsupportedBatchFileIO();
+ Path tablePath = new Path(new Path(tempDir.toUri()), "unsupported");
+ Path partitionPath = new Path(tablePath, "part=p");
+ fileIO.rejectRelisting(partitionPath);
+ List oldFiles = writeOldFiles(fileIO, partitionPath, 3);
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ TrackingCommitter committer =
+ new TrackingCommitter(
+ new Path(partitionPath, "data-new.csv"),
+ () -> {
+ if (fileIO.activeSingleDeletes() != 0) {
+ throw new IOException("Publish overlapped fallback cleanup");
+ }
+ for (Path oldFile : oldFiles) {
+ if (fileIO.exists(oldFile)) {
+ throw new IOException("Old file survived fallback: " + oldFile);
+ }
+ }
+ });
+
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future result =
+ caller.submit(
+ () ->
+ dynamicOverwrite(tablePath, fileIO, partitionManager, 2)
+ .commit(
+ Collections.singletonList(
+ new TwoPhaseCommitMessage(
+ committer, 1, 1))));
+
+ fileIO.awaitBothSingleDeletesStarted();
+ assertThat(fileIO.discoveryPaths()).containsExactly(oldFiles.get(0));
+ assertThat(fileIO.pathExistedAtDiscovery()).containsExactly(true);
+ assertThat(fileIO.singleDeletePaths())
+ .hasSize(2)
+ .doesNotHaveDuplicates()
+ .isSubsetOf(oldFiles);
+ assertThat(fileIO.activeSingleDeletes()).isEqualTo(2);
+ assertThat(fileIO.maxConcurrentSingleDeletes()).isEqualTo(2);
+ assertThat(committer.commitCalls()).isZero();
+ assertThat(committer.cleanCalls()).isZero();
+
+ fileIO.releaseSingleDeletes();
+ result.get(10, TimeUnit.SECONDS);
+ } finally {
+ fileIO.releaseSingleDeletes();
+ caller.shutdownNow();
+ }
+
+ assertThat(fileIO.discoveryPaths()).containsExactly(oldFiles.get(0));
+ assertThat(fileIO.pathExistedAtDiscovery()).containsExactly(true);
+ assertThat(fileIO.singleDeletePaths())
+ .containsExactlyInAnyOrderElementsOf(oldFiles)
+ .hasSize(oldFiles.size());
+ assertThat(fileIO.recursiveDeleteArguments()).containsOnly(false);
+ assertThat(fileIO.partitionListings()).isOne();
+ assertThat(fileIO.maxConcurrentSingleDeletes()).isEqualTo(2);
+ assertThat(committer.commitCalls()).isOne();
+ assertThat(committer.discardCalls()).isZero();
+ }
+
+ @Test
+ void testLaterRootListingFailureDoesNotSendPartialBatch() throws Exception {
+ Path tablePath = new Path(new Path(tempDir.toUri()), "later-listing-failure");
+ Path firstPartition = new Path(tablePath, "part=p0");
+ Path failingPartition = new Path(tablePath, "part=p1");
+ LaterRootListingFailureFileIO fileIO = new LaterRootListingFailureFileIO(failingPartition);
+ Path oldFile = writeOldFiles(fileIO, firstPartition, 1).get(0);
+ fileIO.mkdirs(failingPartition);
+ TrackingCommitter firstCommitter =
+ new TrackingCommitter(new Path(firstPartition, "data-new.csv"), null);
+ TrackingCommitter secondCommitter =
+ new TrackingCommitter(new Path(failingPartition, "data-new.csv"), null);
+
+ Throwable failure =
+ catchThrowable(
+ () ->
+ dynamicOverwrite(
+ tablePath,
+ fileIO,
+ mock(FormatTablePartitionManager.class),
+ 2)
+ .commit(
+ Arrays.asList(
+ new TwoPhaseCommitMessage(firstCommitter),
+ new TwoPhaseCommitMessage(
+ secondCommitter))));
+
+ assertThat(failure).isNotNull();
+ assertThat(fileIO.discoveryPaths()).containsExactly(oldFile);
+ assertThat(fileIO.maxBatchSizeCalls()).isOne();
+ assertThat(fileIO.batchCalls()).isZero();
+ assertThat(fileIO.singleDeleteCalls()).isZero();
+ assertThat(fileIO.exists(oldFile)).isTrue();
+ assertThat(firstCommitter.commitCalls()).isZero();
+ assertThat(secondCommitter.commitCalls()).isZero();
+ assertThat(firstCommitter.cleanCalls()).isZero();
+ assertThat(secondCommitter.cleanCalls()).isZero();
+ assertThat(firstCommitter.discardCalls()).isOne();
+ assertThat(secondCommitter.discardCalls()).isOne();
+ }
+
+ @Test
+ void testEmptyWrittenPartitionDoesNotDiscoverBatchCapability() throws Exception {
+ RejectingDiscoveryFileIO fileIO = new RejectingDiscoveryFileIO();
+ Path tablePath = new Path(new Path(tempDir.toUri()), "empty-partition");
+ Path partitionPath = new Path(tablePath, "part=p");
+ fileIO.mkdirs(partitionPath);
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ TrackingCommitter committer =
+ new TrackingCommitter(new Path(partitionPath, "data-new.csv"), null);
+
+ dynamicOverwrite(tablePath, fileIO, partitionManager, 2)
+ .commit(Collections.singletonList(new TwoPhaseCommitMessage(committer)));
+
+ assertThat(fileIO.discoveryCalls()).isZero();
+ assertThat(fileIO.singleDeleteCalls()).isZero();
+ assertThat(committer.commitCalls()).isOne();
+ }
+
+ @ParameterizedTest
+ @EnumSource(ExcludedMode.class)
+ void testExcludedModesNeverDiscoverBatchCapability(ExcludedMode mode) throws Exception {
+ RejectingDiscoveryFileIO fileIO = new RejectingDiscoveryFileIO();
+ Path tablePath = new Path(new Path(tempDir.toUri()), "excluded-" + mode.name());
+ Path partitionPath =
+ mode == ExcludedMode.UNPARTITIONED
+ ? tablePath
+ : mode == ExcludedMode.STATIC_PREFIX
+ ? new Path(tablePath, "year=2025/month=10")
+ : new Path(tablePath, "part=p");
+ Path oldFile = writeOldFiles(fileIO, partitionPath, 1).get(0);
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ TrackingCommitter committer =
+ new TrackingCommitter(new Path(partitionPath, "data-new.csv"), null);
+
+ runExcludedMode(mode, tablePath, fileIO, partitionManager, committer);
+
+ assertThat(fileIO.discoveryCalls()).as(mode.name()).isZero();
+ if (mode == ExcludedMode.APPEND) {
+ assertThat(fileIO.singleDeleteCalls()).as(mode.name()).isZero();
+ assertThat(fileIO.exists(oldFile)).as(mode.name()).isTrue();
+ } else {
+ assertThat(fileIO.singleDeleteCalls()).as(mode.name()).isOne();
+ assertThat(fileIO.exists(oldFile)).as(mode.name()).isFalse();
+ }
+ }
+
+ @Test
+ void testNullPartitionManagerNeverDiscoversCapabilityWithConcurrentCleanup() throws Exception {
+ RejectingDiscoveryFileIO fileIO = new RejectingDiscoveryFileIO();
+ Path tablePath = new Path(new Path(tempDir.toUri()), "null-partition-manager");
+ Path partitionPath = new Path(tablePath, "part=p");
+ Path oldFile = writeOldFiles(fileIO, partitionPath, 1).get(0);
+ TrackingCommitter committer =
+ new TrackingCommitter(new Path(partitionPath, "data-new.csv"), null);
+
+ new FormatTableCommit(
+ tablePath.toString(),
+ Collections.singletonList("part"),
+ fileIO,
+ false,
+ PARTITION_DEFAULT_NAME.defaultValue(),
+ true,
+ TABLE,
+ null,
+ null,
+ null,
+ null,
+ true,
+ 2)
+ .commit(Collections.singletonList(new TwoPhaseCommitMessage(committer)));
+
+ assertThat(fileIO.discoveryCalls()).isZero();
+ assertThat(fileIO.singleDeleteCalls()).isOne();
+ assertThat(fileIO.exists(oldFile)).isFalse();
+ assertThat(committer.commitCalls()).isOne();
+ assertThat(committer.cleanCalls()).isOne();
+ assertThat(committer.discardCalls()).isZero();
+ }
+
+ @ParameterizedTest
+ @EnumSource(DiscoveryFailure.class)
+ void testCapabilityDiscoveryErrorsAreHardFailures(DiscoveryFailure outcome) throws Exception {
+ DiscoveryFailureFileIO fileIO = new DiscoveryFailureFileIO(outcome);
+ Path tablePath = new Path(new Path(tempDir.toUri()), "discovery-" + outcome.name());
+ Path partitionPath = new Path(tablePath, "part=p");
+ Path oldFile = writeOldFiles(fileIO, partitionPath, 1).get(0);
+ TrackingCommitter committer =
+ new TrackingCommitter(new Path(partitionPath, "data-new.csv"), null);
+
+ Throwable failure =
+ catchThrowable(
+ () ->
+ dynamicOverwrite(
+ tablePath,
+ fileIO,
+ mock(FormatTablePartitionManager.class),
+ 2)
+ .commit(
+ Collections.singletonList(
+ new TwoPhaseCommitMessage(committer))));
+
+ assertThat(failure).as(outcome.name()).isNotNull();
+ assertThat(fileIO.discoveryPaths()).containsExactly(oldFile);
+ assertThat(fileIO.pathExistedAtDiscovery()).containsExactly(true);
+ assertThat(fileIO.singleDeleteCalls()).isZero();
+ assertThat(fileIO.batchCalls()).isZero();
+ assertThat(fileIO.exists(oldFile)).isTrue();
+ assertThat(committer.commitCalls()).isZero();
+ assertThat(committer.cleanCalls()).isZero();
+ assertThat(committer.discardCalls()).isOne();
+ }
+
+ @ParameterizedTest
+ @EnumSource(MaxBatchSizeFailure.class)
+ void testInvalidProviderBatchSizeIsHardFailure(MaxBatchSizeFailure outcome) throws Exception {
+ MaxBatchSizeFailureFileIO fileIO = new MaxBatchSizeFailureFileIO(outcome);
+ Path tablePath = new Path(new Path(tempDir.toUri()), "max-size-" + outcome.name());
+ Path partitionPath = new Path(tablePath, "part=p");
+ Path oldFile = writeOldFiles(fileIO, partitionPath, 1).get(0);
+ TrackingCommitter committer =
+ new TrackingCommitter(new Path(partitionPath, "data-new.csv"), null);
+
+ Throwable failure =
+ catchThrowable(
+ () ->
+ dynamicOverwrite(
+ tablePath,
+ fileIO,
+ mock(FormatTablePartitionManager.class),
+ 2)
+ .commit(
+ Collections.singletonList(
+ new TwoPhaseCommitMessage(committer))));
+
+ assertThat(failure).as(outcome.name()).isNotNull();
+ assertThat(fileIO.discoveryPaths()).containsExactly(oldFile);
+ assertThat(fileIO.maxBatchSizeCalls()).isOne();
+ assertThat(fileIO.batchCalls()).isZero();
+ assertThat(fileIO.singleDeleteCalls()).isZero();
+ assertThat(fileIO.exists(oldFile)).isTrue();
+ assertThat(committer.commitCalls()).isZero();
+ assertThat(committer.cleanCalls()).isZero();
+ assertThat(committer.discardCalls()).isOne();
+ }
+
+ @Test
+ void testMaximumProviderBatchSizeDoesNotPreallocateRequestList() throws Exception {
+ MaximumBatchSizeFileIO fileIO = new MaximumBatchSizeFileIO();
+ Path tablePath = new Path(new Path(tempDir.toUri()), "maximum-batch-size");
+ Path partitionPath = new Path(tablePath, "part=p");
+ List oldFiles = writeOldFiles(fileIO, partitionPath, 3);
+ TrackingCommitter committer =
+ new TrackingCommitter(new Path(partitionPath, "data-new.csv"), null);
+
+ dynamicOverwrite(tablePath, fileIO, mock(FormatTablePartitionManager.class), 2)
+ .commit(Collections.singletonList(new TwoPhaseCommitMessage(committer)));
+
+ assertThat(fileIO.discoveryPaths()).containsExactly(oldFiles.get(0));
+ assertThat(fileIO.maxBatchSizeCalls()).isOne();
+ assertThat(fileIO.batchCalls()).isOne();
+ assertThat(fileIO.singleDeleteCalls()).isZero();
+ assertThat(fileIO.listStatus(partitionPath)).isEmpty();
+ assertThat(committer.commitCalls()).isOne();
+ assertThat(committer.cleanCalls()).isOne();
+ assertThat(committer.discardCalls()).isZero();
+ }
+
+ @ParameterizedTest
+ @EnumSource(BatchOutcome.class)
+ void testDeleteAndResultErrorsNeverFallBackOrPublish(BatchOutcome outcome) throws Exception {
+ StrictOutcomeFileIO fileIO = new StrictOutcomeFileIO(outcome);
+ Path tablePath = new Path(new Path(tempDir.toUri()), "strict-" + outcome.name());
+ Path partitionPath = new Path(tablePath, "part=p");
+ List oldFiles = writeOldFiles(fileIO, partitionPath, 2);
+ TrackingCommitter committer =
+ new TrackingCommitter(new Path(partitionPath, "data-new.csv"), null);
+
+ Throwable failure =
+ catchThrowable(
+ () ->
+ dynamicOverwrite(
+ tablePath,
+ fileIO,
+ mock(FormatTablePartitionManager.class),
+ 2)
+ .commit(
+ Collections.singletonList(
+ new TwoPhaseCommitMessage(committer))));
+
+ assertThat(failure).as(outcome.name()).isNotNull();
+ assertThat(fileIO.discoveryPaths()).containsExactly(oldFiles.get(0));
+ assertThat(fileIO.maxBatchSizeCalls()).isOne();
+ assertThat(fileIO.batchCalls()).isOne();
+ assertThat(fileIO.batchInputs()).containsExactlyElementsOf(oldFiles);
+ assertThat(fileIO.singleDeleteCalls()).isZero();
+ assertThat(committer.commitCalls()).isZero();
+ assertThat(committer.cleanCalls()).isZero();
+ assertThat(committer.discardCalls()).isOne();
+ if (outcome == BatchOutcome.PARTIAL_DELETE_THEN_THROW) {
+ assertThat(fileIO.exists(oldFiles.get(0))).isFalse();
+ assertThat(fileIO.exists(oldFiles.get(1))).isTrue();
+ }
+ }
+
+ @Test
+ void testSecondBatchFailureStopsThirdBatchAndAbortsBeforePublish() throws Exception {
+ SecondBatchFailureFileIO fileIO = new SecondBatchFailureFileIO();
+ Path tablePath = new Path(new Path(tempDir.toUri()), "second-batch-failure");
+ Path firstPartition = new Path(tablePath, "part=p0");
+ Path secondPartition = new Path(tablePath, "part=p1");
+ List oldFiles = new ArrayList<>();
+ oldFiles.addAll(writeOldFiles(fileIO, firstPartition, 4));
+ oldFiles.addAll(writeOldFiles(fileIO, secondPartition, 1));
+ IOException abortFailure = new IOException("discard failed after batch failure");
+ TrackingCommitter firstCommitter =
+ new TrackingCommitter(
+ new Path(firstPartition, "data-new.csv"),
+ null,
+ () -> {
+ throw abortFailure;
+ });
+ TrackingCommitter secondCommitter =
+ new TrackingCommitter(new Path(secondPartition, "data-new.csv"), null);
+ TrackingCommitMessage firstMessage = new TrackingCommitMessage(firstCommitter, 3, 30);
+ TrackingCommitMessage secondMessage = new TrackingCommitMessage(secondCommitter, 4, 40);
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+
+ Throwable failure =
+ catchThrowable(
+ () ->
+ dynamicOverwrite(tablePath, fileIO, partitionManager, 2)
+ .commit(Arrays.asList(firstMessage, secondMessage)));
+
+ assertThat(failure).isNotNull();
+ assertThat(fileIO.discoveryPaths()).containsExactly(oldFiles.get(0));
+ assertThat(fileIO.maxBatchSizeCalls()).isOne();
+ assertThat(fileIO.batchCalls()).isEqualTo(2);
+ assertThat(fileIO.batchInputs())
+ .containsExactly(oldFiles.subList(0, 2), oldFiles.subList(2, 4));
+ assertThat(fileIO.singleDeleteCalls()).isZero();
+ assertThat(fileIO.exists(oldFiles.get(0))).isFalse();
+ assertThat(fileIO.exists(oldFiles.get(1))).isFalse();
+ assertThat(fileIO.exists(oldFiles.get(2))).isTrue();
+ assertThat(fileIO.exists(oldFiles.get(3))).isTrue();
+ assertThat(fileIO.exists(oldFiles.get(4))).isTrue();
+ assertThat(firstCommitter.commitCalls()).isZero();
+ assertThat(secondCommitter.commitCalls()).isZero();
+ assertThat(firstCommitter.cleanCalls()).isZero();
+ assertThat(secondCommitter.cleanCalls()).isZero();
+ assertThat(firstMessage.statisticsAccessCalls()).isZero();
+ assertThat(secondMessage.statisticsAccessCalls()).isZero();
+ verify(partitionManager, never())
+ .createPartitions(anyList(), eq(true), any(), anyBoolean());
+ assertThat(firstCommitter.discardCalls()).isOne();
+ assertThat(secondCommitter.discardCalls()).isOne();
+ assertThat(failure.getCause()).isSameAs(fileIO.cleanupFailure());
+ assertThat(failureTree(failure)).contains(fileIO.cleanupFailure(), abortFailure);
+ assertThat(fileIO.cleanupFailure().getSuppressed())
+ .singleElement()
+ .satisfies(
+ suppressed -> assertThat(failureTree(suppressed)).contains(abortFailure));
+ }
+
+ @Test
+ void testPendingInterruptAfterBatchStopsRefillAndRestoresCallerFlag() throws Exception {
+ InterruptingBatchFileIO fileIO = new InterruptingBatchFileIO();
+ Path tablePath = new Path(new Path(tempDir.toUri()), "interrupt");
+ Path partitionPath = new Path(tablePath, "part=p");
+ List oldFiles = writeOldFiles(fileIO, partitionPath, 2);
+ TrackingCommitter committer =
+ new TrackingCommitter(new Path(partitionPath, "data-new.csv"), null);
+ FormatTableCommit commit =
+ dynamicOverwrite(tablePath, fileIO, mock(FormatTablePartitionManager.class), 2);
+ AtomicReference failure = new AtomicReference<>();
+ AtomicBoolean interruptRestored = new AtomicBoolean();
+ CountDownLatch callerReturned = new CountDownLatch(1);
+ Thread caller =
+ new Thread(
+ () -> {
+ try {
+ commit.commit(
+ Collections.singletonList(
+ new TwoPhaseCommitMessage(committer)));
+ } catch (Throwable t) {
+ failure.set(t);
+ } finally {
+ interruptRestored.set(Thread.currentThread().isInterrupted());
+ callerReturned.countDown();
+ }
+ },
+ "format-batch-delete-pending-interrupt");
+
+ caller.start();
+ try {
+ assertThat(callerReturned.await(10, TimeUnit.SECONDS)).isTrue();
+ } finally {
+ caller.interrupt();
+ caller.join(TimeUnit.SECONDS.toMillis(10));
+ }
+
+ assertThat(caller.isAlive()).isFalse();
+ assertThat(failure.get()).isNotNull();
+ assertThat(failureTree(failure.get())).anyMatch(InterruptedException.class::isInstance);
+ assertThat(interruptRestored).isTrue();
+ assertThat(fileIO.batchCalls()).isOne();
+ assertThat(fileIO.singleDeleteCalls()).isZero();
+ assertThat(fileIO.exists(oldFiles.get(0))).isFalse();
+ assertThat(fileIO.exists(oldFiles.get(1))).isTrue();
+ assertThat(committer.commitCalls()).isZero();
+ assertThat(committer.cleanCalls()).isZero();
+ assertThat(committer.discardCalls()).isOne();
+ }
+
+ private void runExcludedMode(
+ ExcludedMode mode,
+ Path tablePath,
+ RejectingDiscoveryFileIO fileIO,
+ FormatTablePartitionManager partitionManager,
+ TrackingCommitter committer) {
+ Map options = options(64, true);
+ switch (mode) {
+ case APPEND:
+ ((FormatTableCommit)
+ table(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Collections.singletonList("part"),
+ options)
+ .newBatchWriteBuilder()
+ .newCommit())
+ .commit(Collections.singletonList(new TwoPhaseCommitMessage(committer)));
+ return;
+ case STATIC_PARTITION:
+ overwriteCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Collections.singletonList("part"),
+ options,
+ Collections.singletonMap("part", "p"))
+ .commit(Collections.emptyList());
+ return;
+ case STATIC_PREFIX:
+ overwriteCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Arrays.asList("year", "month"),
+ options,
+ Collections.singletonMap("year", "2025"))
+ .commit(Collections.emptyList());
+ return;
+ case WHOLE_TABLE:
+ when(partitionManager.listPartitions(Collections.emptyMap(), null))
+ .thenReturn(Collections.singletonList(partition("p")));
+ overwriteCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Collections.singletonList("part"),
+ options(64, false),
+ null)
+ .commit(Collections.emptyList());
+ return;
+ case TRUNCATE:
+ when(partitionManager.listPartitionsByNames(anyList()))
+ .thenReturn(Collections.singletonList(partition("p")));
+ ((FormatTableCommit)
+ table(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Collections.singletonList("part"),
+ options)
+ .newBatchWriteBuilder()
+ .newCommit())
+ .truncatePartitions(
+ Collections.singletonList(Collections.singletonMap("part", "p")));
+ return;
+ case TRUNCATE_TABLE:
+ when(partitionManager.listPartitions(Collections.emptyMap(), null))
+ .thenReturn(Collections.singletonList(partition("p")));
+ ((FormatTableCommit)
+ table(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Collections.singletonList("part"),
+ options)
+ .newBatchWriteBuilder()
+ .newCommit())
+ .truncateTable();
+ return;
+ case FILESYSTEM_DISCOVERED:
+ overwriteCommit(
+ tablePath,
+ fileIO,
+ null,
+ Collections.singletonList("part"),
+ options,
+ null)
+ .commit(Collections.singletonList(new TwoPhaseCommitMessage(committer)));
+ return;
+ case UNPARTITIONED:
+ overwriteCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Collections.emptyList(),
+ options,
+ null)
+ .commit(Collections.singletonList(new TwoPhaseCommitMessage(committer)));
+ return;
+ case LEGACY_CONSTRUCTOR:
+ new FormatTableCommit(
+ tablePath.toString(),
+ Collections.singletonList("part"),
+ fileIO,
+ false,
+ PARTITION_DEFAULT_NAME.defaultValue(),
+ true,
+ TABLE,
+ null,
+ null,
+ null,
+ partitionManager,
+ true)
+ .commit(Collections.singletonList(new TwoPhaseCommitMessage(committer)));
+ return;
+ case SINGLE_CLEANUP_THREAD:
+ overwriteCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Collections.singletonList("part"),
+ options(1, true),
+ null)
+ .commit(Collections.singletonList(new TwoPhaseCommitMessage(committer)));
+ return;
+ default:
+ throw new AssertionError("Unknown mode " + mode);
+ }
+ }
+
+ private FormatTableCommit dynamicOverwrite(
+ Path tablePath,
+ FileIO fileIO,
+ FormatTablePartitionManager partitionManager,
+ int cleanupThreadNum) {
+ return overwriteCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Collections.singletonList("part"),
+ options(cleanupThreadNum, true),
+ null);
+ }
+
+ private FormatTableCommit overwriteCommit(
+ Path tablePath,
+ FileIO fileIO,
+ @Nullable FormatTablePartitionManager partitionManager,
+ List partitionKeys,
+ Map options,
+ @Nullable Map staticPartition) {
+ BatchWriteBuilder writeBuilder =
+ table(tablePath, fileIO, partitionManager, partitionKeys, options)
+ .newBatchWriteBuilder();
+ writeBuilder.withOverwrite(staticPartition);
+ return (FormatTableCommit) writeBuilder.newCommit();
+ }
+
+ private FormatTable table(
+ Path tablePath,
+ FileIO fileIO,
+ @Nullable FormatTablePartitionManager partitionManager,
+ List partitionKeys,
+ Map options) {
+ RowType.Builder rowType = RowType.builder();
+ for (String partitionKey : partitionKeys) {
+ rowType.field(partitionKey, DataTypes.STRING());
+ }
+ rowType.field("id", DataTypes.INT());
+ return FormatTable.builder()
+ .fileIO(fileIO)
+ .identifier(TABLE)
+ .rowType(rowType.build())
+ .partitionKeys(partitionKeys)
+ .location(tablePath.toString())
+ .format(FormatTable.Format.CSV)
+ .options(options)
+ .partitionManager(partitionManager)
+ .build();
+ }
+
+ private static Map options(int cleanupThreadNum, boolean dynamicOverwrite) {
+ Map options = new LinkedHashMap<>();
+ options.put(
+ CoreOptions.FORMAT_TABLE_COMMIT_CLEANUP_THREAD_NUM.key(),
+ Integer.toString(cleanupThreadNum));
+ options.put(
+ CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key(), Boolean.toString(dynamicOverwrite));
+ return options;
+ }
+
+ private static Partition partition(String value) {
+ return new Partition(Collections.singletonMap("part", value), 0, 0, 0, 0, -1, false);
+ }
+
+ private static List writeOldFiles(SortedLocalFileIO fileIO, Path partitionPath, int count)
+ throws IOException {
+ List files = new ArrayList<>(count);
+ for (int i = 0; i < count; i++) {
+ Path file = new Path(partitionPath, String.format("data-%05d.csv", i));
+ fileIO.writeFile(file, "old", false);
+ files.add(file);
+ }
+ return files;
+ }
+
+ private static List> chunks(List paths, int size) {
+ List> chunks = new ArrayList<>();
+ for (int start = 0; start < paths.size(); start += size) {
+ chunks.add(new ArrayList<>(paths.subList(start, Math.min(start + size, paths.size()))));
+ }
+ return chunks;
+ }
+
+ private enum ExcludedMode {
+ APPEND,
+ STATIC_PARTITION,
+ STATIC_PREFIX,
+ WHOLE_TABLE,
+ TRUNCATE,
+ TRUNCATE_TABLE,
+ FILESYSTEM_DISCOVERED,
+ UNPARTITIONED,
+ LEGACY_CONSTRUCTOR,
+ SINGLE_CLEANUP_THREAD
+ }
+
+ private enum DiscoveryFailure {
+ THROW_IO_EXCEPTION,
+ RETURN_NULL_OPTIONAL
+ }
+
+ private enum MaxBatchSizeFailure {
+ THROW_RUNTIME_EXCEPTION,
+ RETURN_ZERO,
+ RETURN_NEGATIVE
+ }
+
+ private enum BatchOutcome {
+ PARTIAL_DELETE_THEN_THROW,
+ FILE_NOT_FOUND_EXCEPTION,
+ NULL_RESULT,
+ NULL_RESULT_LIST,
+ RESULT_ACCESS_THROWS,
+ MISSING_PATH,
+ REVERSED_PATHS,
+ EXTRA_PATH,
+ DUPLICATE_PATH
+ }
+
+ private interface CheckedAction {
+ void run() throws IOException;
+ }
+
+ private static final class TrackingCommitter implements TwoPhaseOutputStream.Committer {
+
+ private static final long serialVersionUID = 1L;
+
+ private final Path targetPath;
+ @Nullable private final CheckedAction commitAction;
+ @Nullable private final CheckedAction discardAction;
+ private final AtomicInteger commitCalls = new AtomicInteger();
+ private final AtomicInteger discardCalls = new AtomicInteger();
+ private final AtomicInteger cleanCalls = new AtomicInteger();
+
+ private TrackingCommitter(Path targetPath, @Nullable CheckedAction commitAction) {
+ this(targetPath, commitAction, null);
+ }
+
+ private TrackingCommitter(
+ Path targetPath,
+ @Nullable CheckedAction commitAction,
+ @Nullable CheckedAction discardAction) {
+ this.targetPath = targetPath;
+ this.commitAction = commitAction;
+ this.discardAction = discardAction;
+ }
+
+ @Override
+ public void commit(FileIO fileIO) throws IOException {
+ commitCalls.incrementAndGet();
+ if (commitAction != null) {
+ commitAction.run();
+ }
+ }
+
+ @Override
+ public void discard(FileIO fileIO) throws IOException {
+ discardCalls.incrementAndGet();
+ if (discardAction != null) {
+ discardAction.run();
+ }
+ }
+
+ @Override
+ public Path targetPath() {
+ return targetPath;
+ }
+
+ @Override
+ public void clean(FileIO fileIO) {
+ cleanCalls.incrementAndGet();
+ }
+
+ private int commitCalls() {
+ return commitCalls.get();
+ }
+
+ private int discardCalls() {
+ return discardCalls.get();
+ }
+
+ private int cleanCalls() {
+ return cleanCalls.get();
+ }
+ }
+
+ private static final class TrackingCommitMessage extends TwoPhaseCommitMessage {
+
+ private static final long serialVersionUID = 1L;
+
+ private final AtomicInteger statisticsAccessCalls = new AtomicInteger();
+
+ private TrackingCommitMessage(
+ TwoPhaseOutputStream.Committer committer, long recordCount, long fileSizeInBytes) {
+ super(committer, recordCount, fileSizeInBytes);
+ }
+
+ @Override
+ public long recordCount() {
+ statisticsAccessCalls.incrementAndGet();
+ return super.recordCount();
+ }
+
+ @Override
+ public long fileSizeInBytes() {
+ statisticsAccessCalls.incrementAndGet();
+ return super.fileSizeInBytes();
+ }
+
+ private int statisticsAccessCalls() {
+ return statisticsAccessCalls.get();
+ }
+ }
+
+ private static final class CountingExecutorService extends AbstractExecutorService {
+
+ private final ExecutorService delegate;
+ private final AtomicInteger acceptedTasks = new AtomicInteger();
+
+ private CountingExecutorService(ExecutorService delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void shutdown() {
+ delegate.shutdown();
+ }
+
+ @Override
+ public List shutdownNow() {
+ return delegate.shutdownNow();
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return delegate.isShutdown();
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return delegate.isTerminated();
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ return delegate.awaitTermination(timeout, unit);
+ }
+
+ @Override
+ public void execute(Runnable command) {
+ acceptedTasks.incrementAndGet();
+ delegate.execute(command);
+ }
+
+ private int acceptedTasks() {
+ return acceptedTasks.get();
+ }
+ }
+
+ private abstract static class SortedLocalFileIO extends LocalFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ FileStatus[] statuses = super.listStatus(path);
+ Arrays.sort(statuses, Comparator.comparing(status -> status.getPath().toString()));
+ return statuses;
+ }
+ }
+
+ private abstract static class StrictBatchFileIO extends SortedLocalFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final AtomicInteger discoveryCalls = new AtomicInteger();
+ private final AtomicInteger maxBatchSizeCalls = new AtomicInteger();
+ private final AtomicInteger batchCalls = new AtomicInteger();
+ private final AtomicInteger singleDeleteCalls = new AtomicInteger();
+ private final ConcurrentLinkedQueue discoveryPaths = new ConcurrentLinkedQueue<>();
+ private final ConcurrentLinkedQueue pathExistedAtDiscovery =
+ new ConcurrentLinkedQueue<>();
+
+ final void recordDiscovery(Path path) throws IOException {
+ discoveryCalls.incrementAndGet();
+ discoveryPaths.add(path);
+ pathExistedAtDiscovery.add(exists(path));
+ }
+
+ final int recordMaxBatchSizeCall() {
+ return maxBatchSizeCalls.incrementAndGet();
+ }
+
+ final int recordBatchCall() {
+ return batchCalls.incrementAndGet();
+ }
+
+ boolean deleteInBatch(Path path) throws IOException {
+ return super.delete(path, false);
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive) {
+ singleDeleteCalls.incrementAndGet();
+ throw new AssertionError("Strict batch mode attempted single delete for " + path);
+ }
+
+ final int discoveryCalls() {
+ return discoveryCalls.get();
+ }
+
+ final int maxBatchSizeCalls() {
+ return maxBatchSizeCalls.get();
+ }
+
+ final int batchCalls() {
+ return batchCalls.get();
+ }
+
+ final int singleDeleteCalls() {
+ return singleDeleteCalls.get();
+ }
+
+ final List discoveryPaths() {
+ return new ArrayList<>(discoveryPaths);
+ }
+
+ final List pathExistedAtDiscovery() {
+ return new ArrayList<>(pathExistedAtDiscovery);
+ }
+ }
+
+ private static final class SuccessfulBatchFileIO extends StrictBatchFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final CountDownLatch finalBatchStartedOrCommitReturned;
+ private final CountDownLatch releaseFinalBatch = new CountDownLatch(1);
+ private final AtomicBoolean finalBatchStarted = new AtomicBoolean();
+ private final AtomicInteger activeBatchCalls = new AtomicInteger();
+ private final AtomicInteger maxConcurrentBatchCalls = new AtomicInteger();
+ private final List> batchInputs =
+ Collections.synchronizedList(new ArrayList<>());
+ private final Map> filesByPartition = new HashMap<>();
+ private final Set existingFiles = new HashSet<>();
+
+ private SuccessfulBatchFileIO(CountDownLatch finalBatchStartedOrCommitReturned) {
+ this.finalBatchStartedOrCommitReturned = finalBatchStartedOrCommitReturned;
+ }
+
+ private List addOldFiles(Path partition, int count) {
+ List files = new ArrayList<>(count);
+ for (int i = 0; i < count; i++) {
+ Path file = new Path(partition, String.format("data-%05d.csv", i));
+ files.add(file);
+ existingFiles.add(file);
+ }
+ filesByPartition.put(partition, files);
+ return new ArrayList<>(files);
+ }
+
+ @Override
+ public boolean exists(Path path) {
+ return filesByPartition.containsKey(path) || existingFiles.contains(path);
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path path) {
+ List files = filesByPartition.get(path);
+ if (files == null) {
+ return new FileStatus[0];
+ }
+ List statuses = new ArrayList<>();
+ for (Path file : files) {
+ if (existingFiles.contains(file)) {
+ statuses.add(new SyntheticFileStatus(file));
+ }
+ }
+ return statuses.toArray(new FileStatus[0]);
+ }
+
+ @Override
+ boolean deleteInBatch(Path path) {
+ return existingFiles.remove(path);
+ }
+
+ @Override
+ public Optional batchFileDeleter(Path path) throws IOException {
+ recordDiscovery(path);
+ return Optional.of(
+ new BatchFileDeleter() {
+ @Override
+ public int maxBatchSize() {
+ recordMaxBatchSizeCall();
+ return OSS_BATCH_SIZE;
+ }
+
+ @Override
+ public BatchDeleteResult delete(List files) throws IOException {
+ int active = activeBatchCalls.incrementAndGet();
+ maxConcurrentBatchCalls.accumulateAndGet(active, Math::max);
+ try {
+ int call = recordBatchCall();
+ assertImmutable(files);
+ batchInputs.add(files);
+ if (call == 10) {
+ finalBatchStarted.set(true);
+ finalBatchStartedOrCommitReturned.countDown();
+ await(releaseFinalBatch, "final batch release");
+ }
+ for (Path file : files) {
+ if (!deleteInBatch(file)) {
+ throw new IOException("Provider did not delete " + file);
+ }
+ }
+ List equalButDistinctPaths = new ArrayList<>(files.size());
+ for (Path file : files) {
+ equalButDistinctPaths.add(new Path(file.toString()));
+ }
+ return new BatchDeleteResult(equalButDistinctPaths);
+ } finally {
+ activeBatchCalls.decrementAndGet();
+ }
+ }
+ });
+ }
+
+ private static void assertImmutable(List files) {
+ try {
+ files.set(0, files.get(0));
+ throw new AssertionError("Consumer passed a mutable batch list");
+ } catch (UnsupportedOperationException expected) {
+ // Required: providers may retain a chunk while validating the request.
+ }
+ }
+
+ private void signalCommitReturned() {
+ finalBatchStartedOrCommitReturned.countDown();
+ }
+
+ private boolean finalBatchStarted() {
+ return finalBatchStarted.get();
+ }
+
+ private void releaseFinalBatch() {
+ releaseFinalBatch.countDown();
+ }
+
+ private int maxConcurrentBatchCalls() {
+ return maxConcurrentBatchCalls.get();
+ }
+
+ private List> batchInputs() {
+ synchronized (batchInputs) {
+ return new ArrayList<>(batchInputs);
+ }
+ }
+ }
+
+ private static final class SyntheticFileStatus implements FileStatus {
+
+ private final Path path;
+
+ private SyntheticFileStatus(Path path) {
+ this.path = path;
+ }
+
+ @Override
+ public long getLen() {
+ return 3;
+ }
+
+ @Override
+ public boolean isDir() {
+ return false;
+ }
+
+ @Override
+ public Path getPath() {
+ return path;
+ }
+
+ @Override
+ public long getModificationTime() {
+ return 0;
+ }
+ }
+
+ private static final class UnsupportedBatchFileIO extends SortedLocalFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final ConcurrentLinkedQueue discoveryPaths = new ConcurrentLinkedQueue<>();
+ private final ConcurrentLinkedQueue pathExistedAtDiscovery =
+ new ConcurrentLinkedQueue<>();
+ private final ConcurrentLinkedQueue singleDeletePaths = new ConcurrentLinkedQueue<>();
+ private final ConcurrentLinkedQueue recursiveDeleteArguments =
+ new ConcurrentLinkedQueue<>();
+ private final AtomicInteger activeSingleDeletes = new AtomicInteger();
+ private final AtomicInteger maxConcurrentSingleDeletes = new AtomicInteger();
+ private final AtomicInteger partitionListings = new AtomicInteger();
+ private final CountDownLatch bothSingleDeletesStarted = new CountDownLatch(2);
+ private final CountDownLatch releaseSingleDeletes = new CountDownLatch(1);
+ @Nullable private Path partitionWhichMustNotBeRelisted;
+
+ private void rejectRelisting(Path partitionPath) {
+ partitionWhichMustNotBeRelisted = partitionPath;
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ if (path.equals(partitionWhichMustNotBeRelisted)
+ && partitionListings.incrementAndGet() > 1) {
+ throw new IOException("Unsupported batch fallback relisted " + path);
+ }
+ return super.listStatus(path);
+ }
+
+ @Override
+ public Optional batchFileDeleter(Path path) throws IOException {
+ discoveryPaths.add(path);
+ pathExistedAtDiscovery.add(super.exists(path));
+ return Optional.empty();
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive) throws IOException {
+ singleDeletePaths.add(path);
+ recursiveDeleteArguments.add(recursive);
+ int active = activeSingleDeletes.incrementAndGet();
+ maxConcurrentSingleDeletes.accumulateAndGet(active, Math::max);
+ bothSingleDeletesStarted.countDown();
+ try {
+ await(releaseSingleDeletes, "unsupported fallback single deletes to be released");
+ return super.delete(path, recursive);
+ } finally {
+ activeSingleDeletes.decrementAndGet();
+ }
+ }
+
+ private List discoveryPaths() {
+ return new ArrayList<>(discoveryPaths);
+ }
+
+ private List pathExistedAtDiscovery() {
+ return new ArrayList<>(pathExistedAtDiscovery);
+ }
+
+ private List singleDeletePaths() {
+ return new ArrayList<>(singleDeletePaths);
+ }
+
+ private List recursiveDeleteArguments() {
+ return new ArrayList<>(recursiveDeleteArguments);
+ }
+
+ private int activeSingleDeletes() {
+ return activeSingleDeletes.get();
+ }
+
+ private int maxConcurrentSingleDeletes() {
+ return maxConcurrentSingleDeletes.get();
+ }
+
+ private int partitionListings() {
+ return partitionListings.get();
+ }
+
+ private void awaitBothSingleDeletesStarted() throws IOException {
+ await(bothSingleDeletesStarted, "both unsupported fallback single deletes to start");
+ }
+
+ private void releaseSingleDeletes() {
+ releaseSingleDeletes.countDown();
+ }
+ }
+
+ private static final class RejectingDiscoveryFileIO extends SortedLocalFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final AtomicInteger discoveryCalls = new AtomicInteger();
+ private final AtomicInteger singleDeleteCalls = new AtomicInteger();
+
+ @Override
+ public Optional batchFileDeleter(Path path) {
+ discoveryCalls.incrementAndGet();
+ throw new AssertionError("Excluded cleanup discovered batch capability for " + path);
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive) throws IOException {
+ singleDeleteCalls.incrementAndGet();
+ return super.delete(path, recursive);
+ }
+
+ private int discoveryCalls() {
+ return discoveryCalls.get();
+ }
+
+ private int singleDeleteCalls() {
+ return singleDeleteCalls.get();
+ }
+ }
+
+ private static final class DiscoveryFailureFileIO extends StrictBatchFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final DiscoveryFailure outcome;
+
+ private DiscoveryFailureFileIO(DiscoveryFailure outcome) {
+ this.outcome = outcome;
+ }
+
+ @Override
+ public Optional batchFileDeleter(Path path) throws IOException {
+ recordDiscovery(path);
+ if (outcome == DiscoveryFailure.THROW_IO_EXCEPTION) {
+ throw new IOException("capability discovery failed");
+ }
+ return null;
+ }
+ }
+
+ private static final class MaxBatchSizeFailureFileIO extends StrictBatchFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final MaxBatchSizeFailure outcome;
+
+ private MaxBatchSizeFailureFileIO(MaxBatchSizeFailure outcome) {
+ this.outcome = outcome;
+ }
+
+ @Override
+ public Optional batchFileDeleter(Path path) throws IOException {
+ recordDiscovery(path);
+ return Optional.of(
+ new BatchFileDeleter() {
+ @Override
+ public int maxBatchSize() {
+ recordMaxBatchSizeCall();
+ if (outcome == MaxBatchSizeFailure.THROW_RUNTIME_EXCEPTION) {
+ throw new IllegalStateException("provider size failed");
+ }
+ return outcome == MaxBatchSizeFailure.RETURN_ZERO ? 0 : -1;
+ }
+
+ @Override
+ public BatchDeleteResult delete(List files) {
+ recordBatchCall();
+ throw new AssertionError("Delete called after invalid max batch size");
+ }
+ });
+ }
+ }
+
+ private static final class MaximumBatchSizeFileIO extends StrictBatchFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Optional batchFileDeleter(Path path) throws IOException {
+ recordDiscovery(path);
+ return Optional.of(
+ new BatchFileDeleter() {
+ @Override
+ public int maxBatchSize() {
+ recordMaxBatchSizeCall();
+ return Integer.MAX_VALUE;
+ }
+
+ @Override
+ public BatchDeleteResult delete(List files) throws IOException {
+ recordBatchCall();
+ List confirmed = new ArrayList<>(files.size());
+ for (Path file : files) {
+ if (!deleteInBatch(file)) {
+ throw new IOException("Provider did not delete " + file);
+ }
+ confirmed.add(new Path(file.toString()));
+ }
+ return new BatchDeleteResult(confirmed);
+ }
+ });
+ }
+ }
+
+ private static final class StrictOutcomeFileIO extends StrictBatchFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final BatchOutcome outcome;
+ private final List batchInputs = new ArrayList<>();
+
+ private StrictOutcomeFileIO(BatchOutcome outcome) {
+ this.outcome = outcome;
+ }
+
+ @Override
+ public Optional batchFileDeleter(Path path) throws IOException {
+ recordDiscovery(path);
+ return Optional.of(
+ new BatchFileDeleter() {
+ @Override
+ public int maxBatchSize() {
+ recordMaxBatchSizeCall();
+ return OSS_BATCH_SIZE;
+ }
+
+ @Override
+ public BatchDeleteResult delete(List files) throws IOException {
+ recordBatchCall();
+ batchInputs.addAll(files);
+ switch (outcome) {
+ case PARTIAL_DELETE_THEN_THROW:
+ if (!deleteInBatch(files.get(0))) {
+ throw new IOException("Could not create partial success");
+ }
+ throw new IOException("response lost after partial success");
+ case FILE_NOT_FOUND_EXCEPTION:
+ throw new FileNotFoundException("provider batch disappeared");
+ case NULL_RESULT:
+ return null;
+ case NULL_RESULT_LIST:
+ BatchDeleteResult nullList = mock(BatchDeleteResult.class);
+ when(nullList.deletedOrNotFound()).thenReturn(null);
+ return nullList;
+ case RESULT_ACCESS_THROWS:
+ BatchDeleteResult throwingResult =
+ mock(BatchDeleteResult.class);
+ when(throwingResult.deletedOrNotFound())
+ .thenThrow(
+ new IllegalStateException(
+ "result access failed"));
+ return throwingResult;
+ case MISSING_PATH:
+ return new BatchDeleteResult(
+ Collections.singletonList(files.get(0)));
+ case REVERSED_PATHS:
+ return new BatchDeleteResult(
+ Arrays.asList(files.get(1), files.get(0)));
+ case EXTRA_PATH:
+ return new BatchDeleteResult(
+ Arrays.asList(
+ files.get(0),
+ files.get(1),
+ new Path(
+ files.get(0).getParent(),
+ "unrequested.csv")));
+ case DUPLICATE_PATH:
+ return new BatchDeleteResult(
+ Arrays.asList(files.get(0), files.get(0)));
+ default:
+ throw new AssertionError("Unknown outcome " + outcome);
+ }
+ }
+ });
+ }
+
+ private List batchInputs() {
+ return new ArrayList<>(batchInputs);
+ }
+ }
+
+ private static final class SecondBatchFailureFileIO extends StrictBatchFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final IOException cleanupFailure = new IOException("batch 2 failed");
+ private final List> batchInputs = new ArrayList<>();
+
+ @Override
+ public Optional batchFileDeleter(Path path) throws IOException {
+ recordDiscovery(path);
+ return Optional.of(
+ new BatchFileDeleter() {
+ @Override
+ public int maxBatchSize() {
+ recordMaxBatchSizeCall();
+ return 2;
+ }
+
+ @Override
+ public BatchDeleteResult delete(List files) throws IOException {
+ int call = recordBatchCall();
+ batchInputs.add(new ArrayList<>(files));
+ if (call == 2) {
+ throw cleanupFailure;
+ }
+ if (call > 2) {
+ throw new AssertionError("Batch 3 started after batch 2 failed");
+ }
+ for (Path file : files) {
+ if (!deleteInBatch(file)) {
+ throw new IOException("First batch did not delete " + file);
+ }
+ }
+ return new BatchDeleteResult(files);
+ }
+ });
+ }
+
+ private List> batchInputs() {
+ return new ArrayList<>(batchInputs);
+ }
+
+ private IOException cleanupFailure() {
+ return cleanupFailure;
+ }
+ }
+
+ private static final class LaterRootListingFailureFileIO extends StrictBatchFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final Path failingRoot;
+
+ private LaterRootListingFailureFileIO(Path failingRoot) {
+ this.failingRoot = failingRoot;
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ if (failingRoot.equals(path)) {
+ throw new IOException("later partition listing failed");
+ }
+ return super.listStatus(path);
+ }
+
+ @Override
+ public Optional batchFileDeleter(Path path) throws IOException {
+ recordDiscovery(path);
+ return Optional.of(
+ new BatchFileDeleter() {
+ @Override
+ public int maxBatchSize() {
+ recordMaxBatchSizeCall();
+ return OSS_BATCH_SIZE;
+ }
+
+ @Override
+ public BatchDeleteResult delete(List files) {
+ recordBatchCall();
+ throw new AssertionError("Partial batch sent before listing completed");
+ }
+ });
+ }
+ }
+
+ private static final class InterruptingBatchFileIO extends StrictBatchFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Optional batchFileDeleter(Path path) throws IOException {
+ recordDiscovery(path);
+ return Optional.of(
+ new BatchFileDeleter() {
+ @Override
+ public int maxBatchSize() {
+ recordMaxBatchSizeCall();
+ return 1;
+ }
+
+ @Override
+ public BatchDeleteResult delete(List files) throws IOException {
+ int call = recordBatchCall();
+ if (call != 1) {
+ throw new AssertionError(
+ "Batch refill started despite pending interrupt");
+ }
+ if (!deleteInBatch(files.get(0))) {
+ throw new IOException("First interrupt batch was not deleted");
+ }
+ Thread.currentThread().interrupt();
+ return new BatchDeleteResult(files);
+ }
+ });
+ }
+ }
+
+ private static void await(CountDownLatch latch, String description) throws IOException {
+ try {
+ if (!latch.await(30, TimeUnit.SECONDS)) {
+ throw new IOException("Timed out waiting for " + description);
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while waiting for " + description, e);
+ }
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitPublishTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitPublishTest.java
new file mode 100644
index 000000000000..05d4ebfe3ebc
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitPublishTest.java
@@ -0,0 +1,2131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.FileStatus;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.TwoPhaseOutputStream;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.partition.PartitionStatistics;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.format.FormatTableCommitTestUtils.PartialBarrierDeleteFileIO;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.AbstractExecutorService;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.apache.paimon.CoreOptions.PARTITION_DEFAULT_NAME;
+import static org.apache.paimon.table.format.FormatTableCommitTestUtils.awaitFailure;
+import static org.apache.paimon.table.format.FormatTableCommitTestUtils.failureTree;
+import static org.apache.paimon.table.format.FormatTableCommitTestUtils.observeContextClassLoaders;
+import static org.apache.paimon.table.format.FormatTableCommitTestUtils.rootCause;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+/** Tests concurrent publication for {@link FormatTableCommit}. */
+class FormatTableCommitPublishTest {
+
+ @TempDir java.nio.file.Path tempDir;
+
+ @Test
+ void testCatalogManagedBuilderUses64WayPublishByDefault() throws Exception {
+ FileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ ParallelPublishProbe probe = new ParallelPublishProbe(64);
+ List messages = new ArrayList<>();
+ for (int i = 0; i < 65; i++) {
+ Path partitionPath = new Path(tablePath, "part=p" + i);
+ messages.add(
+ new TwoPhaseCommitMessage(
+ new ProbeCommitter(
+ new Path(partitionPath, "data-new.csv"), probe::publish)));
+ }
+ FormatTableCommit commit =
+ builderAppendCommit(tablePath, fileIO, partitionManager, Collections.emptyMap());
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future result = caller.submit(() -> commit.commit(messages));
+
+ assertThat(probe.awaitFirstWave()).isTrue();
+ assertThat(probe.publishCalls()).isEqualTo(64);
+ assertThat(probe.awaitUnexpectedExtraPublish()).isFalse();
+
+ probe.releaseFirstWave();
+ result.get(10, TimeUnit.SECONDS);
+ assertThat(probe.publishCalls()).isEqualTo(65);
+ assertThat(probe.maxConcurrentPublishes()).isEqualTo(64);
+ } finally {
+ probe.releaseFirstWave();
+ caller.shutdownNow();
+ }
+ }
+
+ @Test
+ void testCatalogManagedBuilderHonorsConfiguredPublishConcurrency() throws Exception {
+ FileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ ParallelPublishProbe probe = new ParallelPublishProbe(2);
+ List messages = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ Path partitionPath = new Path(tablePath, "part=p" + i);
+ messages.add(
+ new TwoPhaseCommitMessage(
+ new ProbeCommitter(
+ new Path(partitionPath, "data-new.csv"), probe::publish)));
+ }
+ FormatTableCommit commit =
+ builderAppendCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Collections.singletonMap(
+ CoreOptions.FORMAT_TABLE_COMMIT_PUBLISH_THREAD_NUM.key(), "2"));
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future result = caller.submit(() -> commit.commit(messages));
+
+ assertThat(probe.awaitFirstWave()).isTrue();
+ assertThat(probe.publishCalls()).isEqualTo(2);
+ assertThat(probe.awaitUnexpectedExtraPublish()).isFalse();
+
+ probe.releaseFirstWave();
+ result.get(10, TimeUnit.SECONDS);
+ assertThat(probe.publishCalls()).isEqualTo(3);
+ assertThat(probe.maxConcurrentPublishes()).isEqualTo(2);
+ } finally {
+ probe.releaseFirstWave();
+ caller.shutdownNow();
+ }
+ }
+
+ @Test
+ void testConfiguredSerialPublishRunsOnTheCaller() {
+ FileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ ConcurrentLinkedQueue publishingThreads = new ConcurrentLinkedQueue<>();
+ List messages = publishMessages(tablePath, publishingThreads, true);
+ FormatTableCommit commit =
+ builderAppendCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Collections.singletonMap(
+ CoreOptions.FORMAT_TABLE_COMMIT_PUBLISH_THREAD_NUM.key(), "1"));
+ Thread caller = Thread.currentThread();
+
+ commit.commit(messages);
+
+ assertThat(publishingThreads).hasSize(3).containsOnly(caller);
+ }
+
+ @Test
+ void testPublishConcurrencyStaysOffOutsideCatalogManagedPartitionedTables() {
+ FileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Map configured64 =
+ Collections.singletonMap(
+ CoreOptions.FORMAT_TABLE_COMMIT_PUBLISH_THREAD_NUM.key(), "64");
+ Thread caller = Thread.currentThread();
+
+ ConcurrentLinkedQueue filesystemThreads = new ConcurrentLinkedQueue<>();
+ builderAppendCommit(tablePath, fileIO, null, configured64)
+ .commit(publishMessages(tablePath, filesystemThreads, true));
+ assertThat(filesystemThreads).hasSize(3).containsOnly(caller);
+
+ ConcurrentLinkedQueue unpartitionedThreads = new ConcurrentLinkedQueue<>();
+ builderUnpartitionedAppendCommit(
+ new Path(tablePath, "unpartitioned"),
+ fileIO,
+ mock(FormatTablePartitionManager.class),
+ configured64)
+ .commit(
+ publishMessages(
+ new Path(tablePath, "unpartitioned"), unpartitionedThreads, false));
+ assertThat(unpartitionedThreads).hasSize(3).containsOnly(caller);
+
+ FormatTablePartitionManager legacyPartitionManager =
+ mock(FormatTablePartitionManager.class);
+ FormatTableCommit legacy =
+ new FormatTableCommit(
+ tablePath.toString(),
+ Collections.singletonList("part"),
+ fileIO,
+ false,
+ PARTITION_DEFAULT_NAME.defaultValue(),
+ false,
+ Identifier.create("publish_db", "legacy_publish_table"),
+ null,
+ null,
+ null,
+ legacyPartitionManager,
+ /* dynamicPartitionOverwrite */ true);
+ ConcurrentLinkedQueue legacyThreads = new ConcurrentLinkedQueue<>();
+ legacy.commit(publishMessages(tablePath, legacyThreads, true));
+ assertThat(legacyThreads).hasSize(3).containsOnly(caller);
+ }
+
+ @Test
+ void testPublishKeepsSamePartitionOrderedWhileOtherPartitionsOverlap() throws Exception {
+ FileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Path p0 = new Path(tablePath, "part=p0");
+ Path p1 = new Path(tablePath, "part=p1");
+ CountDownLatch p0FirstStarted = new CountDownLatch(1);
+ CountDownLatch p1Started = new CountDownLatch(1);
+ CountDownLatch p0SecondStarted = new CountDownLatch(1);
+ CountDownLatch releaseP0First = new CountDownLatch(1);
+ CountDownLatch releaseP1 = new CountDownLatch(1);
+ ConcurrentLinkedQueue events = new ConcurrentLinkedQueue<>();
+ ProbeCommitter p0First =
+ new ProbeCommitter(
+ new Path(p0, "data-0.csv"),
+ () -> {
+ events.add("p0-first-start");
+ p0FirstStarted.countDown();
+ awaitPublishLatch(releaseP0First, "first p0 publish release");
+ events.add("p0-first-end");
+ });
+ ProbeCommitter p0Second =
+ new ProbeCommitter(
+ new Path(p0, "data-1.csv"),
+ () -> {
+ events.add("p0-second");
+ p0SecondStarted.countDown();
+ });
+ ProbeCommitter p1Only =
+ new ProbeCommitter(
+ new Path(p1, "data-0.csv"),
+ () -> {
+ events.add("p1-start");
+ p1Started.countDown();
+ awaitPublishLatch(releaseP1, "p1 publish release");
+ });
+ List messages =
+ Arrays.asList(
+ new TwoPhaseCommitMessage(p0First),
+ new TwoPhaseCommitMessage(p0Second),
+ new TwoPhaseCommitMessage(p1Only));
+ ExecutorService publishExecutor = Executors.newFixedThreadPool(2);
+ FormatTableCommit commit =
+ newPublishCommit(
+ tablePath,
+ fileIO,
+ mock(FormatTablePartitionManager.class),
+ false,
+ null,
+ 1,
+ 2,
+ publishExecutor);
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future result = caller.submit(() -> commit.commit(messages));
+
+ assertThat(p0FirstStarted.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(p1Started.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(p0SecondStarted.getCount()).isOne();
+
+ releaseP0First.countDown();
+ releaseP1.countDown();
+ result.get(10, TimeUnit.SECONDS);
+
+ assertThat(p0Second.commitCalls()).isOne();
+ assertThat(new ArrayList<>(events)).containsSubsequence("p0-first-end", "p0-second");
+ } finally {
+ releaseP0First.countDown();
+ releaseP1.countDown();
+ caller.shutdownNow();
+ publishExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testCleanupFullyDrainsBeforeConcurrentPublishStarts() throws Exception {
+ PartialBarrierDeleteFileIO fileIO = new PartialBarrierDeleteFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ Path p0 = new Path(tablePath, "part=p0");
+ Path p1 = new Path(tablePath, "part=p1");
+ fileIO.writeFile(new Path(p0, "data-000.csv"), "old", false);
+ fileIO.writeFile(new Path(p1, "data-001.csv"), "old", false);
+ CountDownLatch publishesStarted = new CountDownLatch(2);
+ PublishAction publish =
+ () -> {
+ if (fileIO.activeDeletes() != 0) {
+ throw new IOException("Publish overlapped overwrite cleanup");
+ }
+ publishesStarted.countDown();
+ };
+ List messages =
+ Arrays.asList(
+ new TwoPhaseCommitMessage(
+ new ProbeCommitter(new Path(p0, "data-new.csv"), publish)),
+ new TwoPhaseCommitMessage(
+ new ProbeCommitter(new Path(p1, "data-new.csv"), publish)));
+ ExecutorService publishExecutor = Executors.newFixedThreadPool(2);
+ FormatTableCommit commit =
+ newPublishCommit(
+ tablePath,
+ fileIO,
+ mock(FormatTablePartitionManager.class),
+ true,
+ null,
+ 2,
+ 2,
+ publishExecutor);
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future result = caller.submit(() -> commit.commit(messages));
+
+ assertThat(fileIO.awaitBothDeletesStarted()).isTrue();
+ assertThat(publishesStarted.getCount()).isEqualTo(2);
+
+ fileIO.releaseFirstDelete();
+ assertThat(fileIO.awaitFirstDeleteReturned()).isTrue();
+ assertThat(result.isDone()).isFalse();
+ assertThat(publishesStarted.getCount()).isEqualTo(2);
+
+ fileIO.releaseSecondDelete();
+ assertThat(publishesStarted.await(10, TimeUnit.SECONDS)).isTrue();
+ result.get(10, TimeUnit.SECONDS);
+ } finally {
+ fileIO.releaseFirstDelete();
+ fileIO.releaseSecondDelete();
+ caller.shutdownNow();
+ publishExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testPublishFailureStopsRefillDrainsAcceptedWorkThenAborts() throws Exception {
+ FileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Path p0 = new Path(tablePath, "part=p0");
+ Path p1 = new Path(tablePath, "part=p1");
+ Path p2 = new Path(tablePath, "part=p2");
+ Path p3 = new Path(tablePath, "part=p3");
+ PublishExecutorTracker publishExecutor =
+ new PublishExecutorTracker(Executors.newFixedThreadPool(3));
+ CountDownLatch acceptedPublishesStarted = new CountDownLatch(3);
+ CountDownLatch releaseFirstBlockedPublish = new CountDownLatch(1);
+ CountDownLatch releaseSecondBlockedPublish = new CountDownLatch(1);
+ CountDownLatch firstBlockedPublishReturned = new CountDownLatch(1);
+ CountDownLatch secondBlockedPublishReturned = new CountDownLatch(1);
+ CountDownLatch discards = new CountDownLatch(5);
+ AtomicInteger activePublishes = new AtomicInteger();
+ PublishAction discard =
+ () -> {
+ if (activePublishes.get() != 0) {
+ throw new IOException("Abort overlapped an accepted publish");
+ }
+ discards.countDown();
+ };
+ ProbeCommitter failing =
+ new ProbeCommitter(
+ new Path(p0, "data-fail.csv"),
+ () -> {
+ activePublishes.incrementAndGet();
+ acceptedPublishesStarted.countDown();
+ try {
+ awaitPublishLatch(
+ acceptedPublishesStarted,
+ "three accepted publishes to start");
+ publishExecutor.markCurrentTaskForCompletion();
+ throw new IOException("publish failed");
+ } finally {
+ activePublishes.decrementAndGet();
+ }
+ },
+ PublishAction.NOOP,
+ discard);
+ ProbeCommitter firstBlocked =
+ new ProbeCommitter(
+ new Path(p1, "data-blocked-first.csv"),
+ () -> {
+ activePublishes.incrementAndGet();
+ acceptedPublishesStarted.countDown();
+ try {
+ awaitPublishLatch(
+ acceptedPublishesStarted,
+ "three accepted publishes to start");
+ awaitPublishLatch(
+ releaseFirstBlockedPublish,
+ "first blocked publish release");
+ } finally {
+ activePublishes.decrementAndGet();
+ firstBlockedPublishReturned.countDown();
+ }
+ },
+ PublishAction.NOOP,
+ discard);
+ ProbeCommitter secondBlocked =
+ new ProbeCommitter(
+ new Path(p2, "data-blocked-second.csv"),
+ () -> {
+ activePublishes.incrementAndGet();
+ acceptedPublishesStarted.countDown();
+ try {
+ awaitPublishLatch(
+ acceptedPublishesStarted,
+ "three accepted publishes to start");
+ awaitPublishLatch(
+ releaseSecondBlockedPublish,
+ "second blocked publish release");
+ } finally {
+ activePublishes.decrementAndGet();
+ secondBlockedPublishReturned.countDown();
+ }
+ },
+ PublishAction.NOOP,
+ discard);
+ ProbeCommitter samePartitionPending =
+ new ProbeCommitter(
+ new Path(p1, "data-must-not-start.csv"),
+ () -> {
+ throw new IOException("same-partition refill ran after failure");
+ },
+ PublishAction.NOOP,
+ discard);
+ ProbeCommitter otherPartitionPending =
+ new ProbeCommitter(
+ new Path(p3, "data-must-not-start.csv"),
+ () -> {
+ throw new IOException("new partition ran after failure");
+ },
+ PublishAction.NOOP,
+ discard);
+ List messages =
+ Arrays.asList(
+ new TwoPhaseCommitMessage(failing),
+ new TwoPhaseCommitMessage(firstBlocked),
+ new TwoPhaseCommitMessage(samePartitionPending),
+ new TwoPhaseCommitMessage(secondBlocked),
+ new TwoPhaseCommitMessage(otherPartitionPending));
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ FormatTableCommit commit =
+ newPublishCommit(
+ tablePath, fileIO, partitionManager, false, null, 1, 3, publishExecutor);
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future result = caller.submit(() -> commit.commit(messages));
+
+ assertThat(acceptedPublishesStarted.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(publishExecutor.awaitSelectedTaskCompletion()).isTrue();
+ assertThat(samePartitionPending.commitCalls()).isZero();
+ assertThat(otherPartitionPending.commitCalls()).isZero();
+ assertThat(discards.getCount()).isEqualTo(5);
+
+ releaseFirstBlockedPublish.countDown();
+ assertThat(firstBlockedPublishReturned.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(result.isDone()).isFalse();
+ assertThat(discards.getCount()).isEqualTo(5);
+ assertThat(samePartitionPending.commitCalls()).isZero();
+ assertThat(otherPartitionPending.commitCalls()).isZero();
+
+ releaseSecondBlockedPublish.countDown();
+ assertThat(secondBlockedPublishReturned.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThatThrownBy(() -> result.get(10, TimeUnit.SECONDS))
+ .isInstanceOf(ExecutionException.class)
+ .hasRootCauseMessage("publish failed");
+
+ assertThat(discards.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(activePublishes).hasValue(0);
+ assertThat(samePartitionPending.commitCalls()).isZero();
+ assertThat(otherPartitionPending.commitCalls()).isZero();
+ assertThat(failing.cleanCalls()).isZero();
+ assertThat(firstBlocked.cleanCalls()).isZero();
+ assertThat(secondBlocked.cleanCalls()).isZero();
+ assertThat(samePartitionPending.cleanCalls()).isZero();
+ assertThat(otherPartitionPending.cleanCalls()).isZero();
+ verify(partitionManager, never())
+ .createPartitions(anyList(), eq(true), any(), anyBoolean());
+ } finally {
+ releaseFirstBlockedPublish.countDown();
+ releaseSecondBlockedPublish.countDown();
+ caller.shutdownNow();
+ publishExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testCallerInterruptStopsPublishRefillDrainsAndRestoresFlag() throws Exception {
+ FileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ CountDownLatch acceptedPublishesStarted = new CountDownLatch(2);
+ CountDownLatch releaseFirstPublish = new CountDownLatch(1);
+ CountDownLatch releaseSecondPublish = new CountDownLatch(1);
+ CountDownLatch firstPublishReturned = new CountDownLatch(1);
+ CountDownLatch secondPublishReturned = new CountDownLatch(1);
+ CountDownLatch callerReturned = new CountDownLatch(1);
+ CountDownLatch discards = new CountDownLatch(3);
+ AtomicInteger activePublishes = new AtomicInteger();
+ AtomicReference failure = new AtomicReference<>();
+ AtomicBoolean interruptRestored = new AtomicBoolean();
+ PublishAction discard =
+ () -> {
+ if (activePublishes.get() != 0) {
+ throw new IOException("Abort overlapped an accepted publish");
+ }
+ discards.countDown();
+ };
+ ProbeCommitter first =
+ blockingPublishCommitter(
+ new Path(tablePath, "part=p0/data-first.csv"),
+ acceptedPublishesStarted,
+ releaseFirstPublish,
+ firstPublishReturned,
+ activePublishes,
+ discard);
+ ProbeCommitter second =
+ blockingPublishCommitter(
+ new Path(tablePath, "part=p1/data-second.csv"),
+ acceptedPublishesStarted,
+ releaseSecondPublish,
+ secondPublishReturned,
+ activePublishes,
+ discard);
+ ProbeCommitter pending =
+ new ProbeCommitter(
+ new Path(tablePath, "part=p2/data-must-not-start.csv"),
+ () -> {
+ throw new IOException("Publish refilled after caller interruption");
+ },
+ PublishAction.NOOP,
+ discard);
+ List messages =
+ Arrays.asList(
+ new TwoPhaseCommitMessage(first),
+ new TwoPhaseCommitMessage(second),
+ new TwoPhaseCommitMessage(pending));
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ ExecutorService publishExecutor = Executors.newFixedThreadPool(2);
+ FormatTableCommit commit =
+ newPublishCommit(
+ tablePath, fileIO, partitionManager, false, null, 1, 2, publishExecutor);
+ Thread caller =
+ new Thread(
+ () -> {
+ try {
+ commit.commit(messages);
+ } catch (Throwable t) {
+ failure.set(t);
+ } finally {
+ interruptRestored.set(Thread.currentThread().isInterrupted());
+ callerReturned.countDown();
+ }
+ },
+ "format-publish-interrupted-caller");
+
+ caller.start();
+ try {
+ assertThat(acceptedPublishesStarted.await(10, TimeUnit.SECONDS)).isTrue();
+ caller.interrupt();
+
+ releaseFirstPublish.countDown();
+ assertThat(firstPublishReturned.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(callerReturned.getCount()).isOne();
+ assertThat(discards.getCount()).isEqualTo(3);
+ assertThat(pending.commitCalls()).isZero();
+
+ releaseSecondPublish.countDown();
+ assertThat(secondPublishReturned.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(callerReturned.await(10, TimeUnit.SECONDS)).isTrue();
+
+ assertThat(failure.get()).isNotNull();
+ assertThat(failureTree(failure.get())).anyMatch(InterruptedException.class::isInstance);
+ assertThat(interruptRestored).isTrue();
+ assertThat(pending.commitCalls()).isZero();
+ assertThat(discards.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(activePublishes).hasValue(0);
+ assertThat(first.cleanCalls()).isZero();
+ assertThat(second.cleanCalls()).isZero();
+ assertThat(pending.cleanCalls()).isZero();
+ verify(partitionManager, never())
+ .createPartitions(anyList(), eq(true), any(), anyBoolean());
+ } finally {
+ releaseFirstPublish.countDown();
+ releaseSecondPublish.countDown();
+ caller.interrupt();
+ caller.join(TimeUnit.SECONDS.toMillis(10));
+ publishExecutor.shutdownNow();
+ }
+ assertThat(caller.isAlive()).isFalse();
+
+ assertPendingInterruptStopsSuccessRefill(new Path(tablePath, "completion-interrupt-race"));
+ }
+
+ private void assertPendingInterruptStopsSuccessRefill(Path tablePath) throws Exception {
+ FileIO fileIO = LocalFileIO.create();
+ CountDownLatch secondPublishStarted = new CountDownLatch(1);
+ CountDownLatch releaseSecondPublish = new CountDownLatch(1);
+ CountDownLatch callerReturned = new CountDownLatch(1);
+ AtomicReference failure = new AtomicReference<>();
+ AtomicReference callerThread = new AtomicReference<>();
+ AtomicBoolean interruptRestored = new AtomicBoolean();
+ ProbeCommitter first =
+ new ProbeCommitter(
+ new Path(tablePath, "part=p0/data-first.csv"),
+ () -> callerThread.get().signalPendingInterrupt());
+ ProbeCommitter second =
+ new ProbeCommitter(
+ new Path(tablePath, "part=p1/data-second.csv"),
+ () -> {
+ secondPublishStarted.countDown();
+ try {
+ if (!releaseSecondPublish.await(10, TimeUnit.SECONDS)) {
+ throw new IOException(
+ "Timed out waiting to release second publish");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Second publish was interrupted", e);
+ }
+ });
+ ProbeCommitter pending =
+ new ProbeCommitter(
+ new Path(tablePath, "part=p2/data-must-not-start.csv"),
+ () -> {
+ throw new IOException(
+ "Publish refilled before pending interrupt check");
+ });
+ List messages =
+ Arrays.asList(
+ new TwoPhaseCommitMessage(first),
+ new TwoPhaseCommitMessage(second),
+ new TwoPhaseCommitMessage(pending));
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ ExecutorService publishExecutor = Executors.newFixedThreadPool(2);
+ FormatTableCommit commit =
+ newPublishCommit(
+ tablePath, fileIO, partitionManager, false, null, 1, 2, publishExecutor);
+ PendingInterruptThread caller =
+ new PendingInterruptThread(
+ () -> {
+ try {
+ commit.commit(messages);
+ } catch (Throwable t) {
+ failure.set(t);
+ } finally {
+ interruptRestored.set(Thread.currentThread().isInterrupted());
+ callerReturned.countDown();
+ }
+ },
+ "format-publish-pending-interrupt-caller");
+ callerThread.set(caller);
+
+ caller.start();
+ try {
+ assertThat(secondPublishStarted.await(10, TimeUnit.SECONDS)).isTrue();
+ releaseSecondPublish.countDown();
+ assertThat(callerReturned.await(10, TimeUnit.SECONDS)).isTrue();
+
+ assertThat(failure.get()).isNotNull();
+ assertThat(failureTree(failure.get())).anyMatch(InterruptedException.class::isInstance);
+ assertThat(interruptRestored).isTrue();
+ assertThat(pending.commitCalls()).isZero();
+ assertThat(first.cleanCalls()).isZero();
+ assertThat(second.cleanCalls()).isZero();
+ assertThat(pending.cleanCalls()).isZero();
+ verify(partitionManager, never())
+ .createPartitions(anyList(), eq(true), any(), anyBoolean());
+ } finally {
+ releaseSecondPublish.countDown();
+ caller.interrupt();
+ caller.join(TimeUnit.SECONDS.toMillis(10));
+ publishExecutor.shutdownNow();
+ }
+ assertThat(caller.isAlive()).isFalse();
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ @Test
+ void testSuccessfulPublishBarrierKeepsCleanStatisticsAndCatalogOnCaller() throws Exception {
+ FileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Path p0 = new Path(tablePath, "part=p0");
+ Path p1 = new Path(tablePath, "part=p1");
+ CountDownLatch p0PublishesReturned = new CountDownLatch(2);
+ CountDownLatch p1PublishStarted = new CountDownLatch(1);
+ CountDownLatch releaseP1Publish = new CountDownLatch(1);
+ AtomicInteger activePublishes = new AtomicInteger();
+ AtomicReference callerThread = new AtomicReference<>();
+ ConcurrentLinkedQueue cleanThreads = new ConcurrentLinkedQueue<>();
+ ConcurrentLinkedQueue catalogThreads = new ConcurrentLinkedQueue<>();
+ PublishAction clean =
+ () -> {
+ if (activePublishes.get() != 0) {
+ throw new IOException("Clean overlapped a publish");
+ }
+ cleanThreads.add(Thread.currentThread());
+ };
+ ProbeCommitter p0First =
+ new ProbeCommitter(
+ new Path(p0, "data-0.csv"),
+ p0PublishesReturned::countDown,
+ clean,
+ PublishAction.NOOP);
+ ProbeCommitter p0Second =
+ new ProbeCommitter(
+ new Path(p0, "data-1.csv"),
+ p0PublishesReturned::countDown,
+ clean,
+ PublishAction.NOOP);
+ ProbeCommitter p1Only =
+ new ProbeCommitter(
+ new Path(p1, "data-0.csv"),
+ () -> {
+ activePublishes.incrementAndGet();
+ p1PublishStarted.countDown();
+ try {
+ awaitPublishLatch(releaseP1Publish, "p1 publish release");
+ } finally {
+ activePublishes.decrementAndGet();
+ }
+ },
+ clean,
+ PublishAction.NOOP);
+ TrackingTwoPhaseCommitMessage p0FirstMessage =
+ new TrackingTwoPhaseCommitMessage(p0First, 3, 30);
+ TrackingTwoPhaseCommitMessage p0SecondMessage =
+ new TrackingTwoPhaseCommitMessage(p0Second, 4, 40);
+ TrackingTwoPhaseCommitMessage p1OnlyMessage =
+ new TrackingTwoPhaseCommitMessage(p1Only, 5, 50);
+ List trackedMessages =
+ Arrays.asList(p0FirstMessage, p0SecondMessage, p1OnlyMessage);
+ List messages = new ArrayList<>(trackedMessages);
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ doAnswer(
+ invocation -> {
+ if (activePublishes.get() != 0) {
+ throw new AssertionError("Catalog update overlapped a publish");
+ }
+ catalogThreads.add(Thread.currentThread());
+ return null;
+ })
+ .when(partitionManager)
+ .createPartitions(anyList(), eq(true), anyList(), eq(false));
+ ExecutorService publishExecutor = Executors.newFixedThreadPool(2);
+ FormatTableCommit commit =
+ newPublishCommit(
+ tablePath, fileIO, partitionManager, false, null, 1, 2, publishExecutor);
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future result =
+ caller.submit(
+ () -> {
+ callerThread.set(Thread.currentThread());
+ commit.commit(messages);
+ });
+
+ assertThat(p1PublishStarted.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(p0PublishesReturned.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(activePublishes).hasValue(1);
+ assertThat(result.isDone()).isFalse();
+ assertThat(p0First.cleanCalls()).isZero();
+ assertThat(p0Second.cleanCalls()).isZero();
+ assertThat(p1Only.cleanCalls()).isZero();
+ assertThat(catalogThreads).isEmpty();
+ assertThat(trackedMessages)
+ .allSatisfy(
+ message -> {
+ assertThat(message.recordCountCalls()).isZero();
+ assertThat(message.fileSizeCalls()).isZero();
+ });
+
+ releaseP1Publish.countDown();
+ result.get(10, TimeUnit.SECONDS);
+
+ assertThat(cleanThreads).hasSize(3).containsOnly(callerThread.get());
+ assertThat(catalogThreads).containsOnly(callerThread.get());
+ assertThat(trackedMessages)
+ .allSatisfy(
+ message -> {
+ assertThat(message.recordCountCalls()).isOne();
+ assertThat(message.fileSizeCalls()).isOne();
+ assertThat(message.statisticsAccessThreads())
+ .containsOnly(callerThread.get());
+ });
+ ArgumentCaptor>> specs =
+ ArgumentCaptor.forClass((Class) List.class);
+ ArgumentCaptor> statistics =
+ ArgumentCaptor.forClass((Class) List.class);
+ verify(partitionManager)
+ .createPartitions(specs.capture(), eq(true), statistics.capture(), eq(false));
+ assertThat(specs.getValue())
+ .containsExactlyInAnyOrder(
+ Collections.singletonMap("part", "p0"),
+ Collections.singletonMap("part", "p1"));
+ PartitionStatistics p0Statistics =
+ statistics.getValue().stream()
+ .filter(stat -> "p0".equals(stat.spec().get("part")))
+ .findFirst()
+ .orElseThrow(AssertionError::new);
+ PartitionStatistics p1Statistics =
+ statistics.getValue().stream()
+ .filter(stat -> "p1".equals(stat.spec().get("part")))
+ .findFirst()
+ .orElseThrow(AssertionError::new);
+ assertThat(p0Statistics.recordCount()).isEqualTo(7);
+ assertThat(p0Statistics.fileSizeInBytes()).isEqualTo(70);
+ assertThat(p0Statistics.fileCount()).isEqualTo(2);
+ assertThat(p1Statistics.recordCount()).isEqualTo(5);
+ assertThat(p1Statistics.fileSizeInBytes()).isEqualTo(50);
+ assertThat(p1Statistics.fileCount()).isOne();
+ } finally {
+ releaseP1Publish.countDown();
+ caller.shutdownNow();
+ publishExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testPublishPropagatesAndRestoresTcclAndInitializesFileIoOnCaller() throws Exception {
+ FirstAccessTrackingFileIO fileIO = new FirstAccessTrackingFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ PublishExecutorTracker publishExecutor =
+ new PublishExecutorTracker(Executors.newFixedThreadPool(2));
+ ClassLoader workerLoader = new ClassLoader(null) {};
+ ClassLoader callerLoader = new ClassLoader(null) {};
+ setContextClassLoaderOnWorkers(publishExecutor, 2, workerLoader);
+ ConcurrentLinkedQueue observedLoaders = new ConcurrentLinkedQueue<>();
+ PublishAction successfulPublish =
+ () -> {
+ observedLoaders.add(Thread.currentThread().getContextClassLoader());
+ fileIO.exists(tablePath);
+ };
+ PublishAction failingPublish =
+ () -> {
+ observedLoaders.add(Thread.currentThread().getContextClassLoader());
+ fileIO.exists(tablePath);
+ throw new IOException("tccl publish failure");
+ };
+ List messages =
+ Arrays.asList(
+ new TwoPhaseCommitMessage(
+ new ProbeCommitter(
+ new Path(tablePath, "part=p0/data-0.csv"),
+ successfulPublish)),
+ new TwoPhaseCommitMessage(
+ new ProbeCommitter(
+ new Path(tablePath, "part=p1/data-0.csv"),
+ failingPublish)));
+ FormatTableCommit commit =
+ newPublishCommit(
+ tablePath,
+ fileIO,
+ mock(FormatTablePartitionManager.class),
+ false,
+ null,
+ 1,
+ 2,
+ publishExecutor);
+ Thread caller = Thread.currentThread();
+ ClassLoader previousCallerLoader = caller.getContextClassLoader();
+ try {
+ caller.setContextClassLoader(callerLoader);
+ publishExecutor.armFileIoInitializationCheck(fileIO, caller);
+
+ assertThatThrownBy(() -> commit.commit(messages))
+ .hasRootCauseMessage("tccl publish failure");
+
+ assertThat(fileIO.firstAccessThread()).isSameAs(caller);
+ assertThat(publishExecutor.fileIoAcceptanceChecks()).isEqualTo(2);
+ assertThat(observedLoaders).hasSize(2).containsOnly(callerLoader);
+ assertThat(caller.getContextClassLoader()).isSameAs(callerLoader);
+ publishExecutor.disarmFileIoInitializationCheck();
+ assertThat(observeContextClassLoaders(publishExecutor, 2)).containsOnly(workerLoader);
+ } finally {
+ caller.setContextClassLoader(previousCallerLoader);
+ publishExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testPublishFailureUsesLowestInputIndexAndSuppressesLaterAndAbortFailures()
+ throws Exception {
+ FileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ PublishExecutorTracker publishExecutor =
+ new PublishExecutorTracker(Executors.newFixedThreadPool(2));
+ CountDownLatch bothPublishesStarted = new CountDownLatch(2);
+ CountDownLatch releaseLowerIndexFailure = new CountDownLatch(1);
+ ProbeCommitter lowerIndex =
+ new ProbeCommitter(
+ new Path(tablePath, "part=p0/data-low-index.csv"),
+ () -> {
+ bothPublishesStarted.countDown();
+ awaitPublishLatch(
+ bothPublishesStarted, "both failing publishes to start");
+ awaitPublishLatch(
+ releaseLowerIndexFailure, "lower-index failure release");
+ throw new IOException("lower-index publish failure");
+ });
+ ProbeCommitter higherIndex =
+ new ProbeCommitter(
+ new Path(tablePath, "part=p1/data-high-index.csv"),
+ () -> {
+ bothPublishesStarted.countDown();
+ awaitPublishLatch(
+ bothPublishesStarted, "both failing publishes to start");
+ publishExecutor.markCurrentTaskForCompletion();
+ throw new IOException("higher-index publish failure");
+ },
+ PublishAction.NOOP,
+ () -> {
+ throw new IOException("abort failed after publish failure");
+ });
+ ProbeCommitter afterAbortFailure =
+ new ProbeCommitter(
+ new Path(tablePath, "part=p2/data-after-abort-failure.csv"),
+ () -> {
+ throw new IOException("Publish refilled after failure");
+ });
+ List messages =
+ Arrays.asList(
+ new TwoPhaseCommitMessage(lowerIndex),
+ new TwoPhaseCommitMessage(higherIndex),
+ new TwoPhaseCommitMessage(afterAbortFailure));
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ FormatTableCommit commit =
+ newPublishCommit(
+ tablePath, fileIO, partitionManager, false, null, 1, 2, publishExecutor);
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future result = caller.submit(() -> commit.commit(messages));
+
+ assertThat(publishExecutor.awaitSelectedTaskCompletion()).isTrue();
+ releaseLowerIndexFailure.countDown();
+ ExecutionException failure = awaitFailure(result);
+ Throwable primary = rootCause(failure);
+
+ assertThat(primary).isInstanceOf(IOException.class);
+ assertThat(primary).hasMessage("lower-index publish failure");
+ assertThat(primary.getSuppressed()).hasSize(2);
+ assertThat(failureTree(primary.getSuppressed()[0]))
+ .extracting(Throwable::getMessage)
+ .contains("higher-index publish failure");
+ assertThat(failureTree(primary.getSuppressed()[1]))
+ .extracting(Throwable::getMessage)
+ .contains("abort failed after publish failure");
+ assertThat(lowerIndex.discardCalls()).isOne();
+ assertThat(higherIndex.discardCalls()).isOne();
+ assertThat(afterAbortFailure.discardCalls()).isOne();
+ assertThat(afterAbortFailure.commitCalls()).isZero();
+ assertThat(lowerIndex.cleanCalls()).isZero();
+ assertThat(higherIndex.cleanCalls()).isZero();
+ assertThat(afterAbortFailure.cleanCalls()).isZero();
+ verify(partitionManager, never())
+ .createPartitions(anyList(), eq(true), any(), anyBoolean());
+ } finally {
+ releaseLowerIndexFailure.countDown();
+ caller.shutdownNow();
+ publishExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testSharedPublishExecutorLetsSmallCommitRunBeforeLargeCommitRefill() throws Exception {
+ FileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ CountDownLatch largeFirstWaveStarted = new CountDownLatch(64);
+ CountDownLatch releaseOneLargePublish = new CountDownLatch(1);
+ CountDownLatch releaseRemainingLargePublishes = new CountDownLatch(1);
+ CountDownLatch largeRefillStarted = new CountDownLatch(1);
+ CountDownLatch smallPublishStarted = new CountDownLatch(1);
+ CountDownLatch releaseSmallPublish = new CountDownLatch(1);
+ CountDownLatch largeCallerReturned = new CountDownLatch(1);
+ CountDownLatch smallCallerReturned = new CountDownLatch(1);
+ AtomicReference largeFailure = new AtomicReference<>();
+ AtomicReference smallFailure = new AtomicReference<>();
+ SubmitterTrackingExecutor publishExecutor =
+ new SubmitterTrackingExecutor(Executors.newFixedThreadPool(64));
+ List largeMessages = new ArrayList<>();
+ for (int i = 0; i < 65; i++) {
+ int index = i;
+ largeMessages.add(
+ new TwoPhaseCommitMessage(
+ new ProbeCommitter(
+ new Path(
+ tablePath,
+ "part=large-" + index + "/data-" + index + ".csv"),
+ () -> {
+ if (index < 64) {
+ largeFirstWaveStarted.countDown();
+ awaitPublishLatch(
+ largeFirstWaveStarted,
+ "large publish first wave");
+ awaitPublishLatch(
+ index == 0
+ ? releaseOneLargePublish
+ : releaseRemainingLargePublishes,
+ "large publish release");
+ } else {
+ largeRefillStarted.countDown();
+ awaitPublishLatch(
+ releaseRemainingLargePublishes,
+ "large refill release");
+ }
+ })));
+ }
+ List smallMessages =
+ Arrays.asList(
+ new TwoPhaseCommitMessage(
+ new ProbeCommitter(
+ new Path(tablePath, "part=small-0/data-0.csv"),
+ () -> {
+ smallPublishStarted.countDown();
+ awaitPublishLatch(
+ releaseSmallPublish,
+ "small publish fairness release");
+ })),
+ new TwoPhaseCommitMessage(
+ new ProbeCommitter(
+ new Path(tablePath, "part=small-1/data-0.csv"),
+ PublishAction.NOOP)));
+ FormatTableCommit largeCommit =
+ newPublishCommit(
+ tablePath,
+ fileIO,
+ mock(FormatTablePartitionManager.class),
+ false,
+ null,
+ 1,
+ 64,
+ publishExecutor);
+ FormatTableCommit smallCommit =
+ newPublishCommit(
+ tablePath,
+ fileIO,
+ mock(FormatTablePartitionManager.class),
+ false,
+ null,
+ 1,
+ 64,
+ publishExecutor);
+ Thread largeCaller =
+ commitCaller(
+ "format-publish-large-caller",
+ largeCommit,
+ largeMessages,
+ largeFailure,
+ largeCallerReturned);
+ Thread smallCaller =
+ commitCaller(
+ "format-publish-small-caller",
+ smallCommit,
+ smallMessages,
+ smallFailure,
+ smallCallerReturned);
+ publishExecutor.trackSubmissionsFrom(smallCaller);
+
+ largeCaller.start();
+ try {
+ assertThat(largeFirstWaveStarted.await(10, TimeUnit.SECONDS)).isTrue();
+ smallCaller.start();
+ assertThat(publishExecutor.awaitTrackedSubmission()).isTrue();
+
+ releaseOneLargePublish.countDown();
+ assertThat(smallPublishStarted.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(largeRefillStarted.getCount()).isOne();
+
+ releaseSmallPublish.countDown();
+ releaseRemainingLargePublishes.countDown();
+ assertThat(largeCallerReturned.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(smallCallerReturned.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(largeFailure.get()).isNull();
+ assertThat(smallFailure.get()).isNull();
+ } finally {
+ releaseOneLargePublish.countDown();
+ releaseSmallPublish.countDown();
+ releaseRemainingLargePublishes.countDown();
+ largeCaller.interrupt();
+ smallCaller.interrupt();
+ largeCaller.join(TimeUnit.SECONDS.toMillis(10));
+ smallCaller.join(TimeUnit.SECONDS.toMillis(10));
+ publishExecutor.shutdownNow();
+ }
+ assertThat(largeCaller.isAlive()).isFalse();
+ assertThat(smallCaller.isAlive()).isFalse();
+ }
+
+ @Test
+ void testPartialExecutorRejectionDrainsAcceptedPublishBeforeAbort() throws Exception {
+ FileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ CountDownLatch firstPublishStarted = new CountDownLatch(1);
+ CountDownLatch releaseFirstPublish = new CountDownLatch(1);
+ CountDownLatch firstPublishReturned = new CountDownLatch(1);
+ CountDownLatch higherIndexFailureStarted = new CountDownLatch(1);
+ CountDownLatch releaseHigherIndexFailure = new CountDownLatch(1);
+ CountDownLatch higherIndexFailureReturned = new CountDownLatch(1);
+ CountDownLatch discards = new CountDownLatch(4);
+ AtomicInteger activePublishes = new AtomicInteger();
+ PublishAction discard =
+ () -> {
+ if (activePublishes.get() != 0) {
+ throw new IOException("Abort overlapped accepted publish after rejection");
+ }
+ discards.countDown();
+ };
+ ProbeCommitter first =
+ new ProbeCommitter(
+ new Path(tablePath, "part=p0/data-0.csv"),
+ () -> {
+ activePublishes.incrementAndGet();
+ firstPublishStarted.countDown();
+ try {
+ awaitPublishLatch(releaseFirstPublish, "accepted publish release");
+ } finally {
+ activePublishes.decrementAndGet();
+ firstPublishReturned.countDown();
+ }
+ },
+ PublishAction.NOOP,
+ discard);
+ ProbeCommitter rejected =
+ new ProbeCommitter(
+ new Path(tablePath, "part=p0/data-1.csv"),
+ () -> {
+ throw new IOException("Rejected publish executed");
+ },
+ PublishAction.NOOP,
+ discard);
+ ProbeCommitter higherIndexFailure =
+ new ProbeCommitter(
+ new Path(tablePath, "part=p1/data-0.csv"),
+ () -> {
+ activePublishes.incrementAndGet();
+ higherIndexFailureStarted.countDown();
+ try {
+ awaitPublishLatch(
+ releaseHigherIndexFailure,
+ "higher-index accepted publish failure release");
+ throw new IOException("higher-index accepted publish failed");
+ } finally {
+ activePublishes.decrementAndGet();
+ higherIndexFailureReturned.countDown();
+ }
+ },
+ PublishAction.NOOP,
+ discard);
+ ProbeCommitter pending =
+ new ProbeCommitter(
+ new Path(tablePath, "part=p0/data-2.csv"),
+ () -> {
+ throw new IOException("Publish refilled after rejection");
+ },
+ PublishAction.NOOP,
+ discard);
+ List messages =
+ Arrays.asList(
+ new TwoPhaseCommitMessage(first),
+ new TwoPhaseCommitMessage(rejected),
+ new TwoPhaseCommitMessage(higherIndexFailure),
+ new TwoPhaseCommitMessage(pending));
+ RejectThirdSubmissionExecutor publishExecutor =
+ new RejectThirdSubmissionExecutor(Executors.newFixedThreadPool(2));
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ FormatTableCommit commit =
+ newPublishCommit(
+ tablePath, fileIO, partitionManager, false, null, 1, 2, publishExecutor);
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future result = caller.submit(() -> commit.commit(messages));
+
+ assertThat(firstPublishStarted.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(higherIndexFailureStarted.await(10, TimeUnit.SECONDS)).isTrue();
+ releaseFirstPublish.countDown();
+ assertThat(firstPublishReturned.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(publishExecutor.awaitRejection()).isTrue();
+ assertThat(result.isDone()).isFalse();
+ assertThat(discards.getCount()).isEqualTo(4);
+ assertThat(rejected.commitCalls()).isZero();
+ assertThat(pending.commitCalls()).isZero();
+
+ releaseHigherIndexFailure.countDown();
+ assertThat(higherIndexFailureReturned.await(10, TimeUnit.SECONDS)).isTrue();
+ ExecutionException failure = awaitFailure(result);
+
+ Throwable primary = rootCause(failure);
+ assertThat(primary)
+ .isInstanceOf(RejectedExecutionException.class)
+ .hasMessage("publish submission rejected");
+ assertThat(primary.getSuppressed())
+ .singleElement()
+ .satisfies(
+ suppressed ->
+ assertThat(suppressed)
+ .isInstanceOf(IOException.class)
+ .hasMessage("higher-index accepted publish failed"));
+ assertThat(publishExecutor.submissionCalls()).isEqualTo(3);
+ assertThat(discards.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(activePublishes).hasValue(0);
+ assertThat(first.cleanCalls()).isZero();
+ assertThat(rejected.cleanCalls()).isZero();
+ assertThat(higherIndexFailure.cleanCalls()).isZero();
+ assertThat(pending.cleanCalls()).isZero();
+ verify(partitionManager, never())
+ .createPartitions(anyList(), eq(true), any(), anyBoolean());
+ } finally {
+ releaseFirstPublish.countDown();
+ releaseHigherIndexFailure.countDown();
+ caller.shutdownNow();
+ publishExecutor.shutdownNow();
+ }
+
+ assertHigherIndexRejectionIsSuppressedByLowerIndexWorkerFailure(
+ new Path(tablePath, "symmetric-ordering"));
+ }
+
+ private void assertHigherIndexRejectionIsSuppressedByLowerIndexWorkerFailure(Path tablePath)
+ throws Exception {
+ FileIO fileIO = LocalFileIO.create();
+ CountDownLatch acceptedPublishesStarted = new CountDownLatch(2);
+ CountDownLatch releaseLowerIndexFailure = new CountDownLatch(1);
+ CountDownLatch releaseSuccessfulPublish = new CountDownLatch(1);
+ CountDownLatch lowerIndexFailureReturned = new CountDownLatch(1);
+ CountDownLatch successfulPublishReturned = new CountDownLatch(1);
+ CountDownLatch discards = new CountDownLatch(3);
+ AtomicInteger activePublishes = new AtomicInteger();
+ PublishAction discard =
+ () -> {
+ if (activePublishes.get() != 0) {
+ throw new IOException(
+ "Abort overlapped accepted publish before symmetric rejection drain");
+ }
+ discards.countDown();
+ };
+ ProbeCommitter lowerIndexFailure =
+ new ProbeCommitter(
+ new Path(tablePath, "part=p0/data-0.csv"),
+ () -> {
+ activePublishes.incrementAndGet();
+ acceptedPublishesStarted.countDown();
+ try {
+ awaitPublishLatch(
+ acceptedPublishesStarted,
+ "both symmetric accepted publishes to start");
+ awaitPublishLatch(
+ releaseLowerIndexFailure,
+ "lower-index accepted publish failure release");
+ throw new IOException("lower-index accepted publish failed");
+ } finally {
+ activePublishes.decrementAndGet();
+ lowerIndexFailureReturned.countDown();
+ }
+ },
+ PublishAction.NOOP,
+ discard);
+ ProbeCommitter successful =
+ new ProbeCommitter(
+ new Path(tablePath, "part=p1/data-0.csv"),
+ () -> {
+ activePublishes.incrementAndGet();
+ acceptedPublishesStarted.countDown();
+ try {
+ awaitPublishLatch(
+ acceptedPublishesStarted,
+ "both symmetric accepted publishes to start");
+ awaitPublishLatch(
+ releaseSuccessfulPublish,
+ "successful publish before higher-index rejection");
+ } finally {
+ activePublishes.decrementAndGet();
+ successfulPublishReturned.countDown();
+ }
+ },
+ PublishAction.NOOP,
+ discard);
+ ProbeCommitter rejected =
+ new ProbeCommitter(
+ new Path(tablePath, "part=p1/data-1.csv"),
+ () -> {
+ throw new IOException("Higher-index rejected publish executed");
+ },
+ PublishAction.NOOP,
+ discard);
+ List messages =
+ Arrays.asList(
+ new TwoPhaseCommitMessage(lowerIndexFailure),
+ new TwoPhaseCommitMessage(successful),
+ new TwoPhaseCommitMessage(rejected));
+ RejectThirdSubmissionExecutor publishExecutor =
+ new RejectThirdSubmissionExecutor(Executors.newFixedThreadPool(2));
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ FormatTableCommit commit =
+ newPublishCommit(
+ tablePath, fileIO, partitionManager, false, null, 1, 2, publishExecutor);
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ try {
+ Future result = caller.submit(() -> commit.commit(messages));
+
+ assertThat(acceptedPublishesStarted.await(10, TimeUnit.SECONDS)).isTrue();
+ releaseSuccessfulPublish.countDown();
+ assertThat(successfulPublishReturned.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(publishExecutor.awaitRejection()).isTrue();
+ assertThat(result.isDone()).isFalse();
+ assertThat(discards.getCount()).isEqualTo(3);
+ assertThat(rejected.commitCalls()).isZero();
+
+ releaseLowerIndexFailure.countDown();
+ assertThat(lowerIndexFailureReturned.await(10, TimeUnit.SECONDS)).isTrue();
+ ExecutionException failure = awaitFailure(result);
+
+ Throwable primary = rootCause(failure);
+ assertThat(primary)
+ .isInstanceOf(IOException.class)
+ .hasMessage("lower-index accepted publish failed");
+ assertThat(primary.getSuppressed())
+ .singleElement()
+ .satisfies(
+ suppressed ->
+ assertThat(suppressed)
+ .isInstanceOf(RejectedExecutionException.class)
+ .hasMessage("publish submission rejected"));
+ assertThat(publishExecutor.submissionCalls()).isEqualTo(3);
+ assertThat(discards.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(activePublishes).hasValue(0);
+ assertThat(lowerIndexFailure.cleanCalls()).isZero();
+ assertThat(successful.cleanCalls()).isZero();
+ assertThat(rejected.cleanCalls()).isZero();
+ verify(partitionManager, never())
+ .createPartitions(anyList(), eq(true), any(), anyBoolean());
+ } finally {
+ releaseLowerIndexFailure.countDown();
+ releaseSuccessfulPublish.countDown();
+ caller.shutdownNow();
+ publishExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testSingleTargetPartitionUsesOrderedCallerFastPath() {
+ FileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Path partitionPath = new Path(tablePath, "part=p0");
+ Thread caller = Thread.currentThread();
+ List order = new ArrayList<>();
+ ConcurrentLinkedQueue publishingThreads = new ConcurrentLinkedQueue<>();
+ List messages = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ int index = i;
+ messages.add(
+ new TwoPhaseCommitMessage(
+ new ProbeCommitter(
+ new Path(partitionPath, "data-" + index + ".csv"),
+ () -> {
+ order.add("file-" + index);
+ publishingThreads.add(Thread.currentThread());
+ })));
+ }
+ RejectAllExecutor publishExecutor = new RejectAllExecutor();
+ FormatTableCommit commit =
+ newPublishCommit(
+ tablePath,
+ fileIO,
+ mock(FormatTablePartitionManager.class),
+ false,
+ null,
+ 1,
+ 64,
+ publishExecutor);
+ try {
+ commit.commit(messages);
+
+ assertThat(order).containsExactly("file-0", "file-1", "file-2");
+ assertThat(publishingThreads).hasSize(3).containsOnly(caller);
+ assertThat(publishExecutor.submissionCalls()).isZero();
+ } finally {
+ publishExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testDirectPublishConcurrencyRejectsValuesOutsideSupportedRange() {
+ FileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ ExecutorService publishExecutor = Executors.newSingleThreadExecutor();
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ try {
+ assertThatThrownBy(
+ () ->
+ newPublishCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ false,
+ null,
+ 1,
+ -1,
+ publishExecutor))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Format Table publish thread number must be between 1 and 64, but was -1.");
+ assertThatThrownBy(
+ () ->
+ newPublishCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ false,
+ null,
+ 1,
+ 0,
+ publishExecutor))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Format Table publish thread number must be between 1 and 64, but was 0.");
+ assertThatThrownBy(
+ () ->
+ newPublishCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ false,
+ null,
+ 1,
+ 65,
+ publishExecutor))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Format Table publish thread number must be between 1 and 64, but was 65.");
+ } finally {
+ publishExecutor.shutdownNow();
+ }
+ }
+
+ private FormatTableCommit builderAppendCommit(
+ Path tablePath,
+ FileIO fileIO,
+ FormatTablePartitionManager partitionManager,
+ Map options) {
+ return (FormatTableCommit)
+ formatTable(tablePath, fileIO, partitionManager, options)
+ .newBatchWriteBuilder()
+ .newCommit();
+ }
+
+ private FormatTable formatTable(
+ Path tablePath,
+ FileIO fileIO,
+ FormatTablePartitionManager partitionManager,
+ Map options) {
+ RowType rowType =
+ RowType.builder()
+ .field("part", DataTypes.STRING())
+ .field("id", DataTypes.INT())
+ .build();
+ FormatTable table =
+ FormatTable.builder()
+ .fileIO(fileIO)
+ .identifier(Identifier.create("cleanup_db", "cleanup_table"))
+ .rowType(rowType)
+ .partitionKeys(Collections.singletonList("part"))
+ .location(tablePath.toString())
+ .format(FormatTable.Format.CSV)
+ .options(options)
+ .partitionManager(partitionManager)
+ .build();
+ return table;
+ }
+
+ private FormatTableCommit builderUnpartitionedAppendCommit(
+ Path tablePath,
+ FileIO fileIO,
+ FormatTablePartitionManager partitionManager,
+ Map options) {
+ FormatTable table =
+ FormatTable.builder()
+ .fileIO(fileIO)
+ .identifier(Identifier.create("publish_db", "unpartitioned_publish_table"))
+ .rowType(RowType.builder().field("id", DataTypes.INT()).build())
+ .partitionKeys(Collections.emptyList())
+ .location(tablePath.toString())
+ .format(FormatTable.Format.CSV)
+ .options(options)
+ .partitionManager(partitionManager)
+ .build();
+ return (FormatTableCommit) table.newBatchWriteBuilder().newCommit();
+ }
+
+ private FormatTableCommit newPublishCommit(
+ Path tablePath,
+ FileIO fileIO,
+ FormatTablePartitionManager partitionManager,
+ boolean overwrite,
+ Map staticPartition,
+ int cleanupThreadNum,
+ int publishThreadNum,
+ ExecutorService publishExecutor) {
+ return new FormatTableCommit(
+ tablePath.toString(),
+ Collections.singletonList("part"),
+ fileIO,
+ false,
+ PARTITION_DEFAULT_NAME.defaultValue(),
+ overwrite,
+ Identifier.create("publish_db", "publish_table"),
+ staticPartition,
+ null,
+ null,
+ partitionManager,
+ /* dynamicPartitionOverwrite */ true,
+ cleanupThreadNum,
+ publishThreadNum,
+ publishExecutor);
+ }
+
+ private static ProbeCommitter blockingPublishCommitter(
+ Path targetPath,
+ CountDownLatch acceptedPublishesStarted,
+ CountDownLatch releasePublish,
+ CountDownLatch publishReturned,
+ AtomicInteger activePublishes,
+ PublishAction discard) {
+ return new ProbeCommitter(
+ targetPath,
+ () -> {
+ activePublishes.incrementAndGet();
+ acceptedPublishesStarted.countDown();
+ try {
+ awaitPublishLatch(
+ acceptedPublishesStarted, "all accepted publishes to start");
+ awaitPublishLatch(releasePublish, "accepted publish release");
+ } finally {
+ activePublishes.decrementAndGet();
+ publishReturned.countDown();
+ }
+ },
+ PublishAction.NOOP,
+ discard);
+ }
+
+ private static Thread commitCaller(
+ String name,
+ FormatTableCommit commit,
+ List messages,
+ AtomicReference failure,
+ CountDownLatch returned) {
+ return new Thread(
+ () -> {
+ try {
+ commit.commit(messages);
+ } catch (Throwable t) {
+ failure.set(t);
+ } finally {
+ returned.countDown();
+ }
+ },
+ name);
+ }
+
+ private static List publishMessages(
+ Path tablePath, ConcurrentLinkedQueue publishingThreads, boolean partitioned) {
+ List messages = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ Path parent = partitioned ? new Path(tablePath, "part=p" + i) : tablePath;
+ messages.add(
+ new TwoPhaseCommitMessage(
+ new ProbeCommitter(
+ new Path(parent, "data-" + i + ".csv"),
+ () -> publishingThreads.add(Thread.currentThread()))));
+ }
+ return messages;
+ }
+
+ private static void awaitPublishLatch(CountDownLatch latch, String description)
+ throws IOException {
+ try {
+ if (!latch.await(10, TimeUnit.SECONDS)) {
+ throw new IOException("Timed out waiting for " + description);
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while waiting for " + description, e);
+ }
+ }
+
+ private static void setContextClassLoaderOnWorkers(
+ ExecutorService executor, int workerCount, ClassLoader classLoader) throws Exception {
+ CountDownLatch workersStarted = new CountDownLatch(workerCount);
+ CountDownLatch releaseWorkers = new CountDownLatch(1);
+ List> futures = new ArrayList<>();
+ for (int i = 0; i < workerCount; i++) {
+ futures.add(
+ executor.submit(
+ () -> {
+ Thread.currentThread().setContextClassLoader(classLoader);
+ workersStarted.countDown();
+ try {
+ if (!releaseWorkers.await(10, TimeUnit.SECONDS)) {
+ throw new AssertionError(
+ "Timed out initializing publish executor workers");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(
+ "Interrupted while initializing publish executor workers",
+ e);
+ }
+ }));
+ }
+ try {
+ assertThat(workersStarted.await(10, TimeUnit.SECONDS)).isTrue();
+ } finally {
+ releaseWorkers.countDown();
+ }
+ for (Future future : futures) {
+ future.get(10, TimeUnit.SECONDS);
+ }
+ }
+
+ private static final class PublishExecutorTracker extends AbstractExecutorService {
+
+ private final ExecutorService delegate;
+ private final ThreadLocal selectedTask =
+ ThreadLocal.withInitial(() -> Boolean.FALSE);
+ private final CountDownLatch selectedTaskCompletion = new CountDownLatch(1);
+ private final AtomicReference trackedFileIO =
+ new AtomicReference<>();
+ private final AtomicReference expectedInitializationThread =
+ new AtomicReference<>();
+ private final AtomicInteger fileIoAcceptanceChecks = new AtomicInteger();
+
+ private PublishExecutorTracker(ExecutorService delegate) {
+ this.delegate = delegate;
+ }
+
+ private void markCurrentTaskForCompletion() {
+ selectedTask.set(Boolean.TRUE);
+ }
+
+ private boolean awaitSelectedTaskCompletion() throws InterruptedException {
+ return selectedTaskCompletion.await(10, TimeUnit.SECONDS);
+ }
+
+ private void armFileIoInitializationCheck(
+ FirstAccessTrackingFileIO fileIO, Thread expectedThread) {
+ if (!trackedFileIO.compareAndSet(null, fileIO)
+ || !expectedInitializationThread.compareAndSet(null, expectedThread)) {
+ throw new IllegalStateException("FileIO initialization tracking is already armed.");
+ }
+ }
+
+ private void disarmFileIoInitializationCheck() {
+ trackedFileIO.set(null);
+ expectedInitializationThread.set(null);
+ }
+
+ private int fileIoAcceptanceChecks() {
+ return fileIoAcceptanceChecks.get();
+ }
+
+ @Override
+ public void shutdown() {
+ delegate.shutdown();
+ }
+
+ @Override
+ public List shutdownNow() {
+ return delegate.shutdownNow();
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return delegate.isShutdown();
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return delegate.isTerminated();
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ return delegate.awaitTermination(timeout, unit);
+ }
+
+ @Override
+ public void execute(Runnable command) {
+ Thread expectedThread = expectedInitializationThread.get();
+ if (expectedThread != null) {
+ FirstAccessTrackingFileIO fileIO = trackedFileIO.get();
+ if (fileIO == null || fileIO.firstAccessThread() != expectedThread) {
+ throw new AssertionError(
+ "Publish task accepted before FileIO initialization on the caller");
+ }
+ }
+ delegate.execute(
+ () -> {
+ try {
+ command.run();
+ } finally {
+ if (selectedTask.get()) {
+ selectedTaskCompletion.countDown();
+ }
+ selectedTask.remove();
+ }
+ });
+ if (expectedThread != null) {
+ fileIoAcceptanceChecks.incrementAndGet();
+ }
+ }
+ }
+
+ private static final class SubmitterTrackingExecutor extends AbstractExecutorService {
+
+ private final ExecutorService delegate;
+ private final AtomicReference trackedSubmitter = new AtomicReference<>();
+ private final CountDownLatch trackedSubmission = new CountDownLatch(1);
+
+ private SubmitterTrackingExecutor(ExecutorService delegate) {
+ this.delegate = delegate;
+ }
+
+ private void trackSubmissionsFrom(Thread submitter) {
+ if (!trackedSubmitter.compareAndSet(null, submitter)) {
+ throw new IllegalStateException("A publish submitter is already being tracked.");
+ }
+ }
+
+ private boolean awaitTrackedSubmission() throws InterruptedException {
+ return trackedSubmission.await(10, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void shutdown() {
+ delegate.shutdown();
+ }
+
+ @Override
+ public List shutdownNow() {
+ return delegate.shutdownNow();
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return delegate.isShutdown();
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return delegate.isTerminated();
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ return delegate.awaitTermination(timeout, unit);
+ }
+
+ @Override
+ public void execute(Runnable command) {
+ delegate.execute(command);
+ if (Thread.currentThread() == trackedSubmitter.get()) {
+ trackedSubmission.countDown();
+ }
+ }
+ }
+
+ private static final class RejectThirdSubmissionExecutor extends AbstractExecutorService {
+
+ private final ExecutorService delegate;
+ private final AtomicInteger submissionCalls = new AtomicInteger();
+ private final CountDownLatch rejection = new CountDownLatch(1);
+
+ private RejectThirdSubmissionExecutor(ExecutorService delegate) {
+ this.delegate = delegate;
+ }
+
+ private boolean awaitRejection() throws InterruptedException {
+ return rejection.await(10, TimeUnit.SECONDS);
+ }
+
+ private int submissionCalls() {
+ return submissionCalls.get();
+ }
+
+ @Override
+ public void shutdown() {
+ delegate.shutdown();
+ }
+
+ @Override
+ public List shutdownNow() {
+ return delegate.shutdownNow();
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return delegate.isShutdown();
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return delegate.isTerminated();
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ return delegate.awaitTermination(timeout, unit);
+ }
+
+ @Override
+ public void execute(Runnable command) {
+ int call = submissionCalls.incrementAndGet();
+ if (call >= 3) {
+ rejection.countDown();
+ throw new RejectedExecutionException("publish submission rejected");
+ }
+ delegate.execute(command);
+ }
+ }
+
+ private static final class RejectAllExecutor extends AbstractExecutorService {
+
+ private final AtomicBoolean shutdown = new AtomicBoolean();
+ private final AtomicInteger submissionCalls = new AtomicInteger();
+
+ private int submissionCalls() {
+ return submissionCalls.get();
+ }
+
+ @Override
+ public void shutdown() {
+ shutdown.set(true);
+ }
+
+ @Override
+ public List shutdownNow() {
+ shutdown.set(true);
+ return Collections.emptyList();
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return shutdown.get();
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return shutdown.get();
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) {
+ return shutdown.get();
+ }
+
+ @Override
+ public void execute(Runnable command) {
+ submissionCalls.incrementAndGet();
+ throw new AssertionError("Single-target publish submitted to the worker executor");
+ }
+ }
+
+ @FunctionalInterface
+ private interface PublishAction {
+
+ PublishAction NOOP = () -> {};
+
+ void run() throws IOException;
+ }
+
+ private static class ProbeCommitter implements TwoPhaseOutputStream.Committer {
+
+ private static final long serialVersionUID = 1L;
+
+ private final Path targetPath;
+ private final PublishAction publish;
+ private final PublishAction clean;
+ private final PublishAction discard;
+ private final AtomicInteger commitCalls = new AtomicInteger();
+ private final AtomicInteger cleanCalls = new AtomicInteger();
+ private final AtomicInteger discardCalls = new AtomicInteger();
+
+ private ProbeCommitter(Path targetPath, PublishAction publish) {
+ this(targetPath, publish, PublishAction.NOOP, PublishAction.NOOP);
+ }
+
+ private ProbeCommitter(
+ Path targetPath,
+ PublishAction publish,
+ PublishAction clean,
+ PublishAction discard) {
+ this.targetPath = targetPath;
+ this.publish = publish;
+ this.clean = clean;
+ this.discard = discard;
+ }
+
+ @Override
+ public void commit(FileIO fileIO) throws IOException {
+ commitCalls.incrementAndGet();
+ publish.run();
+ }
+
+ @Override
+ public void discard(FileIO fileIO) throws IOException {
+ discardCalls.incrementAndGet();
+ discard.run();
+ }
+
+ @Override
+ public Path targetPath() {
+ return targetPath;
+ }
+
+ @Override
+ public void clean(FileIO fileIO) throws IOException {
+ cleanCalls.incrementAndGet();
+ clean.run();
+ }
+
+ private int commitCalls() {
+ return commitCalls.get();
+ }
+
+ private int cleanCalls() {
+ return cleanCalls.get();
+ }
+
+ private int discardCalls() {
+ return discardCalls.get();
+ }
+ }
+
+ private static final class PendingInterruptThread extends Thread {
+
+ private final AtomicBoolean pendingInterrupt = new AtomicBoolean();
+
+ private PendingInterruptThread(Runnable target, String name) {
+ super(target, name);
+ }
+
+ private void signalPendingInterrupt() {
+ pendingInterrupt.set(true);
+ }
+
+ @Override
+ public boolean isInterrupted() {
+ return pendingInterrupt.getAndSet(false) || super.isInterrupted();
+ }
+ }
+
+ private static class TrackingTwoPhaseCommitMessage extends TwoPhaseCommitMessage {
+
+ private static final long serialVersionUID = 1L;
+
+ private final AtomicInteger recordCountCalls = new AtomicInteger();
+ private final AtomicInteger fileSizeCalls = new AtomicInteger();
+ private final ConcurrentLinkedQueue statisticsAccessThreads =
+ new ConcurrentLinkedQueue<>();
+
+ private TrackingTwoPhaseCommitMessage(
+ TwoPhaseOutputStream.Committer committer, long recordCount, long fileSizeInBytes) {
+ super(committer, recordCount, fileSizeInBytes);
+ }
+
+ @Override
+ public long recordCount() {
+ recordCountCalls.incrementAndGet();
+ statisticsAccessThreads.add(Thread.currentThread());
+ return super.recordCount();
+ }
+
+ @Override
+ public long fileSizeInBytes() {
+ fileSizeCalls.incrementAndGet();
+ statisticsAccessThreads.add(Thread.currentThread());
+ return super.fileSizeInBytes();
+ }
+
+ private int recordCountCalls() {
+ return recordCountCalls.get();
+ }
+
+ private int fileSizeCalls() {
+ return fileSizeCalls.get();
+ }
+
+ private List statisticsAccessThreads() {
+ return new ArrayList<>(statisticsAccessThreads);
+ }
+ }
+
+ private static class ParallelPublishProbe {
+
+ private final int firstWaveSize;
+ private final CountDownLatch firstWave;
+ private final CountDownLatch releaseFirstWave = new CountDownLatch(1);
+ private final CountDownLatch unexpectedExtraPublish = new CountDownLatch(1);
+ private final AtomicInteger publishCalls = new AtomicInteger();
+ private final AtomicInteger activePublishes = new AtomicInteger();
+ private final AtomicInteger maxConcurrentPublishes = new AtomicInteger();
+
+ private ParallelPublishProbe(int firstWaveSize) {
+ this.firstWaveSize = firstWaveSize;
+ this.firstWave = new CountDownLatch(firstWaveSize);
+ }
+
+ private void publish() throws IOException {
+ int call = publishCalls.incrementAndGet();
+ int active = activePublishes.incrementAndGet();
+ maxConcurrentPublishes.updateAndGet(previous -> Math.max(previous, active));
+ try {
+ if (call <= firstWaveSize) {
+ firstWave.countDown();
+ } else {
+ unexpectedExtraPublish.countDown();
+ }
+ awaitPublishLatch(releaseFirstWave, "publish first-wave release");
+ } finally {
+ activePublishes.decrementAndGet();
+ }
+ }
+
+ private boolean awaitFirstWave() throws InterruptedException {
+ return firstWave.await(10, TimeUnit.SECONDS);
+ }
+
+ private boolean awaitUnexpectedExtraPublish() throws InterruptedException {
+ return unexpectedExtraPublish.await(300, TimeUnit.MILLISECONDS);
+ }
+
+ private void releaseFirstWave() {
+ releaseFirstWave.countDown();
+ }
+
+ private int publishCalls() {
+ return publishCalls.get();
+ }
+
+ private int maxConcurrentPublishes() {
+ return maxConcurrentPublishes.get();
+ }
+ }
+
+ private static class FirstAccessTrackingFileIO extends LocalFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final AtomicReference firstAccessThread = new AtomicReference<>();
+
+ @Override
+ public FileStatus getFileStatus(Path path) throws IOException {
+ recordAccess();
+ return super.getFileStatus(path);
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ recordAccess();
+ return super.listStatus(path);
+ }
+
+ @Override
+ public boolean exists(Path path) throws IOException {
+ recordAccess();
+ return super.exists(path);
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive) throws IOException {
+ recordAccess();
+ return super.delete(path, recursive);
+ }
+
+ @Override
+ public boolean mkdirs(Path path) throws IOException {
+ recordAccess();
+ return super.mkdirs(path);
+ }
+
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ recordAccess();
+ return super.rename(src, dst);
+ }
+
+ private void recordAccess() {
+ firstAccessThread.compareAndSet(null, Thread.currentThread());
+ }
+
+ private Thread firstAccessThread() {
+ return firstAccessThread.get();
+ }
+ }
+
+ private abstract static class SortedLocalFileIO extends LocalFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ FileStatus[] statuses = super.listStatus(path);
+ Arrays.sort(statuses, Comparator.comparing(status -> status.getPath().toString()));
+ return statuses;
+ }
+
+ protected static void await(CountDownLatch latch, String description) throws IOException {
+ try {
+ if (!latch.await(10, TimeUnit.SECONDS)) {
+ throw new IOException("Timed out waiting for " + description);
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while waiting for " + description, e);
+ }
+ }
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java
index 75b141d947f2..9e426488a775 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java
@@ -18,27 +18,56 @@
package org.apache.paimon.table.format;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.FileStatus;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.RenamingTwoPhaseOutputStream;
import org.apache.paimon.fs.TwoPhaseOutputStream;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.partition.Partition;
+import org.apache.paimon.partition.PartitionStatistics;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.format.FormatTableCommitTestUtils.PartialBarrierDeleteFileIO;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.PartitionPathUtils;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.ArgumentCaptor;
+import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.AbstractExecutorService;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import static org.apache.paimon.CoreOptions.PARTITION_DEFAULT_NAME;
+import static org.apache.paimon.table.format.FormatTableCommitTestUtils.awaitFailure;
+import static org.apache.paimon.table.format.FormatTableCommitTestUtils.failureTree;
+import static org.apache.paimon.table.format.FormatTableCommitTestUtils.observeContextClassLoaders;
+import static org.apache.paimon.table.format.FormatTableCommitTestUtils.rootCause;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.assertj.core.api.Assertions.entry;
@@ -46,6 +75,7 @@
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -632,6 +662,836 @@ void testOverwritingTheWholeTableLeavesADirectoryThatIsNoPartitionOfIt() throws
assertThat(fileIO.exists(new Path(tablePath, "loose.csv"))).isTrue();
}
+ @Test
+ void testCatalogManagedBuilderUses64WayCleanupByDefault() throws Exception {
+ ParallelDeleteFileIO fileIO = new ParallelDeleteFileIO(64, true);
+ Path tablePath = new Path(tempDir.toUri());
+ Path partitionPath = new Path(tablePath, "part=p");
+ writeOldFiles(fileIO, partitionPath, 65);
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class);
+ when(committer.targetPath()).thenReturn(new Path(partitionPath, "data-new.csv"));
+ doAnswer(
+ invocation -> {
+ assertThat(fileIO.activeDeletes()).isZero();
+ return null;
+ })
+ .when(committer)
+ .commit(fileIO);
+ FormatTableCommit commit =
+ builderOverwriteCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Collections.emptyMap(),
+ Collections.singletonMap("part", "p"));
+
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ Future result =
+ executor.submit(
+ () ->
+ commit.commit(
+ Collections.singletonList(
+ new TwoPhaseCommitMessage(committer))));
+
+ assertThat(fileIO.awaitFirstWave()).isTrue();
+ assertThat(fileIO.deleteCalls()).isEqualTo(64);
+ assertThat(fileIO.awaitUnexpectedExtraDelete()).isFalse();
+ verify(committer, never()).commit(fileIO);
+
+ fileIO.releaseFirstWave();
+ result.get(10, TimeUnit.SECONDS);
+ assertThat(fileIO.deleteCalls()).isEqualTo(65);
+ assertThat(fileIO.maxConcurrentDeletes()).isEqualTo(64);
+ verify(committer).commit(fileIO);
+ } finally {
+ fileIO.releaseFirstWave();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testCatalogManagedBuilderHonorsConfiguredSerialCleanup() throws Exception {
+ SerialProbeFileIO fileIO = new SerialProbeFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ Path partitionPath = new Path(tablePath, "part=p");
+ writeOldFiles(fileIO, partitionPath, 3);
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class);
+ when(committer.targetPath()).thenReturn(new Path(partitionPath, "data-new.csv"));
+ FormatTableCommit commit =
+ builderOverwriteCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Collections.singletonMap(
+ CoreOptions.FORMAT_TABLE_COMMIT_CLEANUP_THREAD_NUM.key(), "1"),
+ Collections.singletonMap("part", "p"));
+
+ commit.commit(Collections.singletonList(new TwoPhaseCommitMessage(committer)));
+
+ assertThat(fileIO.deleteCalls()).isEqualTo(3);
+ assertThat(fileIO.maxConcurrentDeletes()).isEqualTo(1);
+ }
+
+ @Test
+ void testCatalogManagedBuilderPropagatesConfiguredCleanupConcurrency() throws Exception {
+ ParallelDeleteFileIO fileIO = new ParallelDeleteFileIO(7, true);
+ Path tablePath = new Path(tempDir.toUri());
+ Path partitionPath = new Path(tablePath, "part=p");
+ writeOldFiles(fileIO, partitionPath, 8);
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ FormatTableCommit commit =
+ builderOverwriteCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Collections.singletonMap(
+ CoreOptions.FORMAT_TABLE_COMMIT_CLEANUP_THREAD_NUM.key(), "7"),
+ Collections.singletonMap("part", "p"));
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ Future result = executor.submit(() -> commit.commit(Collections.emptyList()));
+
+ assertThat(fileIO.awaitFirstWave()).isTrue();
+ assertThat(fileIO.deleteCalls()).isEqualTo(7);
+ assertThat(fileIO.awaitUnexpectedExtraDelete()).isFalse();
+
+ fileIO.releaseFirstWave();
+ result.get(10, TimeUnit.SECONDS);
+ assertThat(fileIO.deleteCalls()).isEqualTo(8);
+ assertThat(fileIO.maxConcurrentDeletes()).isEqualTo(7);
+ } finally {
+ fileIO.releaseFirstWave();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testFilesystemDiscoveredFormatTableCleanupRemainsSerial() throws Exception {
+ SerialProbeFileIO fileIO = new SerialProbeFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ Path partitionPath = new Path(tablePath, "part=p");
+ writeOldFiles(fileIO, partitionPath, 3);
+ TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class);
+ when(committer.targetPath()).thenReturn(new Path(partitionPath, "data-new.csv"));
+ FormatTableCommit commit =
+ builderOverwriteCommit(
+ tablePath,
+ fileIO,
+ null,
+ Collections.singletonMap(
+ CoreOptions.FORMAT_TABLE_COMMIT_CLEANUP_THREAD_NUM.key(), "64"),
+ Collections.singletonMap("part", "p"));
+
+ commit.commit(Collections.singletonList(new TwoPhaseCommitMessage(committer)));
+
+ assertThat(fileIO.deleteCalls()).isEqualTo(3);
+ assertThat(fileIO.maxConcurrentDeletes()).isEqualTo(1);
+ }
+
+ @Test
+ void testCleanupIsAHardBarrierBeforePublishingNewFiles() throws Exception {
+ PartialBarrierDeleteFileIO fileIO = new PartialBarrierDeleteFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ Path partitionPath = new Path(tablePath, "part=p");
+ writeOldFiles(fileIO, partitionPath, 2);
+ TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class);
+ when(committer.targetPath()).thenReturn(new Path(partitionPath, "data-new.csv"));
+ doAnswer(
+ invocation -> {
+ assertThat(fileIO.activeDeletes()).isZero();
+ return null;
+ })
+ .when(committer)
+ .commit(fileIO);
+ FormatTableCommit commit =
+ newCleanupCommit(tablePath, fileIO, null, Collections.singletonMap("part", "p"), 2);
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ Future result =
+ executor.submit(
+ () ->
+ commit.commit(
+ Collections.singletonList(
+ new TwoPhaseCommitMessage(committer))));
+
+ assertThat(fileIO.awaitBothDeletesStarted()).isTrue();
+ verify(committer, never()).commit(fileIO);
+
+ fileIO.releaseFirstDelete();
+ assertThat(fileIO.awaitFirstDeleteReturned()).isTrue();
+ assertThatThrownBy(() -> result.get(300, TimeUnit.MILLISECONDS))
+ .isInstanceOf(TimeoutException.class);
+ verify(committer, never()).commit(fileIO);
+
+ fileIO.releaseSecondDelete();
+ result.get(10, TimeUnit.SECONDS);
+ verify(committer).commit(fileIO);
+ } finally {
+ fileIO.releaseFirstDelete();
+ fileIO.releaseSecondDelete();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testUnpartitionedCatalogManagedFormatTableCleanupRemainsSerial() throws Exception {
+ SerialProbeFileIO fileIO = new SerialProbeFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ writeOldFiles(fileIO, tablePath, 3);
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class);
+ when(committer.targetPath()).thenReturn(new Path(tablePath, "data-new.csv"));
+ FormatTableCommit commit =
+ builderUnpartitionedOverwriteCommit(
+ tablePath,
+ fileIO,
+ partitionManager,
+ Collections.singletonMap(
+ CoreOptions.FORMAT_TABLE_COMMIT_CLEANUP_THREAD_NUM.key(), "64"));
+
+ commit.commit(Collections.singletonList(new TwoPhaseCommitMessage(committer)));
+
+ assertThat(fileIO.deleteCalls()).isEqualTo(3);
+ assertThat(fileIO.maxConcurrentDeletes()).isEqualTo(1);
+ }
+
+ @Test
+ void testLegacyPublicConstructorKeepsCleanupSerial() throws Exception {
+ SerialProbeFileIO fileIO = new SerialProbeFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ Path partitionPath = new Path(tablePath, "part=p");
+ writeOldFiles(fileIO, partitionPath, 3);
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ FormatTableCommit commit =
+ new FormatTableCommit(
+ tablePath.toString(),
+ Collections.singletonList("part"),
+ fileIO,
+ false,
+ PARTITION_DEFAULT_NAME.defaultValue(),
+ true,
+ Identifier.create("cleanup_db", "cleanup_table"),
+ Collections.singletonMap("part", "p"),
+ null,
+ null,
+ partitionManager,
+ /* dynamicPartitionOverwrite */ true);
+
+ commit.commit(Collections.emptyList());
+
+ assertThat(fileIO.deleteCalls()).isEqualTo(3);
+ assertThat(fileIO.maxConcurrentDeletes()).isEqualTo(1);
+ }
+
+ @Test
+ void testCleanupFailureStopsNewSubmissionsAndDrainsTheAlreadyRunningDelete() throws Exception {
+ FailureDrainFileIO fileIO = new FailureDrainFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ Path partitionPath = new Path(tablePath, "part=p");
+ writeOldFiles(fileIO, partitionPath, 6);
+ TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class);
+ when(committer.targetPath()).thenReturn(new Path(partitionPath, "data-new.csv"));
+ FormatTableCommit commit =
+ newCleanupCommit(tablePath, fileIO, null, Collections.singletonMap("part", "p"), 2);
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ Future result =
+ executor.submit(
+ () ->
+ commit.commit(
+ Collections.singletonList(
+ new TwoPhaseCommitMessage(committer))));
+
+ assertThat(fileIO.awaitFailureAttempted()).isTrue();
+ assertThatThrownBy(() -> result.get(300, TimeUnit.MILLISECONDS))
+ .isInstanceOf(TimeoutException.class);
+ assertThat(fileIO.attemptedFiles())
+ .containsExactlyInAnyOrder("data-000.csv", "data-001.csv");
+
+ fileIO.releaseSuccessfulSibling();
+ assertThat(rootCause(awaitFailure(result)))
+ .hasMessage("delete failed at input position 0");
+ assertThat(fileIO.attemptedFiles())
+ .containsExactlyInAnyOrder("data-000.csv", "data-001.csv");
+ assertThat(fileIO.successfulFiles()).containsExactly("data-001.csv");
+ verify(committer, never()).commit(fileIO);
+ } finally {
+ fileIO.releaseSuccessfulSibling();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testLaterPartitionListingFailureDrainsAcceptedDeletesBeforeAbort() throws Exception {
+ Path tablePath = new Path(new Path(tempDir.toUri()), "listing-failure");
+ Path firstPartition = new Path(tablePath, "part=p0");
+ Path failingPartition = new Path(tablePath, "part=p1");
+ LaterRootListingFailureFileIO fileIO =
+ new LaterRootListingFailureFileIO(failingPartition, 2);
+ writeOldFiles(fileIO, firstPartition, 2);
+ writeOldFiles(fileIO, failingPartition, 1);
+
+ AtomicInteger discardCalls = new AtomicInteger();
+ ConcurrentLinkedQueue activeDeletesAtDiscard = new ConcurrentLinkedQueue<>();
+ List messages = new ArrayList<>();
+ List committers = new ArrayList<>();
+ for (Path partition : Arrays.asList(firstPartition, failingPartition)) {
+ TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class);
+ when(committer.targetPath()).thenReturn(new Path(partition, "data-new.csv"));
+ doAnswer(
+ invocation -> {
+ activeDeletesAtDiscard.add(fileIO.activeDeletes());
+ discardCalls.incrementAndGet();
+ return null;
+ })
+ .when(committer)
+ .discard(fileIO);
+ committers.add(committer);
+ messages.add(new TwoPhaseCommitMessage(committer));
+ }
+ FormatTableCommit commit = newCleanupCommit(tablePath, fileIO, null, null, 3);
+
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ Future result = executor.submit(() -> commit.commit(messages));
+ try {
+ assertThat(fileIO.awaitListingFailure()).isTrue();
+ assertThatThrownBy(() -> result.get(300, TimeUnit.MILLISECONDS))
+ .isInstanceOf(TimeoutException.class);
+ assertThat(discardCalls).hasValue(0);
+ for (TwoPhaseOutputStream.Committer committer : committers) {
+ verify(committer, never()).commit(fileIO);
+ verify(committer, never()).discard(fileIO);
+ }
+
+ fileIO.releaseFirstDelete();
+ assertThat(fileIO.awaitFirstDeleteReturned()).isTrue();
+ assertThatThrownBy(() -> result.get(300, TimeUnit.MILLISECONDS))
+ .isInstanceOf(TimeoutException.class);
+ assertThat(discardCalls).hasValue(0);
+
+ fileIO.releaseSecondDelete();
+ assertThat(rootCause(awaitFailure(result)))
+ .hasMessage("Failed to list the later partition root.");
+ assertThat(discardCalls).hasValue(2);
+ assertThat(activeDeletesAtDiscard).containsExactly(0, 0);
+ for (TwoPhaseOutputStream.Committer committer : committers) {
+ verify(committer, never()).commit(fileIO);
+ verify(committer).discard(fileIO);
+ }
+ } finally {
+ fileIO.releaseFirstDelete();
+ fileIO.releaseSecondDelete();
+ try {
+ if (!result.isDone()) {
+ try {
+ result.get(10, TimeUnit.SECONDS);
+ } catch (ExecutionException ignored) {
+ // The test expects the listing failure above.
+ }
+ }
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+ }
+
+ @Test
+ void testCleanupFailureAndReplacementSubmissionAreLinearized() throws Exception {
+ FormatTableCommit.CleanupSubmissionState state =
+ new FormatTableCommit.CleanupSubmissionState();
+ CompletableFuture submissionEntered = new CompletableFuture<>();
+ CompletableFuture releaseSubmission = new CompletableFuture<>();
+ ExecutorService executor = Executors.newFixedThreadPool(3);
+ try {
+ Future replacement =
+ executor.submit(
+ () ->
+ state.submitIfRunning(
+ () -> {
+ submissionEntered.complete(null);
+ releaseSubmission.join();
+ }));
+ submissionEntered.get(10, TimeUnit.SECONDS);
+ Future stop = executor.submit(state::stop);
+ long stopPublishDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10);
+ boolean stopped;
+ do {
+ stopped = state.isStopped();
+ if (!stopped) {
+ Thread.yield();
+ }
+ } while (!stopped && System.nanoTime() < stopPublishDeadline);
+ assertThat(stopped).isTrue();
+ assertThat(stop.isDone()).isFalse();
+ AtomicBoolean submittedAfterFailure = new AtomicBoolean();
+ Future rejected =
+ executor.submit(
+ () -> state.submitIfRunning(() -> submittedAfterFailure.set(true)));
+ releaseSubmission.complete(null);
+ assertThat(replacement.get(10, TimeUnit.SECONDS)).isTrue();
+ stop.get(10, TimeUnit.SECONDS);
+ assertThat(rejected.get(10, TimeUnit.SECONDS)).isFalse();
+ assertThat(submittedAfterFailure).isFalse();
+ } finally {
+ releaseSubmission.complete(null);
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testCleanupSelectsLowestInputFailureAndSuppressesTheOtherFailure() throws Exception {
+ OrderedDualFailureFileIO fileIO = new OrderedDualFailureFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ Path partitionPath = new Path(tablePath, "part=p");
+ writeOldFiles(fileIO, partitionPath, 2);
+ FormatTableCommit commit =
+ newCleanupCommit(tablePath, fileIO, null, Collections.singletonMap("part", "p"), 2);
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ Future result = executor.submit(() -> commit.commit(Collections.emptyList()));
+ assertThat(fileIO.awaitHigherPositionFailure()).isTrue();
+ assertThatThrownBy(() -> result.get(300, TimeUnit.MILLISECONDS))
+ .isInstanceOf(TimeoutException.class);
+
+ fileIO.releaseLowerPositionFailure();
+ Throwable primary = rootCause(awaitFailure(result));
+ assertThat(primary).hasMessage("delete failed at input position 0");
+ assertThat(primary.getSuppressed())
+ .extracting(Throwable::getMessage)
+ .containsExactly("delete failed at input position 1");
+ } finally {
+ fileIO.releaseLowerPositionFailure();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testInterruptDrainsCleanupRestoresFlagAndNeverPublishes() throws Exception {
+ BlockingDeleteFileIO fileIO = new BlockingDeleteFileIO(2);
+ Path tablePath = new Path(tempDir.toUri());
+ Path partitionPath = new Path(tablePath, "part=p");
+ writeOldFiles(fileIO, partitionPath, 2);
+ TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class);
+ when(committer.targetPath()).thenReturn(new Path(partitionPath, "data-new.csv"));
+ FormatTableCommit commit =
+ newCleanupCommit(tablePath, fileIO, null, Collections.singletonMap("part", "p"), 2);
+ AtomicReference failure = new AtomicReference<>();
+ AtomicBoolean interruptRestored = new AtomicBoolean();
+ CountDownLatch commitReturned = new CountDownLatch(1);
+ Thread commitThread =
+ new Thread(
+ () -> {
+ try {
+ commit.commit(
+ Collections.singletonList(
+ new TwoPhaseCommitMessage(committer)));
+ } catch (Throwable t) {
+ failure.set(t);
+ } finally {
+ interruptRestored.set(Thread.currentThread().isInterrupted());
+ commitReturned.countDown();
+ }
+ },
+ "format-cleanup-interrupted-caller");
+
+ commitThread.start();
+ try {
+ assertThat(fileIO.awaitDeletesStarted()).isTrue();
+ commitThread.interrupt();
+ assertThat(commitReturned.await(300, TimeUnit.MILLISECONDS)).isFalse();
+
+ fileIO.releaseDeletes();
+ assertThat(commitReturned.await(10, TimeUnit.SECONDS)).isTrue();
+ assertThat(failure.get()).isNotNull();
+ assertThat(causeChain(failure.get())).anyMatch(InterruptedException.class::isInstance);
+ assertThat(interruptRestored).isTrue();
+ verify(committer, never()).commit(fileIO);
+ } finally {
+ fileIO.releaseDeletes();
+ commitThread.interrupt();
+ commitThread.join(TimeUnit.SECONDS.toMillis(10));
+ }
+ assertThat(commitThread.isAlive()).isFalse();
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ @Test
+ void testCleanupStatisticsClaimOnlyFilesDeletedByThisCommit() throws Exception {
+ MixedOwnershipFileIO fileIO = new MixedOwnershipFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ writeOldFiles(fileIO, new Path(tablePath, "year=2025/month=00"), 1);
+ writeOldFiles(fileIO, new Path(tablePath, "year=2025/month=01"), 1);
+ writeOldFiles(fileIO, new Path(tablePath, "year=2025/month=02"), 1);
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ FormatTableCommit commit =
+ new FormatTableCommit(
+ tablePath.toString(),
+ Arrays.asList("year", "month"),
+ fileIO,
+ false,
+ PARTITION_DEFAULT_NAME.defaultValue(),
+ true,
+ Identifier.create("cleanup_db", "cleanup_table"),
+ Collections.singletonMap("year", "2025"),
+ null,
+ null,
+ partitionManager,
+ /* dynamicPartitionOverwrite */ true,
+ 2);
+
+ commit.commit(Collections.emptyList());
+
+ Map owned = partitionSpec("2025", "00");
+ ArgumentCaptor>> specs =
+ ArgumentCaptor.forClass((Class) List.class);
+ ArgumentCaptor> statistics =
+ ArgumentCaptor.forClass((Class) List.class);
+ verify(partitionManager)
+ .createPartitions(specs.capture(), eq(true), statistics.capture(), eq(true));
+ assertThat(specs.getValue()).containsExactly(owned);
+ assertThat(statistics.getValue())
+ .singleElement()
+ .satisfies(
+ stat -> {
+ assertThat(stat.spec()).isEqualTo(owned);
+ assertThat(stat.recordCount()).isZero();
+ assertThat(stat.fileSizeInBytes()).isZero();
+ assertThat(stat.fileCount()).isZero();
+ });
+ assertThat(fileIO.exists(new Path(tablePath, "year=2025/month=00/data-000.csv"))).isFalse();
+ assertThat(fileIO.exists(new Path(tablePath, "year=2025/month=01/data-000.csv"))).isFalse();
+ assertThat(fileIO.exists(new Path(tablePath, "year=2025/month=02/data-000.csv"))).isFalse();
+ }
+
+ @Test
+ void testCleanupRejectsFalseWhenTheOldDataFileStillExists() throws Exception {
+ RefusingDeleteFileIO fileIO = new RefusingDeleteFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ Path partitionPath = new Path(tablePath, "part=p");
+ writeOldFiles(fileIO, partitionPath, 1);
+ TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class);
+ when(committer.targetPath()).thenReturn(new Path(partitionPath, "data-new.csv"));
+ FormatTableCommit commit =
+ newCleanupCommit(tablePath, fileIO, null, Collections.singletonMap("part", "p"), 2);
+
+ assertThatThrownBy(
+ () ->
+ commit.commit(
+ Collections.singletonList(
+ new TwoPhaseCommitMessage(committer))))
+ .hasRootCauseMessage(
+ "Failed to delete data file "
+ + new Path(partitionPath, "data-000.csv")
+ + " of table cleanup_db.cleanup_table.");
+ verify(committer, never()).commit(fileIO);
+ verify(committer).discard(fileIO);
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ @Test
+ void testConcurrentCleanupReportsCompleteStatisticsAfterBarrier() throws Exception {
+ ParallelDeleteFileIO fileIO = new ParallelDeleteFileIO(4);
+ Path tablePath = new Path(tempDir.toUri());
+ for (int month = 0; month < 8; month++) {
+ writeOldFiles(
+ fileIO, new Path(tablePath, String.format("year=2025/month=%02d", month)), 1);
+ }
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ doAnswer(
+ invocation -> {
+ assertThat(fileIO.activeDeletes()).isZero();
+ return null;
+ })
+ .when(partitionManager)
+ .createPartitions(anyList(), eq(true), anyList(), eq(true));
+ FormatTableCommit commit =
+ new FormatTableCommit(
+ tablePath.toString(),
+ Arrays.asList("year", "month"),
+ fileIO,
+ false,
+ PARTITION_DEFAULT_NAME.defaultValue(),
+ true,
+ Identifier.create("cleanup_db", "cleanup_table"),
+ Collections.singletonMap("year", "2025"),
+ null,
+ null,
+ partitionManager,
+ /* dynamicPartitionOverwrite */ true,
+ 4);
+
+ commit.commit(Collections.emptyList());
+
+ List> expectedSpecs = new ArrayList<>();
+ for (int month = 0; month < 8; month++) {
+ expectedSpecs.add(partitionSpec("2025", String.format("%02d", month)));
+ }
+ ArgumentCaptor>> specs =
+ ArgumentCaptor.forClass((Class) List.class);
+ ArgumentCaptor> statistics =
+ ArgumentCaptor.forClass((Class) List.class);
+ verify(partitionManager)
+ .createPartitions(specs.capture(), eq(true), statistics.capture(), eq(true));
+ assertThat(specs.getValue()).containsExactlyInAnyOrderElementsOf(expectedSpecs);
+ assertThat(statistics.getValue())
+ .hasSize(8)
+ .extracting(PartitionStatistics::spec)
+ .containsExactlyInAnyOrderElementsOf(expectedSpecs);
+ assertThat(statistics.getValue())
+ .allSatisfy(
+ stat -> {
+ assertThat(stat.recordCount()).isZero();
+ assertThat(stat.fileSizeInBytes()).isZero();
+ assertThat(stat.fileCount()).isZero();
+ });
+ }
+
+ @Test
+ void testCatalogManagedOverwriteCleanupSpansPartitionDirectories() throws Exception {
+ assertOverwriteCleanupSpansPartitions(/* dynamicPartitionOverwrite */ true);
+ assertOverwriteCleanupSpansPartitions(/* dynamicPartitionOverwrite */ false);
+ }
+
+ @Test
+ void testCleanupDoesNotListEveryPartitionBeforeTheFirstDeleteWindowCompletes()
+ throws Exception {
+ Path tablePath = new Path(new Path(tempDir.toUri()), "lazy-root-listing");
+ Path firstPartition = new Path(tablePath, "part=p0");
+ Path deferredPartition = new Path(tablePath, "part=p1");
+ LazyRootListingFileIO fileIO = new LazyRootListingFileIO(2, deferredPartition);
+ writeOldFiles(fileIO, firstPartition, 2);
+ writeOldFiles(fileIO, deferredPartition, 2);
+
+ List messages = new ArrayList<>();
+ for (Path partition : Arrays.asList(firstPartition, deferredPartition)) {
+ TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class);
+ when(committer.targetPath()).thenReturn(new Path(partition, "data-new.csv"));
+ messages.add(new TwoPhaseCommitMessage(committer));
+ }
+ Map options = new LinkedHashMap<>();
+ options.put(CoreOptions.FORMAT_TABLE_COMMIT_CLEANUP_THREAD_NUM.key(), "2");
+ FormatTableCommit commit =
+ builderOverwriteCommit(
+ tablePath,
+ fileIO,
+ mock(FormatTablePartitionManager.class),
+ options,
+ /* staticPartition */ null);
+
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ Future result = null;
+ try {
+ result = executor.submit(() -> commit.commit(messages));
+ assertThat(fileIO.awaitFirstWave()).isTrue();
+ assertThat(fileIO.deferredRootListed()).isFalse();
+ } finally {
+ fileIO.releaseFirstWave();
+ if (result != null) {
+ result.get(10, TimeUnit.SECONDS);
+ }
+ executor.shutdownNow();
+ }
+
+ assertThat(fileIO.deferredRootListed()).isTrue();
+ assertThat(fileIO.deleteCalls()).isEqualTo(4);
+ }
+
+ @Test
+ void testConcurrentCommitsShareCleanupPoolFairlyAtWorkerBatchBoundary() throws Exception {
+ FairnessDeleteFileIO largeFileIO = new FairnessDeleteFileIO(64);
+ Path largeTablePath = new Path(new Path(tempDir.toUri()), "large");
+ Path largePartitionPath = new Path(largeTablePath, "part=p");
+ writeOldFiles(largeFileIO, largePartitionPath, 128);
+ SubmissionTrackingExecutor cleanupExecutor =
+ new SubmissionTrackingExecutor(Executors.newFixedThreadPool(64));
+ FormatTableCommit largeCommit =
+ newCleanupCommit(
+ largeTablePath,
+ largeFileIO,
+ mock(FormatTablePartitionManager.class),
+ Collections.singletonMap("part", "p"),
+ 64,
+ cleanupExecutor);
+
+ ParallelDeleteFileIO smallFileIO = new ParallelDeleteFileIO(2);
+ Path smallTablePath = new Path(new Path(tempDir.toUri()), "small");
+ Path smallPartitionPath = new Path(smallTablePath, "part=p");
+ writeOldFiles(smallFileIO, smallPartitionPath, 2);
+ FormatTableCommit smallCommit =
+ newCleanupCommit(
+ smallTablePath,
+ smallFileIO,
+ mock(FormatTablePartitionManager.class),
+ Collections.singletonMap("part", "p"),
+ 64,
+ cleanupExecutor);
+
+ ExecutorService commits = Executors.newFixedThreadPool(2);
+ Future largeResult = null;
+ Future smallResult = null;
+ try {
+ largeResult = commits.submit(() -> largeCommit.commit(Collections.emptyList()));
+ assertThat(largeFileIO.awaitFirstWave()).isTrue();
+
+ cleanupExecutor.armTwoAcceptedSubmissions();
+ smallResult = commits.submit(() -> smallCommit.commit(Collections.emptyList()));
+ assertThat(cleanupExecutor.awaitTwoAcceptedSubmissions()).isTrue();
+ largeFileIO.releaseFirstWave();
+
+ assertThat(smallFileIO.awaitFirstWave(2, TimeUnit.SECONDS)).isTrue();
+ assertThat(largeResult.isDone()).isFalse();
+ } finally {
+ largeFileIO.releaseAllDeletes();
+ try {
+ if (largeResult != null) {
+ largeResult.get(10, TimeUnit.SECONDS);
+ }
+ if (smallResult != null) {
+ smallResult.get(10, TimeUnit.SECONDS);
+ }
+ } finally {
+ commits.shutdownNow();
+ cleanupExecutor.shutdownNow();
+ }
+ }
+ }
+
+ @Test
+ void testCleanupPropagatesAndRestoresContextClassLoaderOnReusedWorkers() throws Exception {
+ ExecutorService cleanupExecutor = Executors.newFixedThreadPool(4);
+ ClassLoader originalClassLoader = Thread.currentThread().getContextClassLoader();
+ ClassLoader callerClassLoader = new ClassLoader(originalClassLoader) {};
+ Path tablePath = new Path(new Path(tempDir.toUri()), "context-class-loader");
+ Path partitionPath = new Path(tablePath, "part=p");
+ ContextClassLoaderRecordingFileIO fileIO = new ContextClassLoaderRecordingFileIO();
+ writeOldFiles(fileIO, partitionPath, 4);
+ FormatTableCommit commit =
+ newCleanupCommit(
+ tablePath,
+ fileIO,
+ null,
+ Collections.singletonMap("part", "p"),
+ 4,
+ cleanupExecutor);
+
+ try {
+ assertThat(observeContextClassLoaders(cleanupExecutor, 4))
+ .allMatch(loader -> loader == originalClassLoader);
+ Thread.currentThread().setContextClassLoader(callerClassLoader);
+
+ commit.commit(Collections.emptyList());
+
+ assertThat(fileIO.contextClassLoaders())
+ .hasSize(4)
+ .allMatch(loader -> loader == callerClassLoader);
+ assertThat(observeContextClassLoaders(cleanupExecutor, 4))
+ .allMatch(loader -> loader == originalClassLoader);
+ } finally {
+ Thread.currentThread().setContextClassLoader(originalClassLoader);
+ cleanupExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ void testBuilderCleanupConcurrencyDoesNotApplyToTruncateOperations() throws Exception {
+ SerialProbeFileIO tableFileIO = new SerialProbeFileIO();
+ Path tablePath = new Path(new Path(tempDir.toUri()), "truncate-table");
+ Path tablePartitionPath = new Path(tablePath, "part=p");
+ writeOldFiles(tableFileIO, tablePartitionPath, 3);
+ FormatTablePartitionManager tableManager = mock(FormatTablePartitionManager.class);
+ when(tableManager.listPartitions(Collections.emptyMap(), null))
+ .thenReturn(
+ Collections.singletonList(
+ new Partition(
+ Collections.singletonMap("part", "p"),
+ 0,
+ 0,
+ 0,
+ 0,
+ -1,
+ false)));
+ builderTruncateCommit(tablePath, tableFileIO, tableManager).truncateTable();
+
+ SerialProbeFileIO partitionFileIO = new SerialProbeFileIO();
+ Path partitionsPath = new Path(new Path(tempDir.toUri()), "truncate-partitions");
+ Path namedPartitionPath = new Path(partitionsPath, "part=p");
+ writeOldFiles(partitionFileIO, namedPartitionPath, 3);
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ when(partitionManager.listPartitionsByNames(anyList()))
+ .thenReturn(
+ Collections.singletonList(
+ new Partition(
+ Collections.singletonMap("part", "p"),
+ 0,
+ 0,
+ 0,
+ 0,
+ -1,
+ false)));
+ builderTruncateCommit(partitionsPath, partitionFileIO, partitionManager)
+ .truncatePartitions(
+ Collections.singletonList(Collections.singletonMap("part", "p")));
+
+ assertThat(tableFileIO.maxConcurrentDeletes()).isEqualTo(1);
+ assertThat(partitionFileIO.maxConcurrentDeletes()).isEqualTo(1);
+ }
+
+ @Test
+ void testAbortFailureDoesNotReplaceInterruptedCleanupFailure() throws Exception {
+ BlockingDeleteFileIO fileIO = new BlockingDeleteFileIO(2);
+ Path tablePath = new Path(new Path(tempDir.toUri()), "abort-failure");
+ Path partitionPath = new Path(tablePath, "part=p");
+ writeOldFiles(fileIO, partitionPath, 2);
+ TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class);
+ when(committer.targetPath()).thenReturn(new Path(partitionPath, "data-new.csv"));
+ doThrow(new IOException("discard failed after cleanup interruption"))
+ .when(committer)
+ .discard(fileIO);
+ FormatTableCommit commit =
+ newCleanupCommit(tablePath, fileIO, null, Collections.singletonMap("part", "p"), 2);
+ AtomicReference failure = new AtomicReference<>();
+ AtomicBoolean interruptRestored = new AtomicBoolean();
+ Thread commitThread =
+ new Thread(
+ () -> {
+ try {
+ commit.commit(
+ Collections.singletonList(
+ new TwoPhaseCommitMessage(committer)));
+ } catch (Throwable t) {
+ failure.set(t);
+ } finally {
+ interruptRestored.set(Thread.currentThread().isInterrupted());
+ }
+ },
+ "format-cleanup-abort-failure-caller");
+
+ commitThread.start();
+ try {
+ assertThat(fileIO.awaitDeletesStarted()).isTrue();
+ commitThread.interrupt();
+ fileIO.releaseDeletes();
+ commitThread.join(TimeUnit.SECONDS.toMillis(10));
+ } finally {
+ fileIO.releaseDeletes();
+ commitThread.interrupt();
+ commitThread.join(TimeUnit.SECONDS.toMillis(10));
+ }
+
+ assertThat(commitThread.isAlive()).isFalse();
+ assertThat(failure.get()).isNotNull();
+ assertThat(rootCause(failure.get())).isInstanceOf(InterruptedException.class);
+ assertThat(failureTree(failure.get()))
+ .extracting(Throwable::getMessage)
+ .contains("discard failed after cleanup interruption");
+ assertThat(interruptRestored).isTrue();
+ verify(committer).discard(fileIO);
+ verify(committer, never()).commit(fileIO);
+ }
+
/**
* An overwrite that names no partition: what INSERT OVERWRITE without a PARTITION clause is.
*/
@@ -655,6 +1515,679 @@ private FormatTableCommit overwritingCommit(
dynamicPartitionOverwrite);
}
+ private void assertOverwriteCleanupSpansPartitions(boolean dynamicPartitionOverwrite)
+ throws Exception {
+ ParallelDeleteFileIO fileIO = new ParallelDeleteFileIO(4);
+ Path tablePath =
+ new Path(
+ new Path(tempDir.toUri()),
+ dynamicPartitionOverwrite ? "dynamic-roots" : "whole-roots");
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ List partitions = new ArrayList<>();
+ List messages = new ArrayList<>();
+ for (int i = 0; i < 4; i++) {
+ String value = "p" + i;
+ Path partitionPath = new Path(tablePath, "part=" + value);
+ writeOldFiles(fileIO, partitionPath, 1);
+ partitions.add(
+ new Partition(Collections.singletonMap("part", value), 0, 0, 0, 0, -1, false));
+ if (dynamicPartitionOverwrite) {
+ TwoPhaseOutputStream.Committer committer =
+ mock(TwoPhaseOutputStream.Committer.class);
+ when(committer.targetPath()).thenReturn(new Path(partitionPath, "data-new.csv"));
+ messages.add(new TwoPhaseCommitMessage(committer));
+ }
+ }
+ if (!dynamicPartitionOverwrite) {
+ when(partitionManager.listPartitions(Collections.emptyMap(), null))
+ .thenReturn(partitions);
+ }
+ Map options = new LinkedHashMap<>();
+ options.put(CoreOptions.FORMAT_TABLE_COMMIT_CLEANUP_THREAD_NUM.key(), "4");
+ options.put(
+ CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key(),
+ Boolean.toString(dynamicPartitionOverwrite));
+ FormatTableCommit commit =
+ builderOverwriteCommit(
+ tablePath, fileIO, partitionManager, options, /* staticPartition */ null);
+
+ commit.commit(messages);
+
+ assertThat(fileIO.deleteCalls()).isEqualTo(4);
+ assertThat(fileIO.maxConcurrentDeletes()).isEqualTo(4);
+ }
+
+ private FormatTableCommit builderOverwriteCommit(
+ Path tablePath,
+ FileIO fileIO,
+ FormatTablePartitionManager partitionManager,
+ Map options,
+ Map staticPartition) {
+ FormatTable table = formatTable(tablePath, fileIO, partitionManager, options);
+ BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder();
+ writeBuilder.withOverwrite(staticPartition);
+ return (FormatTableCommit) writeBuilder.newCommit();
+ }
+
+ private FormatTableCommit builderTruncateCommit(
+ Path tablePath, FileIO fileIO, FormatTablePartitionManager partitionManager) {
+ return (FormatTableCommit)
+ formatTable(tablePath, fileIO, partitionManager, Collections.emptyMap())
+ .newBatchWriteBuilder()
+ .newCommit();
+ }
+
+ private FormatTable formatTable(
+ Path tablePath,
+ FileIO fileIO,
+ FormatTablePartitionManager partitionManager,
+ Map options) {
+ RowType rowType =
+ RowType.builder()
+ .field("part", DataTypes.STRING())
+ .field("id", DataTypes.INT())
+ .build();
+ FormatTable table =
+ FormatTable.builder()
+ .fileIO(fileIO)
+ .identifier(Identifier.create("cleanup_db", "cleanup_table"))
+ .rowType(rowType)
+ .partitionKeys(Collections.singletonList("part"))
+ .location(tablePath.toString())
+ .format(FormatTable.Format.CSV)
+ .options(options)
+ .partitionManager(partitionManager)
+ .build();
+ return table;
+ }
+
+ private FormatTableCommit builderUnpartitionedOverwriteCommit(
+ Path tablePath,
+ FileIO fileIO,
+ FormatTablePartitionManager partitionManager,
+ Map options) {
+ FormatTable table =
+ FormatTable.builder()
+ .fileIO(fileIO)
+ .identifier(Identifier.create("cleanup_db", "cleanup_table"))
+ .rowType(RowType.builder().field("id", DataTypes.INT()).build())
+ .partitionKeys(Collections.emptyList())
+ .location(tablePath.toString())
+ .format(FormatTable.Format.CSV)
+ .options(options)
+ .partitionManager(partitionManager)
+ .build();
+ BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder();
+ writeBuilder.withOverwrite(null);
+ return (FormatTableCommit) writeBuilder.newCommit();
+ }
+
+ private FormatTableCommit newCleanupCommit(
+ Path tablePath,
+ FileIO fileIO,
+ FormatTablePartitionManager partitionManager,
+ Map staticPartition,
+ int cleanupThreadNum) {
+ return new FormatTableCommit(
+ tablePath.toString(),
+ Collections.singletonList("part"),
+ fileIO,
+ false,
+ PARTITION_DEFAULT_NAME.defaultValue(),
+ true,
+ Identifier.create("cleanup_db", "cleanup_table"),
+ staticPartition,
+ null,
+ null,
+ partitionManager,
+ /* dynamicPartitionOverwrite */ true,
+ cleanupThreadNum);
+ }
+
+ private FormatTableCommit newCleanupCommit(
+ Path tablePath,
+ FileIO fileIO,
+ FormatTablePartitionManager partitionManager,
+ Map staticPartition,
+ int cleanupThreadNum,
+ ExecutorService cleanupExecutor) {
+ return new FormatTableCommit(
+ tablePath.toString(),
+ Collections.singletonList("part"),
+ fileIO,
+ false,
+ PARTITION_DEFAULT_NAME.defaultValue(),
+ true,
+ Identifier.create("cleanup_db", "cleanup_table"),
+ staticPartition,
+ null,
+ null,
+ partitionManager,
+ /* dynamicPartitionOverwrite */ true,
+ cleanupThreadNum,
+ cleanupExecutor);
+ }
+
+ private static void writeOldFiles(LocalFileIO fileIO, Path partitionPath, int count)
+ throws IOException {
+ for (int i = 0; i < count; i++) {
+ fileIO.writeFile(
+ new Path(partitionPath, String.format("data-%03d.csv", i)), "old", false);
+ }
+ }
+
+ private static List causeChain(Throwable throwable) {
+ List chain = new ArrayList<>();
+ Throwable current = throwable;
+ while (current != null) {
+ chain.add(current);
+ current = current.getCause();
+ }
+ return chain;
+ }
+
+ private static final class SubmissionTrackingExecutor extends AbstractExecutorService {
+
+ private final ExecutorService delegate;
+ private final AtomicReference acceptedSubmissions = new AtomicReference<>();
+
+ private SubmissionTrackingExecutor(ExecutorService delegate) {
+ this.delegate = delegate;
+ }
+
+ private void armTwoAcceptedSubmissions() {
+ if (!acceptedSubmissions.compareAndSet(null, new CountDownLatch(2))) {
+ throw new IllegalStateException("Accepted-submission tracking is already armed.");
+ }
+ }
+
+ private boolean awaitTwoAcceptedSubmissions() throws InterruptedException {
+ CountDownLatch submissions = acceptedSubmissions.get();
+ if (submissions == null) {
+ throw new IllegalStateException("Accepted-submission tracking is not armed.");
+ }
+ return submissions.await(10, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void shutdown() {
+ delegate.shutdown();
+ }
+
+ @Override
+ public List shutdownNow() {
+ return delegate.shutdownNow();
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return delegate.isShutdown();
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return delegate.isTerminated();
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ return delegate.awaitTermination(timeout, unit);
+ }
+
+ @Override
+ public void execute(Runnable command) {
+ delegate.execute(command);
+ CountDownLatch submissions = acceptedSubmissions.get();
+ if (submissions != null) {
+ submissions.countDown();
+ }
+ }
+ }
+
+ private static class FairnessDeleteFileIO extends LocalFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int firstWaveSize;
+ private final CountDownLatch firstWave;
+ private final CountDownLatch releaseFirstWave = new CountDownLatch(1);
+ private final CountDownLatch releaseAllDeletes = new CountDownLatch(1);
+ private final AtomicInteger deleteCalls = new AtomicInteger();
+
+ private FairnessDeleteFileIO(int firstWaveSize) {
+ this.firstWaveSize = firstWaveSize;
+ this.firstWave = new CountDownLatch(firstWaveSize);
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive) throws IOException {
+ int call = deleteCalls.incrementAndGet();
+ try {
+ if (call <= firstWaveSize) {
+ firstWave.countDown();
+ if (!firstWave.await(10, TimeUnit.SECONDS)
+ || !releaseFirstWave.await(10, TimeUnit.SECONDS)) {
+ throw new IOException("Timed out waiting for the first fairness wave");
+ }
+ } else if (!releaseAllDeletes.await(10, TimeUnit.SECONDS)) {
+ throw new IOException("Timed out waiting for remaining fairness deletes");
+ }
+ return super.delete(path, recursive);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted in cleanup fairness fixture", e);
+ }
+ }
+
+ private boolean awaitFirstWave() throws InterruptedException {
+ return firstWave.await(10, TimeUnit.SECONDS);
+ }
+
+ private void releaseFirstWave() {
+ releaseFirstWave.countDown();
+ }
+
+ private void releaseAllDeletes() {
+ releaseFirstWave.countDown();
+ releaseAllDeletes.countDown();
+ }
+ }
+
+ private static class ParallelDeleteFileIO extends LocalFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int firstWaveSize;
+ private final boolean holdFirstWave;
+ private final CountDownLatch firstWave;
+ private final CountDownLatch releaseFirstWave = new CountDownLatch(1);
+ private final CountDownLatch unexpectedExtraDelete = new CountDownLatch(1);
+ private final AtomicInteger deleteCalls = new AtomicInteger();
+ private final AtomicInteger activeDeletes = new AtomicInteger();
+ private final AtomicInteger maxConcurrentDeletes = new AtomicInteger();
+
+ private ParallelDeleteFileIO(int firstWaveSize) {
+ this(firstWaveSize, false);
+ }
+
+ private ParallelDeleteFileIO(int firstWaveSize, boolean holdFirstWave) {
+ this.firstWaveSize = firstWaveSize;
+ this.holdFirstWave = holdFirstWave;
+ this.firstWave = new CountDownLatch(firstWaveSize);
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive) throws IOException {
+ int call = deleteCalls.incrementAndGet();
+ int active = activeDeletes.incrementAndGet();
+ maxConcurrentDeletes.updateAndGet(previous -> Math.max(previous, active));
+ try {
+ if (call <= firstWaveSize) {
+ firstWave.countDown();
+ if (!firstWave.await(10, TimeUnit.SECONDS)) {
+ throw new IOException("Expected cleanup delete calls did not overlap");
+ }
+ if (holdFirstWave && !releaseFirstWave.await(10, TimeUnit.SECONDS)) {
+ throw new IOException("Test did not release the first cleanup wave");
+ }
+ } else {
+ unexpectedExtraDelete.countDown();
+ }
+ return super.delete(path, recursive);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while observing cleanup concurrency", e);
+ } finally {
+ activeDeletes.decrementAndGet();
+ }
+ }
+
+ protected int deleteCalls() {
+ return deleteCalls.get();
+ }
+
+ private int maxConcurrentDeletes() {
+ return maxConcurrentDeletes.get();
+ }
+
+ private int activeDeletes() {
+ return activeDeletes.get();
+ }
+
+ protected boolean awaitFirstWave() throws InterruptedException {
+ return firstWave.await(10, TimeUnit.SECONDS);
+ }
+
+ private boolean awaitFirstWave(long timeout, TimeUnit unit) throws InterruptedException {
+ return firstWave.await(timeout, unit);
+ }
+
+ private boolean awaitUnexpectedExtraDelete() throws InterruptedException {
+ return unexpectedExtraDelete.await(300, TimeUnit.MILLISECONDS);
+ }
+
+ protected void releaseFirstWave() {
+ releaseFirstWave.countDown();
+ }
+ }
+
+ private static class LazyRootListingFileIO extends ParallelDeleteFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final Path deferredRoot;
+ private final AtomicBoolean deferredRootListed = new AtomicBoolean();
+
+ private LazyRootListingFileIO(int firstWaveSize, Path deferredRoot) {
+ super(firstWaveSize, true);
+ this.deferredRoot = deferredRoot;
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ if (deferredRoot.equals(path)) {
+ deferredRootListed.set(true);
+ }
+ return super.listStatus(path);
+ }
+
+ private boolean deferredRootListed() {
+ return deferredRootListed.get();
+ }
+ }
+
+ private static class SerialProbeFileIO extends LocalFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final CountDownLatch secondDeleteStarted = new CountDownLatch(1);
+ private final AtomicInteger deleteCalls = new AtomicInteger();
+ private final AtomicInteger activeDeletes = new AtomicInteger();
+ private final AtomicInteger maxConcurrentDeletes = new AtomicInteger();
+
+ @Override
+ public boolean delete(Path path, boolean recursive) throws IOException {
+ int call = deleteCalls.incrementAndGet();
+ int active = activeDeletes.incrementAndGet();
+ maxConcurrentDeletes.updateAndGet(previous -> Math.max(previous, active));
+ try {
+ if (call == 1) {
+ secondDeleteStarted.await(300, TimeUnit.MILLISECONDS);
+ } else {
+ secondDeleteStarted.countDown();
+ }
+ return super.delete(path, recursive);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while probing serial cleanup", e);
+ } finally {
+ activeDeletes.decrementAndGet();
+ }
+ }
+
+ private int deleteCalls() {
+ return deleteCalls.get();
+ }
+
+ private int maxConcurrentDeletes() {
+ return maxConcurrentDeletes.get();
+ }
+ }
+
+ private static class BlockingDeleteFileIO extends LocalFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final CountDownLatch deletesStarted;
+ private final CountDownLatch releaseDeletes = new CountDownLatch(1);
+ private final AtomicInteger activeDeletes = new AtomicInteger();
+
+ private BlockingDeleteFileIO(int deleteCount) {
+ this.deletesStarted = new CountDownLatch(deleteCount);
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive) throws IOException {
+ activeDeletes.incrementAndGet();
+ deletesStarted.countDown();
+ try {
+ if (!releaseDeletes.await(10, TimeUnit.SECONDS)) {
+ throw new IOException("Test did not release blocked cleanup deletes");
+ }
+ return super.delete(path, recursive);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while blocking cleanup delete", e);
+ } finally {
+ activeDeletes.decrementAndGet();
+ }
+ }
+
+ private boolean awaitDeletesStarted() throws InterruptedException {
+ return deletesStarted.await(10, TimeUnit.SECONDS);
+ }
+
+ private void releaseDeletes() {
+ releaseDeletes.countDown();
+ }
+
+ private int activeDeletes() {
+ return activeDeletes.get();
+ }
+ }
+
+ private abstract static class SortedLocalFileIO extends LocalFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ FileStatus[] statuses = super.listStatus(path);
+ Arrays.sort(statuses, Comparator.comparing(status -> status.getPath().toString()));
+ return statuses;
+ }
+
+ protected static void await(CountDownLatch latch, String description) throws IOException {
+ try {
+ if (!latch.await(10, TimeUnit.SECONDS)) {
+ throw new IOException("Timed out waiting for " + description);
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while waiting for " + description, e);
+ }
+ }
+ }
+
+ private static class LaterRootListingFailureFileIO extends SortedLocalFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final Path failingRoot;
+ private final CountDownLatch deletesStarted;
+ private final CountDownLatch listingFailure = new CountDownLatch(1);
+ private final CountDownLatch releaseFirstDelete = new CountDownLatch(1);
+ private final CountDownLatch releaseSecondDelete = new CountDownLatch(1);
+ private final CountDownLatch firstDeleteReturned = new CountDownLatch(1);
+ private final AtomicInteger activeDeletes = new AtomicInteger();
+
+ private LaterRootListingFailureFileIO(Path failingRoot, int deleteCount) {
+ this.failingRoot = failingRoot;
+ this.deletesStarted = new CountDownLatch(deleteCount);
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ if (failingRoot.equals(path)) {
+ await(deletesStarted, "accepted deletes before later-root listing failure");
+ listingFailure.countDown();
+ throw new IOException("Failed to list the later partition root.");
+ }
+ return super.listStatus(path);
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive) throws IOException {
+ activeDeletes.incrementAndGet();
+ deletesStarted.countDown();
+ try {
+ if ("data-000.csv".equals(path.getName())) {
+ await(releaseFirstDelete, "release of the first accepted delete");
+ } else {
+ await(releaseSecondDelete, "release of the second accepted delete");
+ }
+ return super.delete(path, recursive);
+ } finally {
+ activeDeletes.decrementAndGet();
+ if ("data-000.csv".equals(path.getName())) {
+ firstDeleteReturned.countDown();
+ }
+ }
+ }
+
+ private boolean awaitListingFailure() throws InterruptedException {
+ return listingFailure.await(10, TimeUnit.SECONDS);
+ }
+
+ private void releaseFirstDelete() {
+ releaseFirstDelete.countDown();
+ }
+
+ private boolean awaitFirstDeleteReturned() throws InterruptedException {
+ return firstDeleteReturned.await(10, TimeUnit.SECONDS);
+ }
+
+ private void releaseSecondDelete() {
+ releaseSecondDelete.countDown();
+ }
+
+ private int activeDeletes() {
+ return activeDeletes.get();
+ }
+ }
+
+ private static class ContextClassLoaderRecordingFileIO extends LocalFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final ConcurrentLinkedQueue contextClassLoaders =
+ new ConcurrentLinkedQueue<>();
+
+ @Override
+ public boolean delete(Path path, boolean recursive) throws IOException {
+ contextClassLoaders.add(Thread.currentThread().getContextClassLoader());
+ return super.delete(path, recursive);
+ }
+
+ private ConcurrentLinkedQueue contextClassLoaders() {
+ return contextClassLoaders;
+ }
+ }
+
+ private static class FailureDrainFileIO extends SortedLocalFileIO {
+
+ private static final long serialVersionUID = 1L;
+
+ private final CountDownLatch firstPairStarted = new CountDownLatch(2);
+ private final CountDownLatch failureAttempted = new CountDownLatch(1);
+ private final CountDownLatch releaseSuccessfulSibling = new CountDownLatch(1);
+ private final ConcurrentLinkedQueue attemptedFiles = new ConcurrentLinkedQueue<>();
+ private final ConcurrentLinkedQueue