diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b3dd39c5..d7951f862 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,7 @@ All notable changes to this project will be documented in this file. - opensearch-dashboards: Remove `3.4.0` ([#1675]). - druid: Remove `30.0.1`, `35.0.1` ([#1677]). - nifi: Remove `2.7.2` ([#1667]). +- hadoop: Remove `3.4.2` ([#1683]). [#1593]: https://github.com/stackabletech/docker-images/pull/1593 [#1595]: https://github.com/stackabletech/docker-images/pull/1595 @@ -115,6 +116,7 @@ All notable changes to this project will be documented in this file. [#1676]: https://github.com/stackabletech/docker-images/pull/1676 [#1677]: https://github.com/stackabletech/docker-images/pull/1677 [#1681]: https://github.com/stackabletech/docker-images/pull/1681 +[#1683]: https://github.com/stackabletech/docker-images/pull/1683 ## [26.7.0] - 2026-07-21 diff --git a/hadoop/boil-config.toml b/hadoop/boil-config.toml index e082c3fe2..a9279c640 100644 --- a/hadoop/boil-config.toml +++ b/hadoop/boil-config.toml @@ -12,16 +12,6 @@ async-profiler-version = "2.9" jmx-exporter-version = "1.3.0" hdfs-utils-version = "0.4.0" -[versions."3.4.2".local-images] -"hadoop/hadoop" = "3.4.2" -java-base = "11" -java-devel = "11" - -[versions."3.4.2".build-arguments] -async-profiler-version = "2.9" -jmx-exporter-version = "1.3.0" -hdfs-utils-version = "0.6.0" - [versions."3.4.3".local-images] "hadoop/hadoop" = "3.4.3" java-base = "11" diff --git a/hadoop/hadoop/Dockerfile b/hadoop/hadoop/Dockerfile index de701628a..8b573fb4d 100644 --- a/hadoop/hadoop/Dockerfile +++ b/hadoop/hadoop/Dockerfile @@ -168,25 +168,12 @@ NEW_VERSION=${PRODUCT_VERSION}-stackable${RELEASE_VERSION} # have one central place for dependant images to pull them from mkdir -p /stackable/hadoop-cloud-libraries/ -if [[ "$PRODUCT_VERSION" == "3.3.6" || "$PRODUCT_VERSION" == "3.4.2" ]]; then - cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/ - # For some reason it was renamed from "aws-java-sdk-bundle" to "bundle" - if [[ "$PRODUCT_VERSION" == "3.3.6" ]]; then - cp /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar - else - cp /stackable/hadoop/share/hadoop/tools/lib/bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar - fi - cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/ - cp /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar /stackable/hadoop-cloud-libraries/ - cp /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar /stackable/hadoop-cloud-libraries/ -else - cp /stackable/hadoop/share/hadoop/common/lib/hadoop-aws-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/ - # Starting with Hadoop 3.4.3 the aws-java-sdk-bundle is not included any more. - curl -o /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar --fail "https://repo.stackable.tech/repository/packages/aws/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar" - # Starting with Hadoop 3.4.3 the S3A InputStreamType factory references analyticsaccelerator-s3 (software.amazon.s3.analyticsaccelerator.request.ObjectClient) during S3AFileSystem.initialize. - curl -o /stackable/hadoop-cloud-libraries/analyticsaccelerator-s3-${ANALYTICSACCELERATOR_S3_VERSION}.jar --fail "https://repo.stackable.tech/repository/packages/aws/analyticsaccelerator-s3-${ANALYTICSACCELERATOR_S3_VERSION}.jar" - cp /stackable/hadoop/share/hadoop/common/lib/hadoop-azure-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/ - cp /stackable/hadoop/share/hadoop/common/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar /stackable/hadoop-cloud-libraries/ - cp /stackable/hadoop/share/hadoop/common/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar /stackable/hadoop-cloud-libraries/ -fi +cp /stackable/hadoop/share/hadoop/common/lib/hadoop-aws-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/ +# Starting with Hadoop 3.4.3 the aws-java-sdk-bundle is not included any more. +curl -o /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar --fail "https://repo.stackable.tech/repository/packages/aws/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar" +# Starting with Hadoop 3.4.3 the S3A InputStreamType factory references analyticsaccelerator-s3 (software.amazon.s3.analyticsaccelerator.request.ObjectClient) during S3AFileSystem.initialize. +curl -o /stackable/hadoop-cloud-libraries/analyticsaccelerator-s3-${ANALYTICSACCELERATOR_S3_VERSION}.jar --fail "https://repo.stackable.tech/repository/packages/aws/analyticsaccelerator-s3-${ANALYTICSACCELERATOR_S3_VERSION}.jar" +cp /stackable/hadoop/share/hadoop/common/lib/hadoop-azure-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/ +cp /stackable/hadoop/share/hadoop/common/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar /stackable/hadoop-cloud-libraries/ +cp /stackable/hadoop/share/hadoop/common/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar /stackable/hadoop-cloud-libraries/ EOF diff --git a/hadoop/hadoop/boil-config.toml b/hadoop/hadoop/boil-config.toml index 9b2cdea1c..8dd9fc048 100644 --- a/hadoop/hadoop/boil-config.toml +++ b/hadoop/hadoop/boil-config.toml @@ -8,19 +8,6 @@ aws-java-sdk-bundle-version = "1.12.367" # Needs to match https://mvnrepository azure-storage-version = "7.0.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.6 azure-keyvault-core-version = "1.0.0" # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 - -[versions."3.4.2".local-images] -java-devel = "11" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions -"shared/boost" = "1.72.0" # https://github.com/apache/hadoop/blob/rel/release-3.4.2/BUILDING.txt - -[versions."3.4.2".build-arguments] -protobuf-version = "3.21.12" # https://github.com/apache/hadoop/blob/rel/release-3.4.2/BUILDING.txt -aws-java-sdk-bundle-version = "2.29.52" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.2 -azure-storage-version = "7.0.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.2 -azure-keyvault-core-version = "1.0.0" # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 - - - [versions."3.4.3".local-images] java-devel = "11" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions "shared/boost" = "1.72.0" # https://github.com/apache/hadoop/blob/rel/release-3.4.3/BUILDING.txt @@ -32,8 +19,6 @@ azure-storage-version = "7.0.1" # Needs to match https://mvnrepository azure-keyvault-core-version = "1.0.0" # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 analyticsaccelerator-s3-version = "1.3.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.3 - - [versions."3.5.0".local-images] java-devel = "17" # https://cwiki.apache.org/confluence/display/HADOOP/Hadoop+Java+Versions "shared/boost" = "1.86.0" # https://github.com/apache/hadoop/blob/rel/release-3.5.0/BUILDING.txt diff --git a/hadoop/hadoop/stackable/patches/3.4.2/0001-YARN-11527-Update-node.js.patch b/hadoop/hadoop/stackable/patches/3.4.2/0001-YARN-11527-Update-node.js.patch deleted file mode 100644 index 497600fa1..000000000 --- a/hadoop/hadoop/stackable/patches/3.4.2/0001-YARN-11527-Update-node.js.patch +++ /dev/null @@ -1,22 +0,0 @@ -From c4dbb05b4f92f93c7e8f11d6a622b73f40f4664c Mon Sep 17 00:00:00 2001 -From: xeniape -Date: Wed, 10 Sep 2025 14:18:38 +0200 -Subject: YARN-11527-Update-node.js - ---- - hadoop-project/pom.xml | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml -index b9eacd5ba3..70f64bf55c 100644 ---- a/hadoop-project/pom.xml -+++ b/hadoop-project/pom.xml -@@ -234,7 +234,7 @@ - 1.0.2 - 5.4.0 - 9.37.2 -- v12.22.1 -+ v14.17.0 - v1.22.5 - 1.10.13 - 1.20 diff --git a/hadoop/hadoop/stackable/patches/3.4.2/0002-Allow-overriding-datanode-registration-addresses.patch b/hadoop/hadoop/stackable/patches/3.4.2/0002-Allow-overriding-datanode-registration-addresses.patch deleted file mode 100644 index 0f2b9d916..000000000 --- a/hadoop/hadoop/stackable/patches/3.4.2/0002-Allow-overriding-datanode-registration-addresses.patch +++ /dev/null @@ -1,259 +0,0 @@ -From adc337817824ba29e7eb669c13730acdbb0b9630 Mon Sep 17 00:00:00 2001 -From: xeniape -Date: Wed, 10 Sep 2025 14:36:20 +0200 -Subject: Allow-overriding-datanode-registration-addresses - ---- - .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 9 +++ - .../blockmanagement/DatanodeManager.java | 43 +++++++----- - .../hadoop/hdfs/server/datanode/DNConf.java | 70 +++++++++++++++++++ - .../hadoop/hdfs/server/datanode/DataNode.java | 35 ++++++++-- - 4 files changed, 135 insertions(+), 22 deletions(-) - -diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java -index f92a2ad565..25bcd438c7 100755 ---- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java -+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java -@@ -152,6 +152,13 @@ public class DFSConfigKeys extends CommonConfigurationKeys { - public static final boolean DFS_DATANODE_DROP_CACHE_BEHIND_READS_DEFAULT = false; - public static final String DFS_DATANODE_USE_DN_HOSTNAME = "dfs.datanode.use.datanode.hostname"; - public static final boolean DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT = false; -+ -+ public static final String DFS_DATANODE_REGISTERED_HOSTNAME = "dfs.datanode.registered.hostname"; -+ public static final String DFS_DATANODE_REGISTERED_DATA_PORT = "dfs.datanode.registered.port"; -+ public static final String DFS_DATANODE_REGISTERED_HTTP_PORT = "dfs.datanode.registered.http.port"; -+ public static final String DFS_DATANODE_REGISTERED_HTTPS_PORT = "dfs.datanode.registered.https.port"; -+ public static final String DFS_DATANODE_REGISTERED_IPC_PORT = "dfs.datanode.registered.ipc.port"; -+ - public static final String DFS_DATANODE_MAX_LOCKED_MEMORY_KEY = "dfs.datanode.max.locked.memory"; - public static final long DFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT = 0; - public static final String DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY = "dfs.datanode.fsdatasetcache.max.threads.per.volume"; -@@ -491,6 +498,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { - public static final long DFS_DATANODE_PROCESS_COMMANDS_THRESHOLD_DEFAULT = - TimeUnit.SECONDS.toMillis(2); - -+ public static final String DFS_NAMENODE_DATANODE_REGISTRATION_UNSAFE_ALLOW_ADDRESS_OVERRIDE_KEY = "dfs.namenode.datanode.registration.unsafe.allow-address-override"; -+ public static final boolean DFS_NAMENODE_DATANODE_REGISTRATION_UNSAFE_ALLOW_ADDRESS_OVERRIDE_DEFAULT = false; - public static final String DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY = "dfs.namenode.datanode.registration.ip-hostname-check"; - public static final boolean DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT = true; - -diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java -index ebd2fa992e..c56f254478 100644 ---- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java -+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java -@@ -181,6 +181,8 @@ public class DatanodeManager { - private boolean hasClusterEverBeenMultiRack = false; - - private final boolean checkIpHostnameInRegistration; -+ private final boolean allowRegistrationAddressOverride; -+ - /** - * Whether we should tell datanodes what to cache in replies to - * heartbeat messages. -@@ -314,6 +316,11 @@ public class DatanodeManager { - // Block invalidate limit also has some dependency on heartbeat interval. - // Check setBlockInvalidateLimit(). - setBlockInvalidateLimit(configuredBlockInvalidateLimit); -+ this.allowRegistrationAddressOverride = conf.getBoolean( -+ DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_UNSAFE_ALLOW_ADDRESS_OVERRIDE_KEY, -+ DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_UNSAFE_ALLOW_ADDRESS_OVERRIDE_DEFAULT); -+ LOG.info(DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_UNSAFE_ALLOW_ADDRESS_OVERRIDE_KEY -+ + "=" + allowRegistrationAddressOverride); - this.checkIpHostnameInRegistration = conf.getBoolean( - DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY, - DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT); -@@ -1158,27 +1165,29 @@ public class DatanodeManager { - */ - public void registerDatanode(DatanodeRegistration nodeReg) - throws DisallowedDatanodeException, UnresolvedTopologyException { -- InetAddress dnAddress = Server.getRemoteIp(); -- if (dnAddress != null) { -- // Mostly called inside an RPC, update ip and peer hostname -- String hostname = dnAddress.getHostName(); -- String ip = dnAddress.getHostAddress(); -- if (checkIpHostnameInRegistration && !isNameResolved(dnAddress)) { -- // Reject registration of unresolved datanode to prevent performance -- // impact of repetitive DNS lookups later. -- final String message = "hostname cannot be resolved (ip=" -- + ip + ", hostname=" + hostname + ")"; -- LOG.warn("Unresolved datanode registration: " + message); -- throw new DisallowedDatanodeException(nodeReg, message); -+ if (!allowRegistrationAddressOverride) { -+ InetAddress dnAddress = Server.getRemoteIp(); -+ if (dnAddress != null) { -+ // Mostly called inside an RPC, update ip and peer hostname -+ String hostname = dnAddress.getHostName(); -+ String ip = dnAddress.getHostAddress(); -+ if (checkIpHostnameInRegistration && !isNameResolved(dnAddress)) { -+ // Reject registration of unresolved datanode to prevent performance -+ // impact of repetitive DNS lookups later. -+ final String message = "hostname cannot be resolved (ip=" -+ + ip + ", hostname=" + hostname + ")"; -+ LOG.warn("Unresolved datanode registration: " + message); -+ throw new DisallowedDatanodeException(nodeReg, message); -+ } -+ // update node registration with the ip and hostname from rpc request -+ nodeReg.setIpAddr(ip); -+ nodeReg.setPeerHostName(hostname); - } -- // update node registration with the ip and hostname from rpc request -- nodeReg.setIpAddr(ip); -- nodeReg.setPeerHostName(hostname); - } -- -+ - try { - nodeReg.setExportedKeys(blockManager.getBlockKeys()); -- -+ - // Checks if the node is not on the hosts list. If it is not, then - // it will be disallowed from registering. - if (!hostConfigManager.isIncluded(nodeReg)) { -diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java -index 21b92db307..5d3437239c 100644 ---- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java -+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java -@@ -101,6 +101,11 @@ public class DNConf { - final boolean syncOnClose; - final boolean encryptDataTransfer; - final boolean connectToDnViaHostname; -+ private final String registeredHostname; -+ private final int registeredDataPort; -+ private final int registeredHttpPort; -+ private final int registeredHttpsPort; -+ private final int registeredIpcPort; - final boolean overwriteDownstreamDerivedQOP; - private final boolean pmemCacheRecoveryEnabled; - -@@ -189,6 +194,11 @@ public class DNConf { - connectToDnViaHostname = getConf().getBoolean( - DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, - DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT); -+ registeredHostname = getConf().get(DFSConfigKeys.DFS_DATANODE_REGISTERED_HOSTNAME); -+ registeredDataPort = getConf().getInt(DFSConfigKeys.DFS_DATANODE_REGISTERED_DATA_PORT, -1); -+ registeredHttpPort = getConf().getInt(DFSConfigKeys.DFS_DATANODE_REGISTERED_HTTP_PORT, -1); -+ registeredHttpsPort = getConf().getInt(DFSConfigKeys.DFS_DATANODE_REGISTERED_HTTPS_PORT, -1); -+ registeredIpcPort = getConf().getInt(DFSConfigKeys.DFS_DATANODE_REGISTERED_IPC_PORT, -1); - this.blockReportInterval = getConf().getLong( - DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, - DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT); -@@ -363,6 +373,66 @@ public class DNConf { - return connectToDnViaHostname; - } - -+ /** -+ * Returns a hostname to register with the cluster instead of the system -+ * hostname. -+ * This is an expert setting and can be used in multihoming scenarios to -+ * override the detected hostname. -+ * -+ * @return null if the system hostname should be used, otherwise a hostname -+ */ -+ public String getRegisteredHostname() { -+ return registeredHostname; -+ } -+ -+ /** -+ * Returns a port number to register with the cluster instead of the -+ * data port that the node is listening on. -+ * This is an expert setting and can be used in multihoming scenarios to -+ * override the detected port. -+ * -+ * @return -1 if the actual port should be used, otherwise a port number -+ */ -+ public int getRegisteredDataPort() { -+ return registeredDataPort; -+ } -+ -+ /** -+ * Returns a port number to register with the cluster instead of the -+ * HTTP port that the node is listening on. -+ * This is an expert setting and can be used in multihoming scenarios to -+ * override the detected port. -+ * -+ * @return -1 if the actual port should be used, otherwise a port number -+ */ -+ public int getRegisteredHttpPort() { -+ return registeredHttpPort; -+ } -+ -+ /** -+ * Returns a port number to register with the cluster instead of the -+ * HTTPS port that the node is listening on. -+ * This is an expert setting and can be used in multihoming scenarios to -+ * override the detected port. -+ * -+ * @return -1 if the actual port should be used, otherwise a port number -+ */ -+ public int getRegisteredHttpsPort() { -+ return registeredHttpsPort; -+ } -+ -+ /** -+ * Returns a port number to register with the cluster instead of the -+ * IPC port that the node is listening on. -+ * This is an expert setting and can be used in multihoming scenarios to -+ * override the detected port. -+ * -+ * @return -1 if the actual port should be used, otherwise a port number -+ */ -+ public int getRegisteredIpcPort() { -+ return registeredIpcPort; -+ } -+ - /** - * Returns socket timeout - * -diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java -index 956f5bbe51..22ae127d98 100644 ---- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java -+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java -@@ -135,6 +135,7 @@ import java.util.HashSet; - import java.util.Iterator; - import java.util.List; - import java.util.Map; -+import java.util.Optional; - import java.util.Map.Entry; - import java.util.Set; - import java.util.UUID; -@@ -2076,11 +2077,35 @@ public class DataNode extends ReconfigurableBase - NodeType.DATA_NODE); - } - -- DatanodeID dnId = new DatanodeID( -- streamingAddr.getAddress().getHostAddress(), hostName, -- storage.getDatanodeUuid(), getXferPort(), getInfoPort(), -- infoSecurePort, getIpcPort()); -- return new DatanodeRegistration(dnId, storageInfo, -+ String registeredHostname = Optional -+ .ofNullable(dnConf.getRegisteredHostname()) -+ .orElseGet(() -> streamingAddr.getAddress().getHostAddress()); -+ int registeredDataPort = dnConf.getRegisteredDataPort(); -+ if (registeredDataPort == -1) { -+ registeredDataPort = getXferPort(); -+ } -+ int registeredHttpPort = dnConf.getRegisteredHttpPort(); -+ if (registeredHttpPort == -1) { -+ registeredHttpPort = getInfoPort(); -+ } -+ int registeredHttpsPort = dnConf.getRegisteredHttpsPort(); -+ if (registeredHttpsPort == -1) { -+ registeredHttpsPort = getInfoSecurePort(); -+ } -+ int registeredIpcPort = dnConf.getRegisteredIpcPort(); -+ if (registeredIpcPort == -1) { -+ registeredIpcPort = getIpcPort(); -+ } -+ -+ DatanodeID dnId = new DatanodeID(registeredHostname, -+ registeredHostname, -+ storage.getDatanodeUuid(), -+ registeredDataPort, -+ registeredHttpPort, -+ registeredHttpsPort, -+ registeredIpcPort); -+ -+ return new DatanodeRegistration(dnId, storageInfo, - new ExportedBlockKeys(), VersionInfo.getVersion()); - } - diff --git a/hadoop/hadoop/stackable/patches/3.4.2/0003-Async-profiler-also-grab-itimer-events.patch b/hadoop/hadoop/stackable/patches/3.4.2/0003-Async-profiler-also-grab-itimer-events.patch deleted file mode 100644 index 5191b1465..000000000 --- a/hadoop/hadoop/stackable/patches/3.4.2/0003-Async-profiler-also-grab-itimer-events.patch +++ /dev/null @@ -1,29 +0,0 @@ -From ab9550bd7b71c16c381a105a22732f6e71f2dba6 Mon Sep 17 00:00:00 2001 -From: xeniape -Date: Wed, 10 Sep 2025 14:39:20 +0200 -Subject: Async-profiler-also-grab-itimer-events - ---- - .../src/main/java/org/apache/hadoop/http/ProfileServlet.java | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java -index ce53274151..909892ff90 100644 ---- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java -+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/ProfileServlet.java -@@ -76,6 +76,7 @@ import org.apache.hadoop.util.ProcessUtils; - * Following event types are supported (default is 'cpu') (NOTE: not all OS'es support all events) - * // Perf events: - * // cpu -+ * // itimer - * // page-faults - * // context-switches - * // cycles -@@ -118,6 +119,7 @@ public class ProfileServlet extends HttpServlet { - private enum Event { - - CPU("cpu"), -+ ITIMER("itimer"), - ALLOC("alloc"), - LOCK("lock"), - PAGE_FAULTS("page-faults"), diff --git a/hadoop/hadoop/stackable/patches/3.4.2/0004-HDFS-17378-Fix-missing-operationType-for-some-operat.patch b/hadoop/hadoop/stackable/patches/3.4.2/0004-HDFS-17378-Fix-missing-operationType-for-some-operat.patch deleted file mode 100644 index f210f7c62..000000000 --- a/hadoop/hadoop/stackable/patches/3.4.2/0004-HDFS-17378-Fix-missing-operationType-for-some-operat.patch +++ /dev/null @@ -1,199 +0,0 @@ -From 2b131b13ad062695d5de8840be744900fa4a71c1 Mon Sep 17 00:00:00 2001 -From: xeniape -Date: Wed, 10 Sep 2025 14:40:41 +0200 -Subject: HDFS-17378-Fix-missing-operationType-for-some-operat - ---- - .../hdfs/server/namenode/FSNamesystem.java | 41 +++++++++++-------- - 1 file changed, 24 insertions(+), 17 deletions(-) - -diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java -index cfc7f24381..e9c5fc0da2 100644 ---- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java -+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java -@@ -2618,15 +2618,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - * @throws IOException - */ - BlockStoragePolicy getStoragePolicy(String src) throws IOException { -+ final String operationName = "getStoragePolicy"; - checkOperation(OperationCategory.READ); - final FSPermissionChecker pc = getPermissionChecker(); -- FSPermissionChecker.setOperationType(null); -+ FSPermissionChecker.setOperationType(operationName); - readLock(); - try { - checkOperation(OperationCategory.READ); - return FSDirAttrOp.getStoragePolicy(dir, pc, blockManager, src); - } finally { -- readUnlock("getStoragePolicy"); -+ readUnlock(operationName); - } - } - -@@ -2646,15 +2647,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - } - - long getPreferredBlockSize(String src) throws IOException { -+ final String operationName = "getPreferredBlockSize"; - checkOperation(OperationCategory.READ); - final FSPermissionChecker pc = getPermissionChecker(); -- FSPermissionChecker.setOperationType(null); -+ FSPermissionChecker.setOperationType(operationName); - readLock(); - try { - checkOperation(OperationCategory.READ); - return FSDirAttrOp.getPreferredBlockSize(dir, pc, src); - } finally { -- readUnlock("getPreferredBlockSize"); -+ readUnlock(operationName); - } - } - -@@ -2707,7 +2709,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - boolean createParent, short replication, long blockSize, - CryptoProtocolVersion[] supportedVersions, String ecPolicyName, - String storagePolicy, boolean logRetryCache) throws IOException { -- - HdfsFileStatus status; - try { - status = startFileInt(src, permissions, holder, clientMachine, flag, -@@ -2727,6 +2728,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - long blockSize, CryptoProtocolVersion[] supportedVersions, - String ecPolicyName, String storagePolicy, boolean logRetryCache) - throws IOException { -+ final String operationName = "create"; - if (NameNode.stateChangeLog.isDebugEnabled()) { - StringBuilder builder = new StringBuilder(); - builder.append("DIR* NameSystem.startFile: src=").append(src) -@@ -2764,7 +2766,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - - checkOperation(OperationCategory.WRITE); - final FSPermissionChecker pc = getPermissionChecker(); -- FSPermissionChecker.setOperationType(null); -+ FSPermissionChecker.setOperationType(operationName); - writeLock(); - try { - checkOperation(OperationCategory.WRITE); -@@ -2827,7 +2829,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - dir.writeUnlock(); - } - } finally { -- writeUnlock("create", getLockReportInfoSupplier(src, null, stat)); -+ writeUnlock(operationName, getLockReportInfoSupplier(src, null, stat)); - // There might be transactions logged while trying to recover the lease. - // They need to be sync'ed even when an exception was thrown. - if (!skipSync) { -@@ -2856,10 +2858,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - */ - boolean recoverLease(String src, String holder, String clientMachine) - throws IOException { -+ final String operationName = "recoverLease"; - boolean skipSync = false; - checkOperation(OperationCategory.WRITE); - final FSPermissionChecker pc = getPermissionChecker(); -- FSPermissionChecker.setOperationType(null); -+ FSPermissionChecker.setOperationType(operationName); - writeLock(); - try { - checkOperation(OperationCategory.WRITE); -@@ -2880,7 +2883,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - skipSync = true; - throw se; - } finally { -- writeUnlock("recoverLease"); -+ writeUnlock(operationName); - // There might be transactions logged while trying to recover the lease. - // They need to be sync'ed even when an exception was thrown. - if (!skipSync) { -@@ -3096,6 +3099,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - final Set excludes, - final int numAdditionalNodes, final String clientName - ) throws IOException { -+ final String operationName = "getAdditionalDatanode"; - //check if the feature is enabled - dtpReplaceDatanodeOnFailure.checkEnabled(); - -@@ -3107,7 +3111,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - final BlockType blockType; - checkOperation(OperationCategory.WRITE); - final FSPermissionChecker pc = getPermissionChecker(); -- FSPermissionChecker.setOperationType(null); -+ FSPermissionChecker.setOperationType(operationName); - readLock(); - try { - // Changing this operation category to WRITE instead of making getAdditionalDatanode as a -@@ -3133,7 +3137,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - "src=%s, fileId=%d, blk=%s, clientName=%s, clientMachine=%s", - src, fileId, blk, clientName, clientMachine)); - } finally { -- readUnlock("getAdditionalDatanode"); -+ readUnlock(operationName); - } - - if (clientnode == null) { -@@ -3155,10 +3159,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - */ - void abandonBlock(ExtendedBlock b, long fileId, String src, String holder) - throws IOException { -+ final String operationName = "abandonBlock"; - NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: {} of file {}", b, src); - checkOperation(OperationCategory.WRITE); - final FSPermissionChecker pc = getPermissionChecker(); -- FSPermissionChecker.setOperationType(null); -+ FSPermissionChecker.setOperationType(operationName); - writeLock(); - try { - checkOperation(OperationCategory.WRITE); -@@ -3167,7 +3172,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - NameNode.stateChangeLog.debug( - "BLOCK* NameSystem.abandonBlock: {} is removed from pendingCreates", b); - } finally { -- writeUnlock("abandonBlock"); -+ writeUnlock(operationName); - } - getEditLog().logSync(); - } -@@ -3221,10 +3226,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - boolean completeFile(final String src, String holder, - ExtendedBlock last, long fileId) - throws IOException { -+ final String operationName = "completeFile"; - boolean success = false; - checkOperation(OperationCategory.WRITE); - final FSPermissionChecker pc = getPermissionChecker(); -- FSPermissionChecker.setOperationType(null); -+ FSPermissionChecker.setOperationType(operationName); - writeLock(); - try { - checkOperation(OperationCategory.WRITE); -@@ -3232,7 +3238,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - success = FSDirWriteFileOp.completeFile(this, pc, src, holder, last, - fileId); - } finally { -- writeUnlock("completeFile"); -+ writeUnlock(operationName); - } - getEditLog().logSync(); - if (success) { -@@ -3666,10 +3672,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - */ - void fsync(String src, long fileId, String clientName, long lastBlockLength) - throws IOException { -+ final String operationName = "fsync"; - NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName); - checkOperation(OperationCategory.WRITE); - final FSPermissionChecker pc = getPermissionChecker(); -- FSPermissionChecker.setOperationType(null); -+ FSPermissionChecker.setOperationType(operationName); - writeLock(); - try { - checkOperation(OperationCategory.WRITE); -@@ -3683,7 +3690,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - } - FSDirWriteFileOp.persistBlocks(dir, src, pendingFile, false); - } finally { -- writeUnlock("fsync"); -+ writeUnlock(operationName); - } - getEditLog().logSync(); - } diff --git a/hadoop/hadoop/stackable/patches/3.4.2/0005-Update-CycloneDX-plugin.patch b/hadoop/hadoop/stackable/patches/3.4.2/0005-Update-CycloneDX-plugin.patch deleted file mode 100644 index 39dd0f143..000000000 --- a/hadoop/hadoop/stackable/patches/3.4.2/0005-Update-CycloneDX-plugin.patch +++ /dev/null @@ -1,44 +0,0 @@ -From 18a413bfaec1acd447b070538be994ea9691899a Mon Sep 17 00:00:00 2001 -From: xeniape -Date: Wed, 10 Sep 2025 14:41:18 +0200 -Subject: Update-CycloneDX-plugin - ---- - pom.xml | 10 ++++++---- - 1 file changed, 6 insertions(+), 4 deletions(-) - -diff --git a/pom.xml b/pom.xml -index 9a9a0de744..c7838e3674 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -121,7 +121,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x - 4.2.0 - 1.1.1 - 3.10.1 -- 2.7.10 -+ 2.8.0 - - bash - -@@ -773,6 +773,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x - org.cyclonedx - cyclonedx-maven-plugin - ${cyclonedx.version} -+ -+ application -+ 1.5 -+ false -+ - - - package -@@ -781,9 +786,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x - - - -- -- xml -- - - - diff --git a/hadoop/hadoop/stackable/patches/3.4.2/0006-HADOOP-19352.-Hadoop-OSS-Connector-adds-support-for.patch b/hadoop/hadoop/stackable/patches/3.4.2/0006-HADOOP-19352.-Hadoop-OSS-Connector-adds-support-for.patch deleted file mode 100644 index f8e486155..000000000 --- a/hadoop/hadoop/stackable/patches/3.4.2/0006-HADOOP-19352.-Hadoop-OSS-Connector-adds-support-for.patch +++ /dev/null @@ -1,245 +0,0 @@ -From e67476ea801e929feb3d89d023d915efbf82336f Mon Sep 17 00:00:00 2001 -From: xeniape -Date: Wed, 10 Sep 2025 14:41:46 +0200 -Subject: HADOOP-19352.-Hadoop-OSS-Connector-adds-support-for- - ---- - hadoop-project/pom.xml | 2 +- - hadoop-tools/hadoop-aliyun/pom.xml | 26 +++++ - .../aliyun/oss/AliyunOSSFileSystemStore.java | 16 +++ - .../hadoop/fs/aliyun/oss/Constants.java | 15 +++ - .../fs/aliyun/oss/ITAliyunOSSSignatureV4.java | 98 +++++++++++++++++++ - .../src/test/resources/log4j.properties | 3 + - 6 files changed, 159 insertions(+), 1 deletion(-) - create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java - -diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml -index 70f64bf55c..6d05a218d0 100644 ---- a/hadoop-project/pom.xml -+++ b/hadoop-project/pom.xml -@@ -1663,7 +1663,7 @@ - - com.aliyun.oss - aliyun-sdk-oss -- 3.13.2 -+ 3.18.1 - - - org.apache.httpcomponents -diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml -index c2e04623ad..5b12e4a4ef 100644 ---- a/hadoop-tools/hadoop-aliyun/pom.xml -+++ b/hadoop-tools/hadoop-aliyun/pom.xml -@@ -165,5 +165,31 @@ - test - jar - -+ -+ -+ org.junit.jupiter -+ junit-jupiter-api -+ test -+ -+ -+ org.junit.jupiter -+ junit-jupiter-engine -+ test -+ -+ -+ org.junit.jupiter -+ junit-jupiter-params -+ test -+ -+ -+ org.junit.platform -+ junit-platform-launcher -+ test -+ -+ -+ org.junit.vintage -+ junit-vintage-engine -+ test -+ - - -diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java -index ccd5d1ea25..30da259fd5 100644 ---- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java -+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java -@@ -73,6 +73,7 @@ import java.util.List; - import java.util.ListIterator; - import java.util.NoSuchElementException; - import java.util.stream.Collectors; -+import com.aliyun.oss.common.comm.SignVersion; - - import static org.apache.hadoop.fs.aliyun.oss.Constants.*; - -@@ -113,6 +114,16 @@ public class AliyunOSSFileSystemStore { - conf.get(USER_AGENT_PREFIX, USER_AGENT_PREFIX_DEFAULT) + ", Hadoop/" - + VersionInfo.getVersion()); - -+ String region = conf.get(REGION_KEY, ""); -+ String signatureVersion = conf.get(SIGNATURE_VERSION_KEY, SIGNATURE_VERSION_DEFAULT); -+ if ("V4".equalsIgnoreCase(signatureVersion)) { -+ clientConf.setSignatureVersion(SignVersion.V4); -+ if (StringUtils.isEmpty(region)) { -+ LOG.error("Signature version is V4 ,but region is empty."); -+ throw new IOException("SignVersion is V4 but region is empty"); -+ } -+ } -+ - String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, ""); - int proxyPort = conf.getInt(PROXY_PORT_KEY, -1); - if (StringUtils.isNotEmpty(proxyHost)) { -@@ -171,6 +182,11 @@ public class AliyunOSSFileSystemStore { - statistics.incrementWriteOps(1); - } - -+ if (StringUtils.isNotEmpty(region)) { -+ ossClient.setRegion(region); -+ LOG.debug("ossClient setRegion {}", region); -+ } -+ - maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT); - int listVersion = conf.getInt(LIST_VERSION, DEFAULT_LIST_VERSION); - if (listVersion < 1 || listVersion > 2) { -diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java -index baeb919937..176669ed15 100644 ---- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java -+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java -@@ -211,4 +211,19 @@ public final class Constants { - public static final String LIST_VERSION = "fs.oss.list.version"; - - public static final int DEFAULT_LIST_VERSION = 2; -+ -+ /** -+ * OSS signature version. -+ */ -+ public static final String SIGNATURE_VERSION_KEY = "fs.oss.signatureversion"; -+ -+ /** -+ * OSS signature version DEFAULT {@value}. -+ */ -+ public static final String SIGNATURE_VERSION_DEFAULT = "V1"; -+ -+ /** -+ * OSS region {@value}. -+ */ -+ public static final String REGION_KEY = "fs.oss.region"; - } -diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java -new file mode 100644 -index 0000000000..5070f2a581 ---- /dev/null -+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java -@@ -0,0 +1,98 @@ -+/** -+ * Licensed to the Apache Software Foundation (ASF) under one -+ * or more contributor license agreements. See the NOTICE file -+ * distributed with this work for additional information -+ * regarding copyright ownership. The ASF licenses this file -+ * to you under the Apache License, Version 2.0 (the -+ * "License"); you may not use this file except in compliance -+ * with the License. You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. -+ */ -+ -+package org.apache.hadoop.fs.aliyun.oss; -+ -+import org.apache.hadoop.conf.Configuration; -+import org.apache.hadoop.fs.FileStatus; -+import org.apache.hadoop.fs.Path; -+import org.junit.Before; -+import org.junit.Test; -+import org.slf4j.Logger; -+import org.slf4j.LoggerFactory; -+ -+import java.io.IOException; -+import java.net.URI; -+ -+import static org.apache.hadoop.fs.aliyun.oss.Constants.REGION_KEY; -+import static org.apache.hadoop.fs.aliyun.oss.Constants.SIGNATURE_VERSION_KEY; -+import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; -+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; -+import static org.junit.Assert.*; -+import static org.junit.Assume.assumeNotNull; -+ -+/** -+ * Tests Aliyun OSS system. -+ */ -+public class ITAliyunOSSSignatureV4 { -+ private static final Logger LOG = LoggerFactory.getLogger(ITAliyunOSSSignatureV4.class); -+ private Configuration conf; -+ private URI testURI; -+ private Path testFile = new Path("ITAliyunOSSSignatureV4/atestr"); -+ -+ @Before -+ public void setUp() throws Exception { -+ conf = new Configuration(); -+ String bucketUri = conf.get("test.fs.oss.name"); -+ LOG.debug("bucketUri={}", bucketUri); -+ testURI = URI.create(bucketUri); -+ } -+ -+ @Test -+ public void testV4() throws IOException { -+ conf.set(SIGNATURE_VERSION_KEY, "V4"); -+ conf.set(REGION_KEY, "cn-hongkong"); -+ AliyunOSSFileSystem fs = new AliyunOSSFileSystem(); -+ fs.initialize(testURI, conf); -+ assumeNotNull(fs); -+ -+ createFile(fs, testFile, true, dataset(256, 0, 255)); -+ FileStatus status = fs.getFileStatus(testFile); -+ fs.delete(testFile); -+ fs.close(); -+ } -+ -+ @Test -+ public void testDefaultSignatureVersion() throws IOException { -+ AliyunOSSFileSystem fs = new AliyunOSSFileSystem(); -+ fs.initialize(testURI, conf); -+ assumeNotNull(fs); -+ -+ Path testFile2 = new Path("/test/atestr"); -+ createFile(fs, testFile2, true, dataset(256, 0, 255)); -+ FileStatus status = fs.getFileStatus(testFile2); -+ fs.delete(testFile2); -+ fs.close(); -+ } -+ -+ @Test -+ public void testV4WithoutRegion() throws IOException { -+ conf.set(SIGNATURE_VERSION_KEY, "V4"); -+ AliyunOSSFileSystem fs = new AliyunOSSFileSystem(); -+ IOException expectedException = null; -+ try { -+ fs.initialize(testURI, conf); -+ } catch (IOException e) { -+ LOG.warn("use V4 , but do not set region, get exception={}", e); -+ expectedException = e; -+ assertEquals("use V4 , but do not set region", e.getMessage(), -+ "SignVersion is V4 but region is empty"); -+ } -+ assertNotNull(expectedException); -+ } -+} -diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties -index bb5cbe5ec3..2167f68811 100644 ---- a/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties -+++ b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties -@@ -21,3 +21,6 @@ log4j.threshold=ALL - log4j.appender.stdout=org.apache.log4j.ConsoleAppender - log4j.appender.stdout.layout=org.apache.log4j.PatternLayout - log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n -+ -+# Log all oss classes -+log4j.logger.org.apache.hadoop.fs.aliyun.oss=DEBUG -\ No newline at end of file diff --git a/hadoop/hadoop/stackable/patches/3.4.2/0007-Upgrade-nimbus-jose-jwt-to-9.37.4-to-fix-CVE-2025-53.patch b/hadoop/hadoop/stackable/patches/3.4.2/0007-Upgrade-nimbus-jose-jwt-to-9.37.4-to-fix-CVE-2025-53.patch deleted file mode 100644 index c6bc3e1ed..000000000 --- a/hadoop/hadoop/stackable/patches/3.4.2/0007-Upgrade-nimbus-jose-jwt-to-9.37.4-to-fix-CVE-2025-53.patch +++ /dev/null @@ -1,37 +0,0 @@ -From 95beed1d7436d51e4c8655be107e6c527a1c0eaa Mon Sep 17 00:00:00 2001 -From: xeniape -Date: Thu, 11 Sep 2025 12:12:12 +0200 -Subject: Upgrade-nimbus-jose-jwt-to-9.37.4-to-fix-CVE-2025-53864, Upstream - reference: https://github.com/apache/hadoop/pull/7870 - ---- - LICENSE-binary | 2 +- - hadoop-project/pom.xml | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/LICENSE-binary b/LICENSE-binary -index 3ad4b30045..07f4e5a45f 100644 ---- a/LICENSE-binary -+++ b/LICENSE-binary -@@ -240,7 +240,7 @@ com.google.guava:guava:20.0 - com.google.guava:guava:32.0.1-jre - com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava - com.microsoft.azure:azure-storage:7.0.0 --com.nimbusds:nimbus-jose-jwt:9.37.2 -+com.nimbusds:nimbus-jose-jwt:9.37.4 - com.zaxxer:HikariCP:4.0.3 - commons-beanutils:commons-beanutils:1.9.4 - commons-cli:commons-cli:1.9.0 -diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml -index 6d05a218d0..2e21ff7394 100644 ---- a/hadoop-project/pom.xml -+++ b/hadoop-project/pom.xml -@@ -233,7 +233,7 @@ - 2.1.4.Final - 1.0.2 - 5.4.0 -- 9.37.2 -+ 9.37.4 - v14.17.0 - v1.22.5 - 1.10.13 diff --git a/hadoop/hadoop/stackable/patches/3.4.2/0008-YARN-11873-Update-nodejs-to-LTS-version.patch b/hadoop/hadoop/stackable/patches/3.4.2/0008-YARN-11873-Update-nodejs-to-LTS-version.patch deleted file mode 100644 index f70f2de50..000000000 --- a/hadoop/hadoop/stackable/patches/3.4.2/0008-YARN-11873-Update-nodejs-to-LTS-version.patch +++ /dev/null @@ -1,22 +0,0 @@ -From 08a7206a29f212e1f0e3bd81e0cb0be7907907a4 Mon Sep 17 00:00:00 2001 -From: Andrew Kenworthy -Date: Wed, 1 Oct 2025 17:28:41 +0200 -Subject: Update nodejs to LTS version - ---- - hadoop-project/pom.xml | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml -index 2e21ff7394..a7d5faf760 100644 ---- a/hadoop-project/pom.xml -+++ b/hadoop-project/pom.xml -@@ -234,7 +234,7 @@ - 1.0.2 - 5.4.0 - 9.37.4 -- v14.17.0 -+ v22.20.0 - v1.22.5 - 1.10.13 - 1.20 diff --git a/hadoop/hadoop/stackable/patches/3.4.2/0009-HDFS-17891-fix-for-hostname-resolution-bug-with-data.patch b/hadoop/hadoop/stackable/patches/3.4.2/0009-HDFS-17891-fix-for-hostname-resolution-bug-with-data.patch deleted file mode 100644 index 49fcd1c26..000000000 --- a/hadoop/hadoop/stackable/patches/3.4.2/0009-HDFS-17891-fix-for-hostname-resolution-bug-with-data.patch +++ /dev/null @@ -1,48 +0,0 @@ -From cb5487dcd3e6e170014604478a3d8cef03285007 Mon Sep 17 00:00:00 2001 -From: Jim Halfpenny -Date: Mon, 16 Mar 2026 09:18:46 +0000 -Subject: HDFS-17891 fix for hostname resolution bug with datanodes - ---- - .../hadoop/hdfs/server/blockmanagement/HostSet.java | 11 +++++++++-- - 1 file changed, 9 insertions(+), 2 deletions(-) - -diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java -index d12e5fbae1..3da753f1cb 100644 ---- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java -+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostSet.java -@@ -18,10 +18,11 @@ - package org.apache.hadoop.hdfs.server.blockmanagement; - - --import org.apache.hadoop.util.Preconditions; - import org.apache.hadoop.thirdparty.com.google.common.collect.HashMultimap; - import org.apache.hadoop.thirdparty.com.google.common.collect.Multimap; - import org.apache.hadoop.thirdparty.com.google.common.collect.UnmodifiableIterator; -+import org.slf4j.Logger; -+import org.slf4j.LoggerFactory; - - import java.net.InetAddress; - import java.net.InetSocketAddress; -@@ -38,6 +39,8 @@ - * .getPort() || B.getPort() == 0. - */ - public class HostSet implements Iterable { -+ private static final Logger LOG = LoggerFactory.getLogger(HostSet.class); -+ - // Host -> lists of ports - private final Multimap addrs = HashMultimap.create(); - -@@ -72,7 +75,11 @@ int size() { - } - - void add(InetSocketAddress addr) { -- Preconditions.checkArgument(!addr.isUnresolved()); -+ LOG.debug("Adding address to HostSet: {}", addr); -+ if (addr.isUnresolved()) { -+ LOG.warn("Unresolved address not added to HostSet: {}", addr); -+ return; -+ } - addrs.put(addr.getAddress(), addr.getPort()); - } - diff --git a/hadoop/hadoop/stackable/patches/3.4.2/patchable.toml b/hadoop/hadoop/stackable/patches/3.4.2/patchable.toml deleted file mode 100644 index 07e7ae331..000000000 --- a/hadoop/hadoop/stackable/patches/3.4.2/patchable.toml +++ /dev/null @@ -1,2 +0,0 @@ -mirror = "https://github.com/stackabletech/hadoop.git" -base = "84e8b89ee2ebe6923691205b9e171badde7a495c"