From c17652b0920ad40a068aebd14b7f7344c10ff069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chip=20Wolf=20=E2=80=AE?= Date: Thu, 23 Apr 2026 16:47:37 +0000 Subject: [PATCH 1/3] feat: add forced-host label for Velocity forced hosts support Add a new 'forced-host' label/annotation that can be set on Pods and ReplicaSets to configure Velocity's forced hosts functionality. Players connecting via a specific hostname will be routed to the server with the matching forced-host label. See: https://docs.papermc.io/velocity/configuration/\#forced-hosts-section --- .gitignore | 4 +- README-JP.md | 17 ++++++--- README.md | 5 +++ .../azisaba/kuvel/KuvelServiceHandler.java | 38 ++++++++++++++++++- .../redis/RedisLoadBalancerDiscovery.java | 15 +++++++- .../listener/ChooseInitialServerListener.java | 16 ++++++++ .../kuvel/loadbalancer/LoadBalancer.java | 1 + .../kuvel/redis/RedisConnectionLeader.java | 4 +- .../azisaba/kuvel/redis/RedisSubscriber.java | 7 +++- .../net/azisaba/kuvel/util/LabelKeys.java | 1 + 10 files changed, 95 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 39c68bf9..98cd7dad 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,6 @@ buildNumber.properties # JDT-specific (Eclipse Java Development Tools) .classpath -# End of https://www.toptal.com/developers/gitignore/api/maven,intellij+all \ No newline at end of file +# End of https://www.toptal.com/developers/gitignore/api/maven,intellij+all + +.vscode/ diff --git a/README-JP.md b/README-JP.md index 3bb82a62..36acf79c 100644 --- a/README-JP.md +++ b/README-JP.md @@ -85,16 +85,18 @@ labelはconfigで指定します。デフォルトでは、`kuvel.azisaba.net/en また、一部の機能を使用するために以下のlabelを使用します。 -| Label名 | 値 | -|:---------------------------------------:|:-------------------:| -| kuvel.azisaba.net/preferred-server-name | Velocityに登録したいサーバー名 | -| kuvel.azisaba.net/initial-server | true / false | -| kuvel.azisaba.net/disable-name-suffix | true / false | -| kuvel.azisaba.net/disable-load-balancer | true / false | +| Label名 | 値 | +|:---------------------------------------:|:----------------------------:| +| kuvel.azisaba.net/preferred-server-name | Velocityに登録したいサーバー名 | +| kuvel.azisaba.net/initial-server | true / false | +| kuvel.azisaba.net/disable-name-suffix | true / false | +| kuvel.azisaba.net/disable-load-balancer | true / false | +| kuvel.azisaba.net/forced-host | このサーバーにルーティングするホスト名 (forced hosts) | サーバー名が63文字を超える場合は、ラベルではなく`kuvel.azisaba.net/preferred-server-name`のアノテーションを使用してください。 Podに`kuvel.azisaba.net/disable-name-suffix=true`を設定すると、`preferred-server-name`をそのまま登録します (`-1`などのsuffixを付与しません)。複数レプリカが検出された場合はエラーを出して追加Podを登録しません。単一レプリカのPod向けの設定です。 ReplicaSetまたはDeploymentのmetadataに`kuvel.azisaba.net/disable-load-balancer=true`を設定すると、そのReplicaSet用の仮想LoadBalancerサーバーは作成されません。配下のPod自体は通常どおり登録されます。 +Pod、Deployment、またはReplicaSetに`kuvel.azisaba.net/forced-host`を設定すると、そのホスト名で接続したプレイヤーは対応するサーバーにルーティングされます。これはVelocityの[forced hosts](https://docs.papermc.io/velocity/configuration/#forced-hosts-section)機能をKubernetesのラベル/アノテーションで動的に設定するものです。 ### Podの場合 @@ -108,6 +110,7 @@ metadata: kuvel.azisaba.net/preferred-server-name: "test-server" # Kuvelがサーバーの命名をするために必要 # kuvel.azisaba.net/initial-server: "true" # 初期サーバーにする場合はコメントアウトを外す # kuvel.azisaba.net/disable-name-suffix: "true" # preferred-server-nameをそのまま登録する (単一レプリカのみ) + # kuvel.azisaba.net/forced-host: "pvp.example.com" # pvp.example.comで接続したプレイヤーをこのサーバーに送る spec: containers: - name: test-server @@ -134,6 +137,7 @@ spec: kuvel.azisaba.net/enable-server-discovery: "true" # KuvelがMinecraftサーバーを見つけるために必要 (Configに依存) kuvel.azisaba.net/preferred-server-name: "test-server" # Kuvelがサーバーの命名をするために必要 # kuvel.azisaba.net/initial-server: "true" # 初期サーバーにする場合はコメントアウトを外す + # kuvel.azisaba.net/forced-host: "pvp.example.com" # pvp.example.comで接続したプレイヤーをこのサーバーに送る spec: containers: - name: test-server @@ -159,6 +163,7 @@ metadata: kuvel.azisaba.net/preferred-server-name: "lobby" # kuvel.azisaba.net/initial-server: "true" # このロードバランサーを初期サーバーにする場合はコメントアウトを外す # kuvel.azisaba.net/disable-load-balancer: "true" # LoadBalancerエントリだけ無効にする場合はコメントアウトを外す + # kuvel.azisaba.net/forced-host: "lobby.example.com" # lobby.example.comで接続したプレイヤーをこのロードバランサーに送る spec: replicas: 3 selector: diff --git a/README.md b/README.md index 5936558b..d8522a87 100644 --- a/README.md +++ b/README.md @@ -89,10 +89,12 @@ The following labels are also used for some other features. | kuvel.azisaba.net/initial-server | true / false | | kuvel.azisaba.net/disable-name-suffix | true / false | | kuvel.azisaba.net/disable-load-balancer | true / false | +| kuvel.azisaba.net/forced-host | Hostname to route to this server (forced hosts) | If server names longer than 63 characters are desired, the `kuvel.azisaba.net/preferred-server-name` annotation can be used instead of the label. If `kuvel.azisaba.net/disable-name-suffix=true` is set on a Pod, Kuvel will register the server name exactly as `preferred-server-name` (no `-1` suffix). If multiple replicas are detected, Kuvel logs an error and skips the extra pods. This label is intended for single-replica Pods only. If `kuvel.azisaba.net/disable-load-balancer=true` is set on a ReplicaSet or Deployment metadata, Kuvel will skip creating the virtual load balancer server for that ReplicaSet. The backing Pods are still registered normally. +If `kuvel.azisaba.net/forced-host` is set on a Pod, Deployment, or ReplicaSet, players connecting via that hostname will be routed to the corresponding server. This mirrors Velocity's [forced hosts](https://docs.papermc.io/velocity/configuration/#forced-hosts-section) feature but is configured dynamically through Kubernetes labels/annotations. ### Pod @@ -106,6 +108,7 @@ metadata: kuvel.azisaba.net/preferred-server-name: "test-server" # Required for Kuvel to name the server # kuvel.azisaba.net/initial-server: "true" # Uncomment this line if you want to make this server the initial server. # kuvel.azisaba.net/disable-name-suffix: "true" # Use the exact preferred name (no -1 suffix); single replica only. + # kuvel.azisaba.net/forced-host: "pvp.example.com" # Players connecting via pvp.example.com will be sent to this server. spec: containers: - name: test-server @@ -132,6 +135,7 @@ spec: kuvel.azisaba.net/enable-server-discovery: "true" # Required for Kuvel to detect Minecraft servers. Depends on your config. kuvel.azisaba.net/preferred-server-name: "test-server" # Required for Kuvel to name the server # kuvel.azisaba.net/initial-server: "true" # Uncomment this line if you want to make this server the initial server. + # kuvel.azisaba.net/forced-host: "pvp.example.com" # Players connecting via pvp.example.com will be sent to this server. spec: containers: - name: test-server @@ -158,6 +162,7 @@ metadata: kuvel.azisaba.net/preferred-server-name: "lobby" # kuvel.azisaba.net/initial-server: "true" # Uncomment this line if you want to make this load balancer server the initial server. # kuvel.azisaba.net/disable-load-balancer: "true" # Uncomment this line to disable only the load balancer entry. + # kuvel.azisaba.net/forced-host: "lobby.example.com" # Players connecting via lobby.example.com will be sent to this load balancer. spec: replicas: 3 selector: diff --git a/src/main/java/net/azisaba/kuvel/KuvelServiceHandler.java b/src/main/java/net/azisaba/kuvel/KuvelServiceHandler.java index e039f6b2..2586bba0 100644 --- a/src/main/java/net/azisaba/kuvel/KuvelServiceHandler.java +++ b/src/main/java/net/azisaba/kuvel/KuvelServiceHandler.java @@ -10,8 +10,11 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nullable; @@ -38,6 +41,7 @@ public class KuvelServiceHandler { private final UidAndServerNameMap replicaSetUidAndServerNameMap = new UidAndServerNameMap(); private final List initialServerNames = new ArrayList<>(); + private final Map> forcedHosts = new ConcurrentHashMap<>(); private final AtomicReference serverDiscovery = new AtomicReference<>(); private final AtomicReference loadBalancerDiscovery = @@ -59,6 +63,10 @@ public void registerLoadBalancer(LoadBalancer loadBalancer) { initialServerNames.add(serverName); } + if (loadBalancer.getForcedHost() != null) { + addForcedHost(loadBalancer.getForcedHost(), serverName); + } + plugin .getLogger() .info( @@ -101,6 +109,7 @@ public void unregisterLoadBalancer(LoadBalancer loadBalancer) { replicaSetUidAndServerNameMap.unregister(loadBalancer.getReplicaSetUid()); initialServerNames.remove(serverName); + removeForcedHost(serverName); plugin .getLogger() @@ -244,13 +253,26 @@ public boolean registerPod(Pod pod, String serverName) { } } + String labelKeyPrefix = plugin.getKuvelConfig().getLabelKeyPrefix(); String initialServerStr = pod.getMetadata().getLabels().getOrDefault( - LabelKeys.INITIAL_SERVER.getKey(plugin.getKuvelConfig().getLabelKeyPrefix()), "false"); + LabelKeys.INITIAL_SERVER.getKey(labelKeyPrefix), "false"); if (Boolean.parseBoolean(initialServerStr)) { initialServerNames.add(serverName); } + String forcedHost = + pod.getMetadata().getLabels().getOrDefault( + LabelKeys.FORCED_HOST.getKey(labelKeyPrefix), null); + if (forcedHost == null) { + forcedHost = + pod.getMetadata().getAnnotations().getOrDefault( + LabelKeys.FORCED_HOST.getKey(labelKeyPrefix), null); + } + if (forcedHost != null) { + addForcedHost(forcedHost, serverName); + } + plugin .getLogger() .info("Registered server: " + serverName + " (" + pod.getMetadata().getUid() + ")"); @@ -315,6 +337,7 @@ public void unregisterPod(String podUid) { } initialServerNames.remove(serverName); + removeForcedHost(serverName); plugin.getLogger().info("Unregistered server: " + serverName + " (" + podUid + ")"); } @@ -337,4 +360,17 @@ public void unregisterPod(Pod pod) { public boolean isPodRegistered(String podId) { return podUidAndServerNameMap.getServerNameFromUid(podId) != null; } + + private void addForcedHost(String hostname, String serverName) { + forcedHosts.computeIfAbsent(hostname, k -> new CopyOnWriteArrayList<>()).add(serverName); + } + + private void removeForcedHost(String serverName) { + forcedHosts.values().forEach(list -> list.remove(serverName)); + forcedHosts.entrySet().removeIf(entry -> entry.getValue().isEmpty()); + } + + public Map> getForcedHosts() { + return forcedHosts; + } } diff --git a/src/main/java/net/azisaba/kuvel/discovery/impl/redis/RedisLoadBalancerDiscovery.java b/src/main/java/net/azisaba/kuvel/discovery/impl/redis/RedisLoadBalancerDiscovery.java index 4bfd931f..11966f21 100644 --- a/src/main/java/net/azisaba/kuvel/discovery/impl/redis/RedisLoadBalancerDiscovery.java +++ b/src/main/java/net/azisaba/kuvel/discovery/impl/redis/RedisLoadBalancerDiscovery.java @@ -136,6 +136,16 @@ private void registerOrIgnore(ReplicaSet replicaSet, boolean isFetchedFromRedis) .getLabels() .getOrDefault(LabelKeys.INITIAL_SERVER.getKey(labelKeyPrefix), "false") .equalsIgnoreCase("true"); + String forcedHost = + metadata + .getLabels() + .getOrDefault(LabelKeys.FORCED_HOST.getKey(labelKeyPrefix), null); + if (forcedHost == null) { + forcedHost = + metadata + .getAnnotations() + .getOrDefault(LabelKeys.FORCED_HOST.getKey(labelKeyPrefix), null); + } if (isDisableLoadBalancer(metadata)) { return; } @@ -169,7 +179,7 @@ private void registerOrIgnore(ReplicaSet replicaSet, boolean isFetchedFromRedis) kuvelServiceHandler.getReplicaSetUidAndServerNameMap().register(uid, serverName); jedis.hset(RedisKeys.LOAD_BALANCERS_PREFIX.getKey() + groupName, uid, serverName); - redisConnectionLeader.publishNewLoadBalancer(uid, serverName, initialServer); + redisConnectionLeader.publishNewLoadBalancer(uid, serverName, initialServer, forcedHost); RegisteredServer server = plugin @@ -182,7 +192,8 @@ private void registerOrIgnore(ReplicaSet replicaSet, boolean isFetchedFromRedis) server, new RoundRobinLoadBalancingStrategy(), uid, - initialServer)); + initialServer, + forcedHost)); } } diff --git a/src/main/java/net/azisaba/kuvel/listener/ChooseInitialServerListener.java b/src/main/java/net/azisaba/kuvel/listener/ChooseInitialServerListener.java index 492378c8..1c021a2d 100644 --- a/src/main/java/net/azisaba/kuvel/listener/ChooseInitialServerListener.java +++ b/src/main/java/net/azisaba/kuvel/listener/ChooseInitialServerListener.java @@ -4,6 +4,7 @@ import com.velocitypowered.api.event.player.PlayerChooseInitialServerEvent; import com.velocitypowered.api.proxy.ProxyServer; import com.velocitypowered.api.proxy.server.RegisteredServer; +import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -19,6 +20,21 @@ public class ChooseInitialServerListener { @Subscribe public void onInitialServerChoose(PlayerChooseInitialServerEvent event) { + Optional virtualHost = event.getPlayer().getVirtualHost(); + if (virtualHost.isPresent()) { + String hostname = virtualHost.get().getHostString(); + List forcedServerNames = handler.getForcedHosts().get(hostname); + if (forcedServerNames != null && !forcedServerNames.isEmpty()) { + for (String serverName : forcedServerNames) { + Optional optionalServer = proxy.getServer(serverName); + if (optionalServer.isPresent()) { + event.setInitialServer(optionalServer.get()); + return; + } + } + } + } + if (handler.getInitialServerNames().isEmpty()) { return; } diff --git a/src/main/java/net/azisaba/kuvel/loadbalancer/LoadBalancer.java b/src/main/java/net/azisaba/kuvel/loadbalancer/LoadBalancer.java index 16ac3f2a..0e7ae7aa 100644 --- a/src/main/java/net/azisaba/kuvel/loadbalancer/LoadBalancer.java +++ b/src/main/java/net/azisaba/kuvel/loadbalancer/LoadBalancer.java @@ -19,6 +19,7 @@ public class LoadBalancer { private final String replicaSetUid; private final boolean isInitialServer; + private final String forcedHost; private final List endpointServers = new ArrayList<>(); public void addEndpoint(String serverName) { diff --git a/src/main/java/net/azisaba/kuvel/redis/RedisConnectionLeader.java b/src/main/java/net/azisaba/kuvel/redis/RedisConnectionLeader.java index 187784d2..9dcbdd9e 100644 --- a/src/main/java/net/azisaba/kuvel/redis/RedisConnectionLeader.java +++ b/src/main/java/net/azisaba/kuvel/redis/RedisConnectionLeader.java @@ -80,11 +80,11 @@ public void leaveLeader() { } public void publishNewLoadBalancer( - String replicaSetUid, String serverName, boolean initialServer) { + String replicaSetUid, String serverName, boolean initialServer, String forcedHost) { try (Jedis jedis = jedisPool.getResource()) { jedis.publish( RedisKeys.LOAD_BALANCER_ADDED_NOTIFY_PREFIX.getKey() + groupName, - replicaSetUid + ":" + serverName + ":" + initialServer); + replicaSetUid + ":" + serverName + ":" + initialServer + ":" + (forcedHost != null ? forcedHost : "")); } } diff --git a/src/main/java/net/azisaba/kuvel/redis/RedisSubscriber.java b/src/main/java/net/azisaba/kuvel/redis/RedisSubscriber.java index 9503c8b3..3438c6e9 100644 --- a/src/main/java/net/azisaba/kuvel/redis/RedisSubscriber.java +++ b/src/main/java/net/azisaba/kuvel/redis/RedisSubscriber.java @@ -45,6 +45,10 @@ public void onPMessage(String pattern, String channel, String message) { String replicaSetUid = message.split(":")[0]; String serverName = message.split(":")[1]; boolean initialServer = Boolean.parseBoolean(message.split(":")[2]); + String forcedHost = message.split(":").length > 3 ? message.split(":")[3] : null; + if (forcedHost != null && forcedHost.isEmpty()) { + forcedHost = null; + } RegisteredServer server = plugin @@ -56,7 +60,8 @@ public void onPMessage(String pattern, String channel, String message) { server, new RoundRobinLoadBalancingStrategy(), replicaSetUid, - initialServer); + initialServer, + forcedHost); kuvelServiceHandler.registerLoadBalancer(loadBalancer); } else if (channel.startsWith(RedisKeys.POD_DELETED_NOTIFY_PREFIX.getKey())) { kuvelServiceHandler.unregisterPod(message); diff --git a/src/main/java/net/azisaba/kuvel/util/LabelKeys.java b/src/main/java/net/azisaba/kuvel/util/LabelKeys.java index aa5744c6..77969573 100644 --- a/src/main/java/net/azisaba/kuvel/util/LabelKeys.java +++ b/src/main/java/net/azisaba/kuvel/util/LabelKeys.java @@ -9,6 +9,7 @@ public enum LabelKeys { INITIAL_SERVER("initial-server"), DISABLE_NAME_SUFFIX("disable-name-suffix"), DISABLE_LOAD_BALANCER("disable-load-balancer"), + FORCED_HOST("forced-host"), ; private final String key; From 5ddb1c3d33d775305f09cb16b39ad8e61e888281 Mon Sep 17 00:00:00 2001 From: DARCUSK <135838054+darcusk@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:14:28 +0100 Subject: [PATCH 2/3] fix: reconcile ghost backends left registered after a pod rolls (#1) A backend whose Deployment rolls could stay registered in Velocity (and as a load-balancer endpoint) pointing at the dead pod's IP, with no health check to evict it. Players routed there get "Unable to connect you to ", and the only remedy today is a manual `kubectl rollout restart` of the proxy. Three defects combine to produce this: 1. KuvelServiceHandler.unregisterPod early-returned whenever the server name was still mapped to another pod UID, leaving the Velocity ServerInfo / LB endpoint bound to the departed pod's IP and never verifying the surviving UID was actually a live pod. 2. The 5s RedisServerDiscovery poll's delete path ran unregisterPodOrIgnore outside the ReentrantLock that guards registration in processUpdatedPod (TOCTOU), and nothing re-derived state from the live pod list each tick. 3. updateLoadBalancerEndpoints (which rebuilds LB membership from the live pod list) was only ever called from registerLoadBalancer, never from the poll, so a stale endpoint survived in the round-robin rotation. getServersForStartup already prunes dead entries and rebuilds from live Running pods on boot -- which is exactly what a proxy restart does. This change runs that reconciliation continuously. Changes: - Level-triggered reconcile in the 5s poll (RedisServerDiscovery.reconcile): after diff/delete processing each tick, evict any podUidAndServerNameMap UID that is not in the current live-Running labelled-pod set, then call KuvelServiceHandler.reconcileLoadBalancerEndpoints() to rebuild every load balancer's membership from live pods. The delete path and the sweep now run under the same ReentrantLock as registration, closing the TOCTOU. - KuvelServiceHandler.unregisterPod now reconciles instead of early- returning: when the name is still mapped to another UID it verifies that UID is a live (Running) pod and rebinds the Velocity ServerInfo / LB endpoint to the live pod's IP; only when a genuinely-live pod holds the name is teardown skipped. A name is never left bound to a departed UID. Minimal change, reuses existing helpers (registerPod, updateLoadBalancerEndpoints) and mirrors getServersForStartup's Running- phase semantics. The forced-host behaviour from c17652b is preserved. Co-authored-by: Chip Wolf Co-authored-by: Claude Opus 4.8 --- .../azisaba/kuvel/KuvelServiceHandler.java | 76 +++++++++++++++++-- .../impl/redis/RedisServerDiscovery.java | 49 +++++++++++- 2 files changed, 116 insertions(+), 9 deletions(-) diff --git a/src/main/java/net/azisaba/kuvel/KuvelServiceHandler.java b/src/main/java/net/azisaba/kuvel/KuvelServiceHandler.java index 2586bba0..758b0c32 100644 --- a/src/main/java/net/azisaba/kuvel/KuvelServiceHandler.java +++ b/src/main/java/net/azisaba/kuvel/KuvelServiceHandler.java @@ -159,6 +159,44 @@ private void updateLoadBalancerEndpoints(LoadBalancer loadBalancer) { loadBalancer.setEndpoints(endpoints); } + /** + * Rebuild every registered load balancer's endpoint membership from the current live pod list. + * + *

Used by the periodic reconcile so that a backend whose pod has rolled away is dropped from + * the load-balancer rotation even if an incremental update was missed, preventing players from + * being routed to a dead endpoint. + */ + public void reconcileLoadBalancerEndpoints() { + for (LoadBalancer loadBalancer : loadBalancerServerMap.values()) { + updateLoadBalancerEndpoints(loadBalancer); + } + } + + /** + * Look up a live (Running) pod by its UID within the configured namespace and label selectors. + * + * @param podUid The pod UID to look up. + * @return The matching Running pod, or null if no such live pod exists. + */ + @Nullable + private Pod getLivePodByUid(String podUid) { + FilterWatchListDeletable request = + client.pods().inNamespace(namespace); + + for (Entry e : plugin.getKuvelConfig().getLabelSelectors().entrySet()) { + request = request.withLabel(e.getKey(), e.getValue()); + } + + return request + .withField("status.phase", "Running") + .list() + .getItems() + .stream() + .filter(p -> p.getMetadata().getUid().equals(podUid)) + .findFirst() + .orElse(null); + } + /** * Replace new server discovery instance and unregister old one. Specify null for shutdown current * discovery instance. @@ -314,16 +352,40 @@ public void unregisterPod(String podUid) { } String serverName = podUidAndServerNameMap.unregister(podUid); - if (podUidAndServerNameMap.getUidFromServerName(serverName) != null) { + + String survivingUid = podUidAndServerNameMap.getUidFromServerName(serverName); + if (survivingUid != null) { + // The server name is still claimed by another pod UID. Keep the registration + // only when that UID belongs to a genuinely live (Running) pod, and in that + // case rebind the Velocity ServerInfo / load balancer endpoint to the live + // pod's IP. Otherwise fall through to teardown so the name is never left + // pointing at a departed pod (the "ghost backend"). + Pod survivingPod = getLivePodByUid(survivingUid); + if (survivingPod != null && registerPod(survivingPod, serverName)) { + plugin + .getLogger() + .info( + "Unregistered pod: " + + podUid + + " (server name " + + serverName + + " rebound to live pod " + + survivingUid + + ")"); + return; + } + + // Surviving UID is not a live pod (or could not be registered); drop the stale + // mapping before tearing down so the name is not left bound to a departed UID. + podUidAndServerNameMap.unregister(survivingUid); plugin .getLogger() - .info( - "Unregistered pod: " - + podUid - + " (server name " + .warn( + "Server name " + serverName - + " is still in use)"); - return; + + " was still mapped to non-live pod " + + survivingUid + + "; tearing down stale registration."); } plugin .getProxy() diff --git a/src/main/java/net/azisaba/kuvel/discovery/impl/redis/RedisServerDiscovery.java b/src/main/java/net/azisaba/kuvel/discovery/impl/redis/RedisServerDiscovery.java index cee0c7f8..55039116 100644 --- a/src/main/java/net/azisaba/kuvel/discovery/impl/redis/RedisServerDiscovery.java +++ b/src/main/java/net/azisaba/kuvel/discovery/impl/redis/RedisServerDiscovery.java @@ -9,10 +9,12 @@ import java.util.ArrayList; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -71,8 +73,19 @@ public void start() { } } - List uidList = podDiffChecker.getDeletedPodUidList(client); - uidList.forEach(this::unregisterPodOrIgnore); + // Delete detection and the level-triggered reconcile run under the same lock + // that guards registration (processUpdatedPod). Previously the delete path ran + // outside the lock, racing register/unregister (TOCTOU). Holding the lock makes + // eviction and the load-balancer rebuild atomic with respect to registration. + lock.lock(); + try { + List uidList = podDiffChecker.getDeletedPodUidList(client); + uidList.forEach(this::unregisterPodOrIgnore); + + reconcile(podList); + } finally { + lock.unlock(); + } }; taskReference.getAndUpdate( @@ -264,6 +277,38 @@ private void processUpdatedPod(Pod pod) { } } + /** + * Level-triggered reconcile run once per poll. Evicts any registered pod UID that is no longer a + * live (Running) labelled pod, then rebuilds every load balancer's membership from the live pod + * set. + * + *

This mirrors the pruning that {@link #getServersForStartup()} already performs on boot (which + * is what a manual proxy restart does), but applies it continuously so a rolled backend can never + * linger as a "ghost" server registration or load-balancer endpoint pointing at a dead pod IP. + * + *

Must be called while holding {@link #lock}. + * + * @param labelledPods The labelled pods fetched during this poll tick. + */ + private void reconcile(List labelledPods) { + Set liveRunningUids = new HashSet<>(); + for (Pod pod : labelledPods) { + if (pod.getStatus() != null && "Running".equalsIgnoreCase(pod.getStatus().getPhase())) { + liveRunningUids.add(pod.getMetadata().getUid()); + } + } + + // getAllMap() returns a copy, so iterating it while unregisterPodOrIgnore mutates the + // underlying map is safe. + for (String uid : kuvelServiceHandler.getPodUidAndServerNameMap().getAllMap().keySet()) { + if (!liveRunningUids.contains(uid)) { + unregisterPodOrIgnore(uid); + } + } + + kuvelServiceHandler.reconcileLoadBalancerEndpoints(); + } + private void registerPodOrIgnore(Pod pod) { String uid = pod.getMetadata().getUid(); if (kuvelServiceHandler.getPodUidAndServerNameMap().getServerNameFromUid(uid) != null) { From 4e5cb104086baadbefcd7f5c3068b4b53294c1d3 Mon Sep 17 00:00:00 2001 From: DARCUSK <135838054+darcusk@users.noreply.github.com> Date: Sat, 12 Sep 2026 18:45:59 +0100 Subject: [PATCH 3/3] chore: rescue build workflow (JDK 21 + dispatch + artifact) and test suite from stale branches (#2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add workflow_dispatch trigger and upload artifact step for Kuvel build * Update Java version to 21 and improve build workflow - Update Java version from 17 to 21 in pom.xml and build.yml - Switch JDK distribution from adopt to temurin - Add workflow_dispatch trigger - Add artifact upload step * test(forced-host): add unit tests for forced-host concurrency and routing Rescued from stale branch test/k8s-integration (418bcc074). Adds the JUnit 5 + Mockito test-scope dependencies and two test classes: - ForcedHostConcurrencyTest: validates the ConcurrentHashMap + CopyOnWriteArrayList pattern used by KuvelServiceHandler for forced hosts is safe under concurrent add/remove/iterate. - ChooseInitialServerListenerTest: covers forced-host routing, fall through to initial servers, unregistered-server skip, and empty-list cases in ChooseInitialServerListener. The forced-host source changes from that branch are already present on develop/v3 (ConcurrentHashMap + CopyOnWriteArrayList, if/return listener), so only the tests are carried; they compile and pass against the current source unchanged. Co-Authored-By: Claude Opus 4.8 * test(k8s): add forced-host integration harness Rescued from stale branch test/k8s-integration (418bcc074). Adds a Kubernetes harness to exercise forced-host routing end to end: - Dockerfile.kuvel-plugin packages target/Kuvel.jar (matches the pom finalName) into a sidecar image copied into the proxy plugins dir. - backends.yaml, dragonfly.yaml, velocity-config.yaml, velocity-proxy.yaml deploy a Paper backend, a Redis (dragonfly) instance, and a Velocity proxy wired via the KUVEL_* env overrides that KuvelConfig reads, using the kuvel.azisaba.net label prefix and the forced-host label key the current plugin expects. - test_forced_host.py drives the Minecraft handshake against the proxy NodePort (30577) to assert forced-host routing. Labels, jar name, config env vars, and ports are internally consistent with the current plugin, but this harness has NOT been validated against a live cluster in this change. Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Chip Wolf ‮ Co-authored-by: Claude Opus 4.8 --- .github/workflows/build.yml | 11 +- pom.xml | 14 +- .../kuvel/ForcedHostConcurrencyTest.java | 155 +++++++++++++ .../ChooseInitialServerListenerTest.java | 152 ++++++++++++ test-k8s/Dockerfile.kuvel-plugin | 2 + test-k8s/backends.yaml | 41 ++++ test-k8s/dragonfly.yaml | 39 ++++ test-k8s/test_forced_host.py | 216 ++++++++++++++++++ test-k8s/velocity-config.yaml | 59 +++++ test-k8s/velocity-proxy.yaml | 110 +++++++++ 10 files changed, 795 insertions(+), 4 deletions(-) create mode 100644 src/test/java/net/azisaba/kuvel/ForcedHostConcurrencyTest.java create mode 100644 src/test/java/net/azisaba/kuvel/listener/ChooseInitialServerListenerTest.java create mode 100644 test-k8s/Dockerfile.kuvel-plugin create mode 100644 test-k8s/backends.yaml create mode 100644 test-k8s/dragonfly.yaml create mode 100644 test-k8s/test_forced_host.py create mode 100644 test-k8s/velocity-config.yaml create mode 100644 test-k8s/velocity-proxy.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 334f8259..1b1a74c7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,11 +8,12 @@ on: paths-ignore: - "**.md" - "renovate.json" + workflow_dispatch: jobs: build: if: ${{ github.event_name != 'pull_request' || github.repository != github.event.pull_request.head.repo.full_name }} - name: Build with Java 17 + name: Build with Java 21 runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -21,9 +22,13 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 with: - java-version: "17" - distribution: "adopt" + java-version: "21" + distribution: "temurin" cache: "maven" - run: java -version - run: mvn --version - run: mvn -B package + - uses: actions/upload-artifact@v4 + with: + name: Kuvel + path: target/Kuvel.jar diff --git a/pom.xml b/pom.xml index 0e4c3aec..c33ec4c2 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ - 17 + 21 UTF-8 @@ -117,5 +117,17 @@ 1.18.42 provided + + org.junit.jupiter + junit-jupiter + 5.10.2 + test + + + org.mockito + mockito-core + 5.11.0 + test + diff --git a/src/test/java/net/azisaba/kuvel/ForcedHostConcurrencyTest.java b/src/test/java/net/azisaba/kuvel/ForcedHostConcurrencyTest.java new file mode 100644 index 00000000..2859dc3a --- /dev/null +++ b/src/test/java/net/azisaba/kuvel/ForcedHostConcurrencyTest.java @@ -0,0 +1,155 @@ +package net.azisaba.kuvel; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; + +/** + * Tests for the forced-host ConcurrentHashMap + CopyOnWriteArrayList fix + * in KuvelServiceHandler. + * + * The original code used ArrayList inside ConcurrentHashMap.computeIfAbsent, + * which is not thread-safe for concurrent reads during iteration (e.g. in + * ChooseInitialServerListener). The fix switches to CopyOnWriteArrayList. + */ +public class ForcedHostConcurrencyTest { + + /** + * Simulates the fixed addForcedHost / removeForcedHost / iteration pattern. + * Multiple threads add and remove entries while another iterates, + * verifying no ConcurrentModificationException is thrown. + */ + @Test + void concurrentAddRemoveAndIterate_noConcurrentModificationException() throws Exception { + Map> forcedHosts = new ConcurrentHashMap<>(); + int threads = 8; + int opsPerThread = 500; + ExecutorService executor = Executors.newFixedThreadPool(threads); + CountDownLatch latch = new CountDownLatch(threads); + AtomicInteger errors = new AtomicInteger(0); + + // Writers: add and remove forced hosts + for (int t = 0; t < threads / 2; t++) { + final int threadId = t; + executor.submit(() -> { + try { + for (int i = 0; i < opsPerThread; i++) { + String hostname = "host-" + (i % 5) + ".example.com"; + String serverName = "server-" + threadId + "-" + i; + // addForcedHost + forcedHosts.computeIfAbsent(hostname, k -> new CopyOnWriteArrayList<>()) + .add(serverName); + // removeForcedHost (some of the time) + if (i % 3 == 0) { + forcedHosts.values().forEach(list -> list.remove(serverName)); + forcedHosts.entrySet().removeIf(entry -> entry.getValue().isEmpty()); + } + } + } catch (Exception e) { + errors.incrementAndGet(); + } finally { + latch.countDown(); + } + }); + } + + // Readers: iterate forced hosts (like ChooseInitialServerListener does) + for (int t = threads / 2; t < threads; t++) { + executor.submit(() -> { + try { + for (int i = 0; i < opsPerThread; i++) { + for (Map.Entry> entry : forcedHosts.entrySet()) { + List servers = entry.getValue(); + if (servers != null && !servers.isEmpty()) { + // Iterate over the list - this would throw + // ConcurrentModificationException with ArrayList + for (String serverName : servers) { + assertNotNull(serverName); + } + } + } + } + } catch (Exception e) { + errors.incrementAndGet(); + } finally { + latch.countDown(); + } + }); + } + + assertTrue(latch.await(30, TimeUnit.SECONDS), "Threads did not finish in time"); + executor.shutdown(); + assertEquals(0, errors.get(), "Concurrent operations caused errors"); + } + + /** + * Verifies that CopyOnWriteArrayList allows safe iteration while being modified, + * which is the core property we need for ChooseInitialServerListener. + */ + @Test + void copyOnWriteArrayList_safeIterationDuringModification() { + CopyOnWriteArrayList list = new CopyOnWriteArrayList<>(); + list.add("lobby-1"); + list.add("lobby-2"); + list.add("lobby-3"); + + // Iterate and modify simultaneously - should NOT throw + int count = 0; + for (String s : list) { + assertNotNull(s); + count++; + if (count == 1) { + // Modification during iteration is safe with CopyOnWriteArrayList + list.add("lobby-4"); + list.remove("lobby-2"); + } + } + + // Iterator saw the snapshot (3 elements), not the modified list + assertEquals(3, count); + // But the list itself was modified + assertTrue(list.contains("lobby-4")); + assertFalse(list.contains("lobby-2")); + } + + /** + * Verifies the addForcedHost / removeForcedHost logic works correctly. + */ + @Test + void addAndRemoveForcedHost_correctBehavior() { + Map> forcedHosts = new ConcurrentHashMap<>(); + + // Add + forcedHosts.computeIfAbsent("lobby.example.com", k -> new CopyOnWriteArrayList<>()).add("lobby-1"); + forcedHosts.computeIfAbsent("lobby.example.com", k -> new CopyOnWriteArrayList<>()).add("lobby-2"); + forcedHosts.computeIfAbsent("survival.example.com", k -> new CopyOnWriteArrayList<>()).add("survival-1"); + + assertEquals(2, forcedHosts.get("lobby.example.com").size()); + assertEquals(1, forcedHosts.get("survival.example.com").size()); + + // Remove a server from all hosts (like removeForcedHost does) + String toRemove = "lobby-1"; + forcedHosts.values().forEach(list -> list.remove(toRemove)); + forcedHosts.entrySet().removeIf(entry -> entry.getValue().isEmpty()); + + assertEquals(1, forcedHosts.get("lobby.example.com").size()); + assertEquals("lobby-2", forcedHosts.get("lobby.example.com").get(0)); + + // Remove last entry from survival + forcedHosts.values().forEach(list -> list.remove("survival-1")); + forcedHosts.entrySet().removeIf(entry -> entry.getValue().isEmpty()); + + assertNull(forcedHosts.get("survival.example.com"), "Empty host should be removed"); + assertEquals(1, forcedHosts.size()); + } +} diff --git a/src/test/java/net/azisaba/kuvel/listener/ChooseInitialServerListenerTest.java b/src/test/java/net/azisaba/kuvel/listener/ChooseInitialServerListenerTest.java new file mode 100644 index 00000000..5e975af2 --- /dev/null +++ b/src/test/java/net/azisaba/kuvel/listener/ChooseInitialServerListenerTest.java @@ -0,0 +1,152 @@ +package net.azisaba.kuvel.listener; + +import static org.mockito.Mockito.*; + +import com.velocitypowered.api.event.player.PlayerChooseInitialServerEvent; +import com.velocitypowered.api.proxy.Player; +import com.velocitypowered.api.proxy.ProxyServer; +import com.velocitypowered.api.proxy.server.RegisteredServer; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import net.azisaba.kuvel.KuvelServiceHandler; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests for ChooseInitialServerListener, specifically the forced-host + * lookup refactored from lambda-based ifPresent() to explicit if/get. + */ +public class ChooseInitialServerListenerTest { + + private ProxyServer proxy; + private KuvelServiceHandler handler; + private ChooseInitialServerListener listener; + + @BeforeEach + void setUp() { + proxy = mock(ProxyServer.class); + handler = mock(KuvelServiceHandler.class); + listener = new ChooseInitialServerListener(proxy, handler); + } + + @Test + void forcedHost_routesToCorrectServer() { + // Setup forced hosts map + Map> forcedHosts = new ConcurrentHashMap<>(); + forcedHosts.put("lobby.example.com", new CopyOnWriteArrayList<>(Arrays.asList("lobby"))); + when(handler.getForcedHosts()).thenReturn(forcedHosts); + when(handler.getInitialServerNames()).thenReturn(new ArrayList<>()); + + // Setup player with virtual host + Player player = mock(Player.class); + when(player.getVirtualHost()).thenReturn(Optional.of(new InetSocketAddress("lobby.example.com", 25565))); + + // Setup registered server + RegisteredServer lobbyServer = mock(RegisteredServer.class); + when(proxy.getServer("lobby")).thenReturn(Optional.of(lobbyServer)); + + // Create event + PlayerChooseInitialServerEvent event = mock(PlayerChooseInitialServerEvent.class); + when(event.getPlayer()).thenReturn(player); + + // Execute + listener.onInitialServerChoose(event); + + // Verify the forced host was used + verify(event).setInitialServer(lobbyServer); + } + + @Test + void forcedHost_noVirtualHost_fallsThrough() { + // Player without virtual host + Player player = mock(Player.class); + when(player.getVirtualHost()).thenReturn(Optional.empty()); + when(handler.getInitialServerNames()).thenReturn(new ArrayList<>()); + when(handler.getForcedHosts()).thenReturn(new ConcurrentHashMap<>()); + + PlayerChooseInitialServerEvent event = mock(PlayerChooseInitialServerEvent.class); + when(event.getPlayer()).thenReturn(player); + + listener.onInitialServerChoose(event); + + // Should not set any server (no virtual host, no initial servers) + verify(event, never()).setInitialServer(any()); + } + + @Test + void forcedHost_unknownHostname_fallsToInitialServer() { + // Setup forced hosts with a different hostname + Map> forcedHosts = new ConcurrentHashMap<>(); + forcedHosts.put("lobby.example.com", new CopyOnWriteArrayList<>(Arrays.asList("lobby"))); + when(handler.getForcedHosts()).thenReturn(forcedHosts); + + // Player connects with different hostname + Player player = mock(Player.class); + when(player.getVirtualHost()).thenReturn(Optional.of(new InetSocketAddress("unknown.example.com", 25565))); + + // Setup initial server fallback + List initialServers = new ArrayList<>(Arrays.asList("default-server")); + when(handler.getInitialServerNames()).thenReturn(initialServers); + RegisteredServer defaultServer = mock(RegisteredServer.class); + when(proxy.getServer("default-server")).thenReturn(Optional.of(defaultServer)); + + PlayerChooseInitialServerEvent event = mock(PlayerChooseInitialServerEvent.class); + when(event.getPlayer()).thenReturn(player); + + listener.onInitialServerChoose(event); + + // Should fall through to initial server + verify(event).setInitialServer(defaultServer); + } + + @Test + void forcedHost_serverNotRegistered_skipsToNext() { + // Two forced servers, first not registered + Map> forcedHosts = new ConcurrentHashMap<>(); + forcedHosts.put("lobby.example.com", new CopyOnWriteArrayList<>(Arrays.asList("lobby-down", "lobby-up"))); + when(handler.getForcedHosts()).thenReturn(forcedHosts); + when(handler.getInitialServerNames()).thenReturn(new ArrayList<>()); + + Player player = mock(Player.class); + when(player.getVirtualHost()).thenReturn(Optional.of(new InetSocketAddress("lobby.example.com", 25565))); + + // First server not found, second one exists + when(proxy.getServer("lobby-down")).thenReturn(Optional.empty()); + RegisteredServer lobbyUp = mock(RegisteredServer.class); + when(proxy.getServer("lobby-up")).thenReturn(Optional.of(lobbyUp)); + + PlayerChooseInitialServerEvent event = mock(PlayerChooseInitialServerEvent.class); + when(event.getPlayer()).thenReturn(player); + + listener.onInitialServerChoose(event); + + // Should use the second server + verify(event).setInitialServer(lobbyUp); + } + + @Test + void forcedHost_emptyServerList_fallsThrough() { + Map> forcedHosts = new ConcurrentHashMap<>(); + forcedHosts.put("lobby.example.com", new CopyOnWriteArrayList<>()); + when(handler.getForcedHosts()).thenReturn(forcedHosts); + when(handler.getInitialServerNames()).thenReturn(new ArrayList<>()); + + Player player = mock(Player.class); + when(player.getVirtualHost()).thenReturn(Optional.of(new InetSocketAddress("lobby.example.com", 25565))); + + PlayerChooseInitialServerEvent event = mock(PlayerChooseInitialServerEvent.class); + when(event.getPlayer()).thenReturn(player); + + listener.onInitialServerChoose(event); + + // Empty forced host list, no initial servers → no server set + verify(event, never()).setInitialServer(any()); + } +} diff --git a/test-k8s/Dockerfile.kuvel-plugin b/test-k8s/Dockerfile.kuvel-plugin new file mode 100644 index 00000000..154a8ba5 --- /dev/null +++ b/test-k8s/Dockerfile.kuvel-plugin @@ -0,0 +1,2 @@ +FROM busybox:latest +COPY target/Kuvel.jar /plugins/Kuvel.jar diff --git a/test-k8s/backends.yaml b/test-k8s/backends.yaml new file mode 100644 index 00000000..7756b199 --- /dev/null +++ b/test-k8s/backends.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: lobby + namespace: minecraft + labels: + kuvel.azisaba.net/enable-server-discovery: "true" +spec: + replicas: 1 + selector: + matchLabels: + app: lobby + template: + metadata: + labels: + app: lobby + kuvel.azisaba.net/enable-server-discovery: "true" + kuvel.azisaba.net/preferred-server-name: "lobby" + kuvel.azisaba.net/initial-server: "true" + kuvel.azisaba.net/forced-host: "lobby.localhost" + spec: + containers: + - name: paper + image: itzg/minecraft-server:latest + env: + - name: EULA + value: "TRUE" + - name: TYPE + value: "PAPER" + - name: MEMORY + value: "1G" + - name: ONLINE_MODE + value: "FALSE" + ports: + - containerPort: 25565 + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" diff --git a/test-k8s/dragonfly.yaml b/test-k8s/dragonfly.yaml new file mode 100644 index 00000000..e7a4dc38 --- /dev/null +++ b/test-k8s/dragonfly.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dragonfly + namespace: minecraft +spec: + replicas: 1 + selector: + matchLabels: + app: dragonfly + template: + metadata: + labels: + app: dragonfly + spec: + containers: + - name: redis + image: redis:7-alpine + args: ["redis-server", "--requirepass", "password"] + ports: + - containerPort: 6379 + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: dragonfly + namespace: minecraft +spec: + selector: + app: dragonfly + ports: + - port: 6379 + targetPort: 6379 diff --git a/test-k8s/test_forced_host.py b/test-k8s/test_forced_host.py new file mode 100644 index 00000000..ae92d327 --- /dev/null +++ b/test-k8s/test_forced_host.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""Test the forced-host routing in the Kuvel Velocity plugin. + +Sends Minecraft protocol handshake + login start packets to the proxy +with different virtual hosts to verify forced-host routing works. +""" + +import socket +import struct +import sys +import time + + +def write_varint(value): + """Encode an integer as a Minecraft VarInt.""" + result = b"" + while True: + byte = value & 0x7F + value >>= 7 + if value != 0: + byte |= 0x80 + result += struct.pack("B", byte) + if value == 0: + break + return result + + +def read_varint(sock): + """Read a VarInt from a socket.""" + result = 0 + num_read = 0 + while True: + data = sock.recv(1) + if len(data) == 0: + raise ConnectionError("Connection closed") + byte = data[0] + result |= (byte & 0x7F) << (7 * num_read) + num_read += 1 + if num_read > 5: + raise ValueError("VarInt too big") + if (byte & 0x80) == 0: + break + return result + + +def make_handshake_packet(hostname, port, next_state=2): + """Create a Minecraft handshake packet.""" + # Protocol version (767 = 1.21.1, but any recent version works) + protocol_version = write_varint(767) + # Server address (this is the virtual host the proxy checks) + hostname_bytes = hostname.encode("utf-8") + server_address = write_varint(len(hostname_bytes)) + hostname_bytes + # Server port + server_port = struct.pack(">H", port) + # Next state (2 = login) + next_state_bytes = write_varint(next_state) + + # Packet ID 0x00 = Handshake + packet_data = write_varint(0x00) + protocol_version + server_address + server_port + next_state_bytes + # Length-prefixed packet + return write_varint(len(packet_data)) + packet_data + + +def make_login_start_packet(username): + """Create a Minecraft login start packet.""" + # Packet ID 0x00 = Login Start + username_bytes = username.encode("utf-8") + name_field = write_varint(len(username_bytes)) + username_bytes + # UUID (empty/zero for offline mode) + uuid_bytes = b"\x00" * 16 + + packet_data = write_varint(0x00) + name_field + uuid_bytes + return write_varint(len(packet_data)) + packet_data + + +def read_packet(sock, timeout=5): + """Read a full Minecraft packet and return (packet_id, data).""" + sock.settimeout(timeout) + try: + length = read_varint(sock) + data = b"" + while len(data) < length: + chunk = sock.recv(length - len(data)) + if not chunk: + raise ConnectionError("Connection closed while reading packet") + data += chunk + + # Parse packet ID + idx = 0 + packet_id = 0 + shift = 0 + while True: + byte = data[idx] + idx += 1 + packet_id |= (byte & 0x7F) << shift + shift += 7 + if (byte & 0x80) == 0: + break + + return packet_id, data[idx:] + except socket.timeout: + return None, None + + +def read_string(data, offset=0): + """Read a length-prefixed string from data.""" + idx = offset + str_len = 0 + shift = 0 + while True: + byte = data[idx] + idx += 1 + str_len |= (byte & 0x7F) << shift + shift += 7 + if (byte & 0x80) == 0: + break + return data[idx:idx + str_len].decode("utf-8", errors="replace"), idx + str_len + + +def test_connection(host, port, virtual_host, username="TestPlayer"): + """Test connecting to proxy with a given virtual host. + + Returns a tuple (connected, message). + """ + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(10) + try: + sock.connect((host, port)) + + # Send handshake with the virtual host + handshake = make_handshake_packet(virtual_host, port, next_state=2) + sock.sendall(handshake) + + # Send login start + login = make_login_start_packet(username) + sock.sendall(login) + + # Read response + packet_id, data = read_packet(sock, timeout=10) + + if packet_id is None: + return False, "No response (timeout)" + + if packet_id == 0x00: + # Disconnect packet - read the reason + reason, _ = read_string(data) + return False, f"Disconnected: {reason}" + elif packet_id == 0x02: + # Login success + return True, "Login success" + elif packet_id == 0x03: + # Set compression + return True, "Compression set (login proceeding)" + else: + return True, f"Got packet 0x{packet_id:02x} (connection proceeding)" + + except ConnectionRefusedError: + return False, "Connection refused" + except ConnectionError as e: + return False, f"Connection error: {e}" + except socket.timeout: + return False, "Connection timeout" + except Exception as e: + return False, f"Error: {e}" + finally: + sock.close() + + +def main(): + proxy_host = "127.0.0.1" + proxy_port = 30577 + + print("=" * 60) + print("Kuvel Forced-Host Routing Test") + print("=" * 60) + print() + + # Test 1: Connect with forced-host "lobby.localhost" + print("[Test 1] Virtual host: lobby.localhost") + print(" Expected: Routes to 'lobby' server (forced-host match)") + success, msg = test_connection(proxy_host, proxy_port, "lobby.localhost", "TestPlayer1") + print(f" Result: {msg}") + test1_pass = "Disconnected" not in msg or "lobby" in msg.lower() or "could not connect" not in msg.lower() + print(f" Status: {'PASS' if test1_pass else 'FAIL'} - Connection was attempted to a server") + print() + + # Test 2: Connect with unknown host + print("[Test 2] Virtual host: unknown.example.com") + print(" Expected: Falls back to initial server (no forced-host match)") + success, msg = test_connection(proxy_host, proxy_port, "unknown.example.com", "TestPlayer2") + print(f" Result: {msg}") + print(f" Status: Connection handled (fell through to initial server logic)") + print() + + # Test 3: Connect with no virtual host (just IP) + print("[Test 3] Virtual host: 127.0.0.1") + print(" Expected: Falls back to initial server (no forced-host match)") + success, msg = test_connection(proxy_host, proxy_port, "127.0.0.1", "TestPlayer3") + print(f" Result: {msg}") + print(f" Status: Connection handled (fell through to initial server logic)") + print() + + print("=" * 60) + print("Test Summary") + print("=" * 60) + print() + print("The forced-host implementation is working if:") + print(" - Test 1 attempts connection to a backend server") + print(" - Tests 2 & 3 fall through to initial server selection") + print() + print("Check proxy logs for detailed routing decisions:") + print(" kubectl logs -n minecraft -l app=velocity-proxy -c velocity") + + +if __name__ == "__main__": + main() diff --git a/test-k8s/velocity-config.yaml b/test-k8s/velocity-config.yaml new file mode 100644 index 00000000..c0f0c269 --- /dev/null +++ b/test-k8s/velocity-config.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: velocity-config + namespace: minecraft +data: + velocity.toml: | + # Velocity configuration + config-version = "2.7" + bind = "0.0.0.0:25577" + motd = "Kuvel Test Proxy" + show-max-players = 100 + online-mode = false + force-key-authentication = false + prevent-client-proxy-connections = false + player-info-forwarding-mode = "modern" + forwarding-secret-file = "forwarding.secret" + announce-forge = false + kick-existing-players = false + ping-passthrough = "DISABLED" + + [servers] + try = [] + + [forced-hosts] + + [advanced] + compression-threshold = 256 + compression-level = -1 + login-ratelimit = 3000 + connection-timeout = 5000 + read-timeout = 30000 + haproxy-protocol = false + tcp-fast-open = false + bungee-plugin-message-channel = true + show-ping-requests = false + failover-on-unexpected-server-disconnect = true + announce-proxy-commands = true + log-command-executions = false + log-player-connections = true + accept-transfers = false + + [query] + enabled = false + port = 25577 + map = "Velocity" + show-plugins = false + kuvel-config.yml: | + namespace: "minecraft" + label-key-prefix: "kuvel.azisaba.net" + redis: + group-name: "test" + connection: + hostname: "dragonfly.minecraft.svc.cluster.local" + port: 6379 + username: "default" + password: "password" + label-selectors: + - "kuvel.azisaba.net/enable-server-discovery=true" diff --git a/test-k8s/velocity-proxy.yaml b/test-k8s/velocity-proxy.yaml new file mode 100644 index 00000000..bb116e4f --- /dev/null +++ b/test-k8s/velocity-proxy.yaml @@ -0,0 +1,110 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: velocity-proxy + namespace: minecraft +spec: + replicas: 1 + selector: + matchLabels: + app: velocity-proxy + template: + metadata: + labels: + app: velocity-proxy + spec: + serviceAccountName: kuvel-sa + initContainers: + - name: copy-kuvel-plugin + image: kuvel-plugin:latest + imagePullPolicy: Never + command: ['sh', '-c', 'mkdir -p /server-data/plugins && cp /plugins/Kuvel.jar /server-data/plugins/Kuvel.jar'] + volumeMounts: + - name: proxy-data + mountPath: /server-data + containers: + - name: velocity + image: itzg/mc-proxy:latest + env: + - name: TYPE + value: "VELOCITY" + - name: MEMORY + value: "512m" + - name: KUVEL_NAMESPACE + value: "minecraft" + - name: KUVEL_REDIS_CONNECTION_HOSTNAME + value: "dragonfly.minecraft.svc.cluster.local" + - name: KUVEL_REDIS_CONNECTION_PORT + value: "6379" + - name: KUVEL_REDIS_CONNECTION_USERNAME + value: "default" + - name: KUVEL_REDIS_CONNECTION_PASSWORD + value: "password" + - name: KUVEL_REDIS_GROUPNAME + value: "test" + ports: + - containerPort: 25577 + name: minecraft + volumeMounts: + - name: proxy-data + mountPath: /server + - name: velocity-config + mountPath: /config/velocity.toml + subPath: velocity.toml + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + volumes: + - name: proxy-data + emptyDir: {} + - name: velocity-config + configMap: + name: velocity-config +--- +apiVersion: v1 +kind: Service +metadata: + name: velocity-proxy + namespace: minecraft +spec: + type: NodePort + selector: + app: velocity-proxy + ports: + - port: 25577 + targetPort: 25577 + nodePort: 30577 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kuvel-sa + namespace: minecraft +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kuvel-role +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kuvel-rolebinding +subjects: + - kind: ServiceAccount + name: kuvel-sa + namespace: minecraft +roleRef: + kind: ClusterRole + name: kuvel-role + apiGroup: rbac.authorization.k8s.io