diff --git a/.bazelignore b/.bazelignore new file mode 100644 index 00000000000..17dc28aa5bc --- /dev/null +++ b/.bazelignore @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +rocketmq-apis diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index a19bcfc7d55..b93d21b6476 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -21,6 +21,8 @@ jobs: os: [ubuntu-latest] steps: - uses: actions/checkout@v4 + with: + submodules: true - name: Cache Bazel repository uses: actions/cache@v4 with: diff --git a/.github/workflows/codeql_analysis.yml b/.github/workflows/codeql_analysis.yml index fb6ea62a0fd..d310d187acf 100644 --- a/.github/workflows/codeql_analysis.yml +++ b/.github/workflows/codeql_analysis.yml @@ -18,6 +18,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 + with: + submodules: true - name: Cache Maven packages uses: actions/cache@v3 with: @@ -26,11 +28,19 @@ jobs: restore-keys: | ${{ runner.os }}-maven- # Initializes the CodeQL tools for scanning. + - name: Set up JDK 8 + uses: actions/setup-java@v4 + with: + java-version: 8 + distribution: "corretto" - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: java - - name: Autobuild - uses: github/codeql-action/autobuild@v2 + # Explicit build instead of autobuild: rocketmq-proto is generated from + # the rocketmq-apis submodule at compile time, which autobuild cannot + # handle reliably. + - name: Build with Maven + run: mvn -B compile -DskipTests -Dmaven.javadoc.skip=true - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 28d874a3545..517a1b3e876 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -14,6 +14,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@master + with: + submodules: true - name: Set up JDK 8 uses: actions/setup-java@v4 with: diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 6a28111d0b3..ad78075e03f 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -28,6 +28,8 @@ jobs: - name: Checkout uses: actions/checkout@v2 + with: + submodules: true - name: Set up JDK ${{ matrix.jdk }} uses: actions/setup-java@v4 diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml index 15584e2cd16..629b4a245b7 100644 --- a/.github/workflows/maven.yaml +++ b/.github/workflows/maven.yaml @@ -22,6 +22,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + with: + submodules: true - name: Set up JDK ${{ matrix.jdk }} uses: actions/setup-java@v4 with: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..9d5374f16e3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[submodule "rocketmq-apis"] + path = rocketmq-apis + url = https://github.com/apache/rocketmq-apis.git + branch = main diff --git a/WORKSPACE b/WORKSPACE index 4287bddded8..96b1560709b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") RULES_JVM_EXTERNAL_TAG = "4.2" @@ -71,9 +71,9 @@ maven_install( "org.bouncycastle:bcpkix-jdk18on:1.83", "com.google.code.gson:gson:2.9.0", "com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru:1.4.2", - "org.apache.rocketmq:rocketmq-proto:2.1.2", - "com.google.protobuf:protobuf-java:3.20.1", - "com.google.protobuf:protobuf-java-util:3.20.1", + "com.google.protobuf:protobuf-java:3.24.4", + "com.google.protobuf:protobuf-java-util:3.24.4", + "javax.annotation:javax.annotation-api:1.3.2", "com.conversantmedia:disruptor:1.2.10", "org.apache.tomcat:annotations-api:6.0.53", "com.google.code.findbugs:jsr305:3.0.2", @@ -82,13 +82,13 @@ maven_install( "org.openjdk.jmh:jmh-core:1.19", "org.openjdk.jmh:jmh-generator-annprocess:1.19", "com.github.ben-manes.caffeine:caffeine:2.9.3", - "io.grpc:grpc-services:1.47.0", - "io.grpc:grpc-netty-shaded:1.47.0", - "io.grpc:grpc-context:1.47.0", - "io.grpc:grpc-stub:1.47.0", - "io.grpc:grpc-api:1.47.0", - "io.grpc:grpc-protobuf:1.47.0", - "io.grpc:grpc-testing:1.47.0", + "io.grpc:grpc-services:1.50.0", + "io.grpc:grpc-netty-shaded:1.50.0", + "io.grpc:grpc-context:1.50.0", + "io.grpc:grpc-stub:1.50.0", + "io.grpc:grpc-api:1.50.0", + "io.grpc:grpc-protobuf:1.50.0", + "io.grpc:grpc-testing:1.50.0", "org.springframework:spring-core:5.3.27", "io.opentelemetry:opentelemetry-exporter-otlp:1.47.0", "io.opentelemetry:opentelemetry-exporter-prometheus:1.47.0-alpha", @@ -125,6 +125,72 @@ maven_install( ], ) +# Well-known type protos (google/protobuf/{timestamp,duration}.proto). The +# standalone protoc executable does not bundle them, so they have to be placed on +# the include path explicitly. +http_archive( + name = "com_google_protobuf_wkt", + build_file_content = """ +filegroup( + name = "wkt_protos", + srcs = glob(["*.proto"]), + visibility = ["//visibility:public"], +) +""", + sha256 = "616bb3536ac1fff3fb1a141450fa28b875e985712170ea7f1bfe5e5fc41e2cd8", + strip_prefix = "protobuf-24.4/src/google/protobuf", + urls = ["https://github.com/protocolbuffers/protobuf/archive/refs/tags/v24.4.tar.gz"], +) + +# protoc / grpc codegen binaries. rules_jvm_external can only resolve jar +# artifacts, so the prebuilt executables are fetched directly. Using prebuilt +# binaries avoids the grpc-java Bazel toolchain, which would build protoc and +# the grpc plugin from C++ sources. +http_file( + name = "com_google_protobuf_protoc_linux_x86_64", + downloaded_file_path = "protoc", + executable = True, + sha256 = "59a70515db36977cf29ab09323b469dca51dd6572a5e08a731d741f22eef2b6c", + urls = ["https://repo1.maven.org/maven2/com/google/protobuf/protoc/3.24.4/protoc-3.24.4-linux-x86_64.exe"], +) + +http_file( + name = "com_google_protobuf_protoc_osx_x86_64", + downloaded_file_path = "protoc", + executable = True, + sha256 = "8689519587d41b7af9e3b3f2e8c3d1335a23315857d157f61c26c10a1c61a9fa", + urls = ["https://repo1.maven.org/maven2/com/google/protobuf/protoc/3.24.4/protoc-3.24.4-osx-x86_64.exe"], +) + +http_file( + name = "io_grpc_protoc_gen_grpc_java_linux_x86_64", + downloaded_file_path = "protoc-gen-grpc-java", + executable = True, + sha256 = "52dcbe738d3c920d7744780c67417309fe4ed990b380e9d1cf073c0654656191", + urls = ["https://repo1.maven.org/maven2/io/grpc/protoc-gen-grpc-java/1.53.0/protoc-gen-grpc-java-1.53.0-linux-x86_64.exe"], +) + +http_file( + name = "io_grpc_protoc_gen_grpc_java_osx_x86_64", + downloaded_file_path = "protoc-gen-grpc-java", + executable = True, + sha256 = "97da9c1a408fb23391853273272a9c04f1b8ba6a2ce3c18ff95eab9cc91f0388", + urls = ["https://repo1.maven.org/maven2/io/grpc/protoc-gen-grpc-java/1.53.0/protoc-gen-grpc-java-1.53.0-osx-x86_64.exe"], +) + +# The rocketmq-apis git submodule (apache main), which supplies +# apache/rocketmq/v2/*.proto including the admin.proto. +# +# The submodule ships its own BUILD files, but they pull in toolchains this +# workspace does not declare (graknlabs_bazel_distribution, googleapis), so the +# directory is listed in .bazelignore (keeps `bazel build //...` from trying to +# build it) and is surfaced here through a minimal build file instead. +new_local_repository( + name = "rocketmq_apis", + build_file = "//bazel:rocketmq_apis.BUILD", + path = "rocketmq-apis", +) + http_archive( name = "io_buildbuddy_buildbuddy_toolchain", sha256 = "b12273608db627eb14051eb75f8a2134590172cd69392086d392e25f3954ea6e", diff --git a/auth/BUILD.bazel b/auth/BUILD.bazel index da4962e4520..ba3d8ad39f6 100644 --- a/auth/BUILD.bazel +++ b/auth/BUILD.bazel @@ -28,7 +28,7 @@ java_library( "@maven//:org_apache_commons_commons_lang3", "@maven//:commons_collections_commons_collections", "@maven//:com_alibaba_fastjson2_fastjson2", - "@maven//:org_apache_rocketmq_rocketmq_proto", + "//rocketmq-proto:rocketmq-proto", "@maven//:org_slf4j_slf4j_api", "@maven//:com_github_ben_manes_caffeine_caffeine", "@maven//:io_grpc_grpc_api", @@ -55,7 +55,7 @@ java_library( "@maven//:org_apache_commons_commons_lang3", "@maven//:commons_collections_commons_collections", "@maven//:com_alibaba_fastjson2_fastjson2", - "@maven//:org_apache_rocketmq_rocketmq_proto", + "//rocketmq-proto:rocketmq-proto", "@maven//:org_slf4j_slf4j_api", "@maven//:com_github_ben_manes_caffeine_caffeine", "@maven//:io_grpc_grpc_api", diff --git a/bazel/rocketmq_apis.BUILD b/bazel/rocketmq_apis.BUILD new file mode 100644 index 00000000000..ce0c27b9df6 --- /dev/null +++ b/bazel/rocketmq_apis.BUILD @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Build file for the rocketmq-apis git submodule. +# +# The submodule ships its own BUILD files (root + java/ + cpp/), but those pull +# in toolchains this workspace does not declare (graknlabs_bazel_distribution, +# googleapis). Rather than dragging all of that in, the directory is listed in +# .bazelignore (so `bazel build //...` never tries to build it) and is exposed to +# this workspace as the external repository @rocketmq_apis through a minimal +# build file instead. +# +# The proto set is a glob over the whole v2 directory, so admin.proto is +# included alongside definition.proto and service.proto. +filegroup( + name = "v2_protos", + srcs = glob(["apache/rocketmq/v2/*.proto"]), + visibility = ["//visibility:public"], +) diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/GrpcConstants.java b/common/src/main/java/org/apache/rocketmq/common/constant/GrpcConstants.java index 96293e72ecf..9fc4bcf8c62 100644 --- a/common/src/main/java/org/apache/rocketmq/common/constant/GrpcConstants.java +++ b/common/src/main/java/org/apache/rocketmq/common/constant/GrpcConstants.java @@ -73,4 +73,12 @@ public class GrpcConstants { public static final Metadata.Key CHANNEL_ID = Metadata.Key.of("x-mq-channel-id", Metadata.ASCII_STRING_MARSHALLER); + + /** + * Loop-prevention marker for proxy admin RPCs: set to {@code "true"} by a proxy that forwards + * an admin call to the peer proxy owning the target client, so the peer never forwards it + * again. + */ + public static final Metadata.Key ADMIN_FORWARDED + = Metadata.Key.of("x-mq-admin-forwarded", Metadata.ASCII_STRING_MARSHALLER); } diff --git a/docs/proxy-admin-zh.md b/docs/proxy-admin-zh.md new file mode 100644 index 00000000000..8656fb40cc8 --- /dev/null +++ b/docs/proxy-admin-zh.md @@ -0,0 +1,240 @@ +# Proxy Admin 管理接口 + +> 英文版见 `docs/proxy-admin.md`。 + +## 1. 动机 + +RocketMQ 5.0 把客户端访问放到了无状态的 Proxy 后面，但运维侧仍然通过 broker 侧的结构（broker 上的 `ConsumerManager`、Remoting 时代的 admin 命令）来观测客户端。连到 Proxy 的 gRPC 客户端对这些工具不可见：控制面无法回答"哪些 SDK 客户端在线、它们订阅了什么、是否健康"，只能依赖间接的指标推断。 + +控制面 dashboard 需要一个标准的服务端接口来读取完整的 gRPC 客户端数据。本次改动在 Proxy 上实现这个接口。 + +## 2. 契约选择：上游 `Admin` service + +gRPC 契约是**上游 `Admin` service**，定义在 `apache/rocketmq-apis` 仓库（main 分支，作为 git submodule 引入，见 §6）的 `apache/rocketmq/v2/admin.proto` 中。本实现刻意**不**在协议层引入自定义的 `ProxyAdminService`：控制面 RPC 接口位于 dashboard 和多语言 SDK 已经在消费的、经过社区评审的版本化 proto 中，并与上游演进保持增量兼容。 + +proxy 实现 `AdminGrpc.AdminImplBase`（全部 16 个 RPC），并把它绑定在一个独立的 admin gRPC server 上。 + +## 3. 目标 + +1. 在 Proxy 上提供一个独立、专用的 gRPC Admin server，与数据面 `MessagingService` 隔离。 +2. 来自 rocketmq-apis main 的稳定、向后兼容契约（`Admin` service，`apache/rocketmq/v2/admin.proto`）。 +3. 通过专用的 `proxy.admin.*` ACL 2.0 资源实现一等公民的鉴权，区分只读与高权限动作。 +4. 集群级语义但不把拓扑泄漏进协议：每个 proxy 都暴露连接/订阅数据的集群可见视图，并把客户端定向 RPC 转发到持有该客户端的 proxy。 + +## 4. 设计决策 + +### D1 — 服务放置 + +上游 `Admin` service 绑定在自己的 gRPC server/端口（`grpcAdminServerPort`，默认 8088）上，有自己的拦截器链，与数据面（8081）分离。整个接口由全局 kill switch `grpcAdminServerEnable` 门控，**默认为 false**：admin server 是 opt-in 的，除非显式启用否则不会启动。 + +理由：控制面流量绝不能与数据面竞争，且 admin 端口可以只对运维网络开放防火墙。admin server 刻意不暴露 channelz 或 proto reflection。它复用数据面的 `GrpcChannelManager` / `GrpcClientSettingsManager`，使在线客户端对 admin 查询可见。本特性引入的运行时线程池均归实例所有并接入 Proxy 生命周期：admin timeout scheduler 随 `ProxyAdminGrpcService` 关闭，blocking gateway executor 随 `AdminService` 关闭。 + +### D2 — 鉴权：专用 `proxy.admin.*` 资源 + +凭据通过标准 gRPC `Authorization` metadata 传入（与数据面相同的方案，ACL 2.0 签名）。认证以拦截器方式执行，顺序排在标准链路的 `HeaderInterceptor` **之后**（因 gRPC 按注册的逆序调用拦截器，故注册在其之前）。这个顺序是必需的：`HeaderInterceptor` 会把入站的 `x-mq-channel-id` 替换为从传输层派生的 channel id，而认证结果按 channel id 缓存，若在客户端自带的值上做认证，一条连接的请求就能复用另一条连接的缓存结果、跳过签名校验。 + +每个 RPC 映射到一个资源 + 一个动作： + +| 资源 | RPC | 动作 | +|---|---|---| +| `proxy.admin.client` | ListSubscription, ListConsumerConnection (List); DescribeSubscription, DescribeGroupAccumulation, GetConsumerRunningInfo, QueryTimeSpan (Get) | List / Get | +| `proxy.admin.config` | ChangeLogLevel | Update | +| `proxy.admin.connection` | PrintThreadStackTrace, VerifyMessage | Update（高权限）| +| `proxy.admin.route` | GetTopicRoute | Get | +| `proxy.admin.ops` | GetProxyRuntimeStats, DescribeTopicStatus, QueryMessage (Get); ResetGroupOffset (Update); DeleteSubscription (Delete); AdminSendMessage (Pub) | Get / Update / Delete / Pub | + +建模说明：ACL 2.0 的资源类型是 cluster/namespace/topic/group。admin 资源建模为 CLUSTER 类型的字面量，使用保留名称（资源 key `cluster:proxy.admin.`），从而在不与真实集群名冲突、不改动鉴权核心的前提下实现精确的最小权限匹配。 + +模式： +- 集群鉴权关闭、`grpcAdminServerAuthEnable=false` → 开放接口（与数据面语义相同）; +- 集群鉴权开启 → 针对每方法 `proxy.admin.*` 资源的标准 authenticate + authorize 流程; +- `grpcAdminServerAuthEnable=true` → fail-closed。admin 接口必须既能认证调用方**又能**强制每方法 ACL，因此该模式要求集群级的认证**和**授权开关都开启。任一关闭时所有请求都被拒绝（认证关 → `UNAUTHENTICATED`; 授权关 → `FAILED_PRECONDITION`），而不是在没有真正权限校验的情况下放行。这是刻意为之：ACL 2.0 评估器本身受集群授权开关门控，若在授权关闭时服务请求，会静默放行每个调用，包括破坏性的那些。 + +审计：每个被服务的 RPC 都会向鉴权审计 logger 写入 `[PROXY-ADMIN-AUDIT] subject/method/resource/action/sourceIp`（与 ACL 2.0 引擎自身的审计日志一起，满足 Console 用户 + AK + 资源 + 操作的审计四元组要求）。 + +### D3 — 多 proxy 语义：集群可见，协议不含拓扑 + +一个 gRPC 客户端同一时刻只连到一个 proxy，但 dashboard 仍需从任意单个 admin 端点看到整个集群。两个机制在不给协议加 scope/peering 字段的前提下做到这点： + +- **连接 / 订阅数据是集群可见的。** 列表类 RPC（`ListConsumerConnection`、`ListSubscription`、`DescribeSubscription`）把 broker 侧的 `ConsumerConnection` 视图（覆盖 remoting 客户端，以及被 `HeartbeatSyncer` 在 proxy 间同步的 gRPC 客户端）与本 proxy 自己的 `ConsumerManager`（覆盖本地接入的 gRPC v2 客户端）合并。结果按 group + topic 去重，因此应答不随在线消费者数量膨胀，也不取决于查询的是哪个 proxy。 +- **客户端定向 RPC 转发到持有该客户端的 proxy。** `PrintThreadStackTrace`、`VerifyMessage`、`GetConsumerRunningInfo` 需要只有持有方 proxy 才有的实时 telemetry 流。`ProxyAdminForwarder` 识别出属于 peer 的客户端（其 `ConsumerManager` 条目是 `RemoteChannel`，`getRemoteProxyIp()` 为 peer 的 `localServeAddr`），把整个 RPC——连同原始 metadata——转发到 peer 的 admin 端口，并用 `x-mq-admin-forwarded` 头防止转发成环。peer 的 admin 端口按集群统一假设（心跳同步载荷不携带它）; 异构 admin 端口的集群需要先把它加入心跳记录。 + +`GetProxyRuntimeStats` 刻意报告**本进程**的状态（自身的连接/生产者/消费者计数），因此想要按节点统计的 dashboard 直接查询每个 proxy。 + +### D4 — 每个 RPC 的数据源 + +| 数据源 | RPC | 机制 | +|---|---|---| +| 集群可见连接 | ListConsumerConnection, ListSubscription, DescribeSubscription | broker 侧 `ConsumerConnection`（remoting + proxy 间同步的客户端）与本 proxy 的 `ConsumerManager`（本地 gRPC v2 客户端）合并 | +| 客户端定向 telemetry relay | PrintThreadStackTrace, VerifyMessage, GetConsumerRunningInfo | relay 到持有方 `GrpcClientChannel` 的 telemetry command; 客户端在 peer 上时转发到 peer proxy（D3）| +| Broker 网关（异步、多 broker fan-out）| DescribeTopicStatus, DescribeGroupAccumulation, ResetGroupOffset, QueryMessage, QueryTimeSpan, GetTopicRoute, DeleteSubscription | proxy 自己的异步 `AdminService` remoting 网关; 每个 broker hop 有独立 deadline，慢/无响应的 broker 不会挂起整个 RPC | +| Proxy producer 路径 | AdminSendMessage | `MessagingProcessor.sendMessage`（定时 / FIFO 消息通过 system properties 生效）| +| Proxy 运行时 | ChangeLogLevel | root logger 上重定位的 logback API | + +所有 broker 侧调用都走异步 `AdminService` 网关并并发扇出到所有相关 broker，因此 gRPC 执行线程不会被阻塞。 + +### D5 — 协议覆盖 + +连接/订阅列表是集群可见的，同时包含 remoting 客户端（通过 broker 侧 `ConsumerConnection`）和 gRPC v2 客户端（通过 proxy 的 `ConsumerManager`）。客户端定向 telemetry RPC（`PrintThreadStackTrace`、`VerifyMessage`、`GetConsumerRunningInfo`）作用于持有 proxy telemetry 流的 gRPC v2 客户端，并支持跨 proxy 转发; 它们的完整 running-info 载荷受 telemetry 协议能承载的内容限制（见 §8 诚实边界）。 + +## 5. 能力映射 + +| 需求 | 交付为 | 说明 | +|---|---|---| +| ListClients（按 group/topic/clientId 前缀过滤，分页）| `ListConsumerConnection(group[, topic])` | clientId 前缀过滤和分页在上游契约里无法表达; 这是本次部分交付的已记录边界 | +| DescribeClient（SDK 版本、订阅、心跳、鉴权、Pop 进度）| `DescribeSubscription`（每客户端订阅）+ `GetConsumerRunningInfo`（订阅 + 运行信息）| 心跳/鉴权状态/Pop 进度字段在上游契约里不存在 | +| ListClientsByGroup / ListClientsByTopic | `ListConsumerConnection(group[, topic])` | 完全覆盖 | +| 多 proxy 集群聚合 | 集群可见连接数据 + 客户端定向 RPC 转发 | 见 D3 | + +M2+ 项（配置热更新、配额、连接控制、Pop/batch 诊断、路由观测流）不属于上游 `Admin` 契约，不在本次交付范围内。 + +## 6. 构建 proto 源码 + +`apache/rocketmq-apis` 仓库（main 分支，包含 `apache/rocketmq/v2/admin.proto`）作为 **git submodule** 引入（`rocketmq-apis/`，见 `.gitmodules`）; CI workflow 用 `submodules: true` 检出。 + +- **Maven**：`rocketmq-proto` 模块在构建时通过 `protobuf-maven-plugin` 为 `apache/rocketmq/v2/{definition,service,admin}.proto` 生成 Java + gRPC stub; 它的版本跟随 reactor（`${revision}`），因此构建本仓库不需要已发布的 proto 制品。 +- **Bazel**：`//rocketmq-proto:rocketmq-proto` 通过对 submodule 的 `genrule`（作为 `@rocketmq_apis` 外部仓库呈现）构建相同的类，使用 pin 的 `protoc` / `protoc-gen-grpc-java` 二进制和 well-known-type proto。 +- 其他消费者（dashboard、SDK）继续使用 rocketmq-apis 发布的制品; 只有本仓库从源码构建 proto。 + +## 7. 配置参考 + +| Key | 默认值 | 含义 | +|---|---|---| +| `grpcAdminServerEnable` | false | kill switch; admin 接口是 opt-in 的。设为 `true` 启动 admin gRPC server; `false`（默认）= 不启动 | +| `grpcAdminServerPort` | 8088 | 专用 admin gRPC 端口（<=0 禁用）| +| `grpcAdminServerAuthEnable` | false | fail-closed 模式; 要求集群认证**和**授权都已开启（见 D2）| +| `grpcAdminServerRequestTimeoutMillis` | 3000 | admin RPC 扇出的每个 broker hop 的 deadline | + +## 8. 诚实边界 + +admin 接口绝不填写自己无法如实提供的字段; 下列情况以明确 status 回应或留空，而不是伪造： + +- **DeleteSubscription 在开源里事实上是 no-op。** 开源代码没有任何地方写入 `SubscriptionGroupConfig.subscriptionDataSet`（broker 只读它），因此没有持久化的 per-topic 订阅可删。该 RPC 返回 `NOT_FOUND` 并说明原因，**消费组及其位点保持不动**。需要该 RPC 的发行版用外部订阅存储填充同一字段。 +- **QueryMessage** 支持按 `message_id`（unique-key 索引）和 `message_key` 查询。`subscription` / `lite_topic` / 纯时间范围扫描三种变体需要开源 broker 不提供的分页游标，返回 `BAD_REQUEST`。 +- **GetProxyRuntimeStats** 的 `in_tps` / `out_tps` 留空：开源 proxy 没有进程级吞吐计数器，填 0 会与空闲 proxy 无法区分。 +- **GetConsumerRunningInfo** 对 gRPC v2 客户端只返回 `subscriptions`：v2 telemetry 协议没有承载 `properties`、`message_queue_table`、`consume_status_table` 的回包。该 RPC 返回 `OK` 并附说明性 status message，而不是空壳。 +- **DescribeTopicStatus** 的 `create_timestamp` / `tags` 留空，因为 broker 不记录 topic 创建时间。 + +## 9. 里程碑 + +- M1（本次交付）：基于上游 `Admin` service 的在线客户端查询 — ListConsumerConnection / DescribeSubscription / GetConsumerRunningInfo / DescribeGroupAccumulation，以集群可见方式并支持跨 proxy 转发提供，外加完整的 16-RPC `Admin` 接口用于 dashboard 集成。 +- 未来（需要上游 proto 演进）：客户端列表的 clientId 前缀过滤 / 分页、心跳和鉴权状态字段，以及 M2+ 接口（配置/配额/连接/路由观测）。 + +## 10. 验收标准映射 + +| 标准 | 状态 | +|---|---| +| 设计文档 + 稳定向后兼容的 proto 契约 | 本文档 + 上游 `admin.proto`（rocketmq-apis main）| +| 客户端查询 RPC 合入服务端仓库 | `ProxyAdminGrpcService` 在 proxy 上实现全部 16 个 `Admin` RPC | +| 独立 ACL 控制、只读/高危分离、最小权限文档 | D2 资源/动作 + 下方 §11（最小权限配置指南）| +| 与 dashboard 的 E2E | 契约 = 上游 `Admin` service，为 dashboard 集成冻结（跨仓库）| + +## 11. 最小权限配置指南 + +admin 接口对每个 RPC 按专用的 `proxy.admin.*` ACL 2.0 资源鉴权（见上方 §4 决策 D2）。本节给出每个运维角色的最小权限策略。 + +### 11.1 资源与动作模型 + +资源（ACL 2.0 key; 建模为带保留名称的 cluster 类型字面量）： + +| 资源 key | 保护 | +|---|---| +| `cluster:proxy.admin.client` | 在线客户端查询与客户端诊断 | +| `cluster:proxy.admin.config` | proxy 运行时日志级别变更 | +| `cluster:proxy.admin.connection` | 客户端定向 telemetry 命令：线程栈、verify message（高危）| +| `cluster:proxy.admin.route` | topic 路由视图 | +| `cluster:proxy.admin.ops` | broker 侧运维：stats/topic status/message query（读）和 reset offset / delete subscription / admin send（高危）| + +动作分类： + +- 只读：`Get`, `List` +- 高权限（变更 / 破坏性）：`Update`, `Delete`, `Pub` + +服务端把每个 RPC 映射到恰好一个（资源, 动作）对; 授予只读动作绝不会授权高权限 RPC。 + +### 11.2 角色模板 + +所有命令针对集群的任意 broker/namesrv 运行（ACL 2.0 存储）。先创建用户： + +```bash +sh mqadmin createUser -n -u -p +``` + +#### 角色 A — 只读观察者（dashboard 服务账号） + +在线客户端、订阅、堆积、诊断、配置/路由视图。 + +```bash +sh mqadmin updateAcl -n \ + -s user:rip2-ro \ + -r cluster:proxy.admin.client,cluster:proxy.admin.config,cluster:proxy.admin.route,cluster:proxy.admin.ops \ + -a Get,List \ + -d Allow +``` + +说明：`proxy.admin.ops` 上的 `Get,List` 覆盖只读的 broker 侧 RPC; 变更类 ops RPC 需要 `Update`/`Delete`/`Pub`，仍被拒绝。 + +#### 角色 B — on-call 运维（观察者 + 客户端诊断命令） + +角色 A 加上向已连接客户端下发 telemetry 命令（线程栈 / verify message）的能力。 + +```bash +sh mqadmin updateAcl -n \ + -s user:rip2-oncall \ + -r cluster:proxy.admin.connection \ + -a Update \ + -d Allow +# 外加上面的角色 A 授权 +``` + +#### 角色 C — Admin（完全控制，break-glass） + +offset 重置、订阅删除、admin send、客户端诊断命令。 + +```bash +sh mqadmin updateAcl -n \ + -s user:rip2-admin \ + -r cluster:proxy.admin.client,cluster:proxy.admin.config,cluster:proxy.admin.connection,cluster:proxy.admin.route,cluster:proxy.admin.ops \ + -a Get,List,Update,Delete,Pub \ + -d Allow +``` + +（把角色 C 账号控制在最小范围; 每次使用都会以 `[PROXY-ADMIN-AUDIT]` 前缀记录在鉴权审计日志里。） + +#### 环境限制（推荐） + +通过 `-i` sourceIp 选项把 admin 访问限制到运维网络： + +```bash +sh mqadmin updateAcl -n \ + -s user:rip2-ro \ + -r cluster:proxy.admin.client \ + -a Get,List \ + -d Allow \ + -i 10.0.0.0/8 +``` + +### 11.3 Fail-Closed 模式 + +默认情况下 admin server 遵循集群级认证/授权开关（与数据面行为相同）。fail-closed 模式让 admin 接口拒绝服务任何它无法完全授权的请求： + +```properties +# proxy.json / -D grpcAdminServerAuthEnable=true +grpcAdminServerAuthEnable: true +``` + +`grpcAdminServerAuthEnable=true` 时，admin 接口要求集群认证**和**授权开关都开启。认证关闭时请求以 `UNAUTHENTICATED` 拒绝; 授权关闭时以 `FAILED_PRECONDITION` 拒绝。这保证 `proxy.admin.*` ACL 被真正强制执行——当 admin 端口无法做网络隔离时使用。 + +### 11.4 启用 / 禁用接口 + +接口**默认关闭**（`grpcAdminServerEnable=false`）。要启用： + +```properties +grpcAdminServerEnable: true # 在 grpcAdminServerPort 上启动 admin gRPC server +``` + +要保持禁用（默认），保留 `grpcAdminServerEnable: false`，或把 `grpcAdminServerPort` 设为 0 / 负数，此时 admin gRPC server 完全不启动。 + +### 11.5 审计 + +每个被服务的 admin RPC 都会把 subject（Console 登录用户 / AK）、method、resource、action 和 source IP 记录到鉴权审计 logger; 被拒绝的请求由 ACL 2.0 引擎自身记录。这满足审计四元组要求（Console 用户 + AK + 资源 + 操作）。 diff --git a/docs/proxy-admin.md b/docs/proxy-admin.md new file mode 100644 index 00000000000..7380b6f9663 --- /dev/null +++ b/docs/proxy-admin.md @@ -0,0 +1,370 @@ +# Proxy Admin Management Interface + +> 中文版见 `docs/proxy-admin-zh.md`。 + +## 1. Motivation + +RocketMQ 5.0 moved client access behind the stateless Proxy, but operations still +observe clients through broker-side structures (`ConsumerManager` on the broker, +Remoting-era admin commands). gRPC clients attached to a Proxy are invisible to +those tools: the control plane cannot answer "which SDK clients are online, what +do they subscribe to, are they healthy" without indirect metrics heuristics. + +A control-plane dashboard needs a standard server-side interface to read complete +gRPC client data. This change implements that interface on the Proxy itself. + +## 2. Contract choice: upstream `Admin` service + +The gRPC contract is the **upstream `Admin` service** defined in +`apache/rocketmq/v2/admin.proto` of the `apache/rocketmq-apis` repository (main +branch, consumed as a git submodule — see §6). This implementation deliberately +does NOT introduce a bespoke `ProxyAdminService` in the protocol layer: the +control-plane RPC surface lives in the same versioned, community-reviewed proto +that the dashboard and multi-language SDKs already consume, and stays +additive-compatible with upstream evolution. + +The proxy implements `AdminGrpc.AdminImplBase` (all 16 RPCs) and binds it on a +dedicated admin gRPC server. + +## 3. Goals + +1. A dedicated, independent gRPC Admin server on the Proxy, isolated from the + data-plane `MessagingService`. +2. Stable, backward-compatible contract from rocketmq-apis main (`Admin` + service, `apache/rocketmq/v2/admin.proto`). +3. First-class authorization under dedicated `proxy.admin.*` ACL 2.0 resources + with read-only / high-privilege action separation. +4. Cluster-wide semantics without leaking topology into the protocol: every + proxy exposes a cluster-visible view of connection/subscription data and + forwards client-directed RPCs to the proxy that owns the client. + +## 4. Design Decisions + +### D1 — Service placement + +The upstream `Admin` service is bound on its own gRPC server/port +(`grpcAdminServerPort`, default 8088) with its own interceptor chain, separate +from the data plane (8081). The whole surface is gated by a global kill switch +`grpcAdminServerEnable`, which **defaults to false**: the admin server is opt-in +and is not started unless explicitly enabled. + +Rationale: control-plane traffic must never contend with the data plane, and the +admin port can be firewalled to the operations network only. The admin server +intentionally does NOT expose channelz or proto reflection. It reuses the data +plane's `GrpcChannelManager` / `GrpcClientSettingsManager` so online clients are +visible to admin queries. Runtime executors introduced by this feature are +instance-owned and registered in the Proxy lifecycle: the admin timeout scheduler +stops with `ProxyAdminGrpcService`, and the blocking gateway executor stops with +`AdminService`. + +### D2 — Authorization: dedicated `proxy.admin.*` resources + +Credentials arrive in the standard gRPC `Authorization` metadata (same scheme as +the data plane, ACL 2.0 signature). Authentication runs as an interceptor, ordered +**after** the standard pipeline's `HeaderInterceptor` (registered before it, since +gRPC invokes interceptors in reverse registration order). That ordering is +load-bearing: `HeaderInterceptor` replaces the inbound `x-mq-channel-id` with the +channel id derived from the transport, and authentication results are cached per +channel id, so authenticating on the client-supplied value would let a request on +one connection reuse another connection's cached success and skip signature +verification. + +Every RPC maps to one resource + one action: + +| Resource | RPCs | Actions | +|---|---|---| +| `proxy.admin.client` | ListSubscription, ListConsumerConnection (List); DescribeSubscription, DescribeGroupAccumulation, GetConsumerRunningInfo, QueryTimeSpan (Get) | List / Get | +| `proxy.admin.config` | ChangeLogLevel | Update | +| `proxy.admin.connection` | PrintThreadStackTrace, VerifyMessage | Update (high privilege) | +| `proxy.admin.route` | GetTopicRoute | Get | +| `proxy.admin.ops` | GetProxyRuntimeStats, DescribeTopicStatus, QueryMessage (Get); ResetGroupOffset (Update); DeleteSubscription (Delete); AdminSendMessage (Pub) | Get / Update / Delete / Pub | + +Modeling note: ACL 2.0 resource types are cluster/namespace/topic/group. The +admin resources are modeled as CLUSTER-typed literals with reserved names +(resource keys `cluster:proxy.admin.`), which yields exact +least-privilege matching without colliding with real cluster names and without +changing the auth core. + +Modes: +- cluster auth disabled, `grpcAdminServerAuthEnable=false` → open surface (same + semantics as the data plane); +- cluster auth enabled → standard authenticate + authorize pipeline against the + per-method `proxy.admin.*` resource; +- `grpcAdminServerAuthEnable=true` → fail-closed. The admin surface must be able + to both authenticate the caller **and** enforce the per-method ACL, so this + mode requires the cluster-wide authentication **and** authorization switches to + be on. If either is off every request is refused (authentication off → + `UNAUTHENTICATED`; authorization off → `FAILED_PRECONDITION`) rather than served + without a real permission check. This is deliberate: the ACL 2.0 evaluator is + itself gated by the cluster authorization switch, so serving requests with + authorization off would silently pass every call, including the destructive + ones. + +Audit: every served RPC writes `[PROXY-ADMIN-AUDIT] subject/method/resource/ +action/sourceIp` to the auth audit logger (satisfies the Console-user + AK + +resource + operation audit tuple requirement together with the ACL 2.0 engine's +own audit log). + +### D3 — Multi-proxy semantics: cluster-visible, no topology in the protocol + +A gRPC client is attached to exactly one proxy at a time, but a dashboard must +still see the whole cluster from any single admin endpoint. Two mechanisms make +that work without adding scope/peering fields to the protocol: + +- **Connection / subscription data is cluster-visible.** The listing RPCs + (`ListConsumerConnection`, `ListSubscription`, `DescribeSubscription`) merge + the broker-side `ConsumerConnection` view (which covers remoting clients, and + gRPC clients replicated between proxies by `HeartbeatSyncer`) with this proxy's + own `ConsumerManager` (which covers the gRPC v2 clients attached locally). The + result is deduplicated by group + topic, so the answer does not scale with the + number of online consumers and does not depend on which proxy is queried. +- **Client-directed RPCs are forwarded to the owning proxy.** + `PrintThreadStackTrace`, `VerifyMessage` and `GetConsumerRunningInfo` need the + live telemetry stream that only the owning proxy holds. `ProxyAdminForwarder` + detects a client owned by a peer (its `ConsumerManager` entry is a + `RemoteChannel` whose `getRemoteProxyIp()` is the peer's `localServeAddr`) and + forwards the whole RPC — original metadata included — to the peer's admin port, + guarding against loops with the `x-mq-admin-forwarded` header. The peer admin + port is assumed uniform across the cluster (the heartbeat sync payload does not + carry it); a cluster running heterogeneous admin ports would need that added to + the heartbeat record first. + +`GetProxyRuntimeStats` intentionally reports the **local** process (its own +connection/producer/consumer counts), so a dashboard that wants per-node stats +queries each proxy directly. + +### D4 — Data sources per RPC + +| Source | RPCs | Mechanism | +|---|---|---| +| Cluster-visible connections | ListConsumerConnection, ListSubscription, DescribeSubscription | broker-side `ConsumerConnection` (remoting + peer-synced clients) merged with this proxy's `ConsumerManager` (local gRPC v2 clients) | +| Client-directed telemetry relay | PrintThreadStackTrace, VerifyMessage, GetConsumerRunningInfo | telemetry command relayed to the owning `GrpcClientChannel`; forwarded to the peer proxy when the client is remote (D3) | +| Broker gateway (async, multi-broker fan-out) | DescribeTopicStatus, DescribeGroupAccumulation, ResetGroupOffset, QueryMessage, QueryTimeSpan, GetTopicRoute, DeleteSubscription | the proxy's own async `AdminService` remoting gateway; each broker hop is bounded by its own deadline so a slow/unresponsive broker cannot hang the RPC | +| Proxy producer path | AdminSendMessage | `MessagingProcessor.sendMessage` (system properties honoured for timer / FIFO messages) | +| Proxy runtime | ChangeLogLevel | relocated logback API on the root logger | + +Every broker-facing call goes through the asynchronous `AdminService` gateway and +is fanned out to all relevant brokers concurrently, so the gRPC executor thread is +never blocked. + +### D5 — Protocol coverage + +The connection/subscription listings are cluster-visible and include both +remoting clients (via the broker-side `ConsumerConnection`) and gRPC v2 clients +(via the proxy's `ConsumerManager`). The client-directed telemetry RPCs +(`PrintThreadStackTrace`, `VerifyMessage`, `GetConsumerRunningInfo`) operate on the +gRPC v2 client that holds a proxy telemetry stream, with cross-proxy forwarding; +their full running-info payload is bounded by what the telemetry protocol can +carry (see §8, honest boundaries). + +## 5. Capability mapping + +| Requirement | Delivered as | Notes | +|---|---|---| +| ListClients (filter by group/topic/clientId prefix, paged) | `ListConsumerConnection(group[, topic])` | clientId-prefix filter and pagination are not expressible in the upstream contract; documented boundary of this partial delivery | +| DescribeClient (SDK version, subscriptions, heartbeat, auth, Pop progress) | `DescribeSubscription` (per-client subscriptions) + `GetConsumerRunningInfo` (subscriptions + running info) | heartbeat/auth-status/Pop-progress fields do not exist in the upstream contract | +| ListClientsByGroup / ListClientsByTopic | `ListConsumerConnection(group[, topic])` | fully covered | +| Multi-proxy cluster aggregation | cluster-visible connection data + client-directed RPC forwarding | see D3 | + +M2+ items (config hot update, quotas, connection control, Pop/batch diagnostics, +route observation streaming) are not part of the upstream `Admin` contract and +are out of scope for this delivery. + +## 6. Building the proto sources + +The `apache/rocketmq-apis` repository (main branch, containing +`apache/rocketmq/v2/admin.proto`) is consumed as a **git submodule** +(`rocketmq-apis/`, see `.gitmodules`); CI workflows check it out with +`submodules: true`. + +- **Maven**: the `rocketmq-proto` module generates the Java + gRPC stubs for + `apache/rocketmq/v2/{definition,service,admin}.proto` at build time via + `protobuf-maven-plugin`; its version follows the reactor (`${revision}`), so + no published proto artifact is required to build this repository. +- **Bazel**: `//rocketmq-proto:rocketmq-proto` builds the same classes through a + `genrule` over the submodule (surfaced as the `@rocketmq_apis` external + repository), using pinned `protoc` / `protoc-gen-grpc-java` binaries and the + well-known-type protos. +- Other consumers (dashboard, SDKs) continue to use the artifact published by + rocketmq-apis; only this repository builds the proto from source. + +## 7. Configuration Reference + +| Key | Default | Meaning | +|---|---|---| +| `grpcAdminServerEnable` | false | kill switch; the admin surface is opt-in. Set `true` to start the admin gRPC server; `false` (default) = not started | +| `grpcAdminServerPort` | 8088 | dedicated admin gRPC port (<=0 disables) | +| `grpcAdminServerAuthEnable` | false | fail-closed mode; requires cluster authentication **and** authorization to be enabled (see D2) | +| `grpcAdminServerRequestTimeoutMillis` | 3000 | per-broker-hop deadline for the broker calls an admin RPC fans out to | + +## 8. Honest boundaries + +The admin surface never fills a field it cannot truthfully supply; the following +are answered with an explicit status or left unset rather than faked: + +- **DeleteSubscription is effectively a no-op in open source.** Nothing in the + open-source codebase writes `SubscriptionGroupConfig.subscriptionDataSet` (the + broker only reads it), so there is no persisted per-topic subscription to + remove. The RPC returns `NOT_FOUND` with an explanation and **leaves the + consumer group and its offsets untouched**. Distributions that need this RPC + back it with an external subscription store that populates the same field. +- **QueryMessage** supports lookup by `message_id` (unique-key index) and + `message_key`. The `subscription` / `lite_topic` / pure time-range scan variants + need a paging cursor the open-source broker does not provide and return + `BAD_REQUEST`. +- **GetProxyRuntimeStats** leaves `in_tps` / `out_tps` unset: the open-source + proxy keeps no per-process throughput counter, and reporting 0 would be + indistinguishable from an idle proxy. +- **GetConsumerRunningInfo** over a gRPC v2 client returns only `subscriptions`: + the v2 telemetry protocol has no reply message carrying `properties`, + `message_queue_table` or `consume_status_table`. The RPC returns `OK` with an + explanatory status message rather than an empty shell. +- **DescribeTopicStatus** leaves `create_timestamp` / `tags` unset because the + broker does not record a topic creation time. + +## 9. Milestones + +- M1 (this delivery): online client query over the upstream `Admin` service — + ListConsumerConnection / DescribeSubscription / GetConsumerRunningInfo / + DescribeGroupAccumulation, served cluster-visibly with cross-proxy forwarding, + plus the full 16-RPC `Admin` surface for dashboard integration. +- Future (needs upstream proto evolution): clientId-prefix filter / pagination + for client listings, heartbeat & auth-status fields, and the M2+ surfaces + (config/quota/connection/route observation). + +## 10. Acceptance Criteria Mapping + +| Criterion | Status | +|---|---| +| Design document + stable backward-compatible proto contract | this document + upstream `admin.proto` (rocketmq-apis main) | +| Client query RPCs merged into the server repo | `ProxyAdminGrpcService` implements all 16 `Admin` RPCs on the proxy | +| Independent ACL control, read-only/high-risk separation, least-privilege doc | D2 resources/actions + §11 (Least-Privilege Configuration Guide) below | +| E2E with the dashboard | contract = upstream `Admin` service, frozen for dashboard integration (cross-repo) | + +## 11. Least-Privilege Configuration Guide + +The admin surface authorizes every RPC against dedicated `proxy.admin.*` +ACL 2.0 resources (see §4 decision D2 above). This section shows the +minimum-permission policy for each operational role. + +### 11.1 Resource & Action Model + +Resources (ACL 2.0 keys; modeled as cluster-typed literals with reserved names): + +| Resource key | Protects | +|---|---| +| `cluster:proxy.admin.client` | online client query & client diagnostics | +| `cluster:proxy.admin.config` | proxy runtime log-level change | +| `cluster:proxy.admin.connection` | client-directed telemetry commands: thread stack, verify message (HIGH) | +| `cluster:proxy.admin.route` | topic route view | +| `cluster:proxy.admin.ops` | broker-facing ops: stats/topic status/message query (read) and reset offset / delete subscription / admin send (HIGH) | + +Action classes: + +- Read-only: `Get`, `List` +- High privilege (mutating / disruptive): `Update`, `Delete`, `Pub` + +The server maps every RPC to exactly one (resource, action) pair; granting a +read-only action can never authorize a high-privilege RPC. + +### 11.2 Role Templates + +All commands run against any broker/namesrv of the cluster (ACL 2.0 storage). +Create users first: + +```bash +sh mqadmin createUser -n -u -p +``` + +#### Role A — Read-only observer (dashboard service account) + +Online clients, subscriptions, accumulation, diagnostics, config/route views. + +```bash +sh mqadmin updateAcl -n \ + -s user:rip2-ro \ + -r cluster:proxy.admin.client,cluster:proxy.admin.config,cluster:proxy.admin.route,cluster:proxy.admin.ops \ + -a Get,List \ + -d Allow +``` + +Note: `Get,List` on `proxy.admin.ops` covers the read-only broker-facing RPCs; +the mutating ops RPCs require `Update`/`Delete`/`Pub` and stay denied. + +#### Role B — On-call operator (observer + client diagnostics commands) + +Role A plus the ability to direct telemetry commands (thread stack / verify +message) at a connected client. + +```bash +sh mqadmin updateAcl -n \ + -s user:rip2-oncall \ + -r cluster:proxy.admin.connection \ + -a Update \ + -d Allow +# plus the Role A grant above +``` + +#### Role C — Admin (full control, break-glass) + +Offset reset, subscription deletion, admin send, client diagnostics commands. + +```bash +sh mqadmin updateAcl -n \ + -s user:rip2-admin \ + -r cluster:proxy.admin.client,cluster:proxy.admin.config,cluster:proxy.admin.connection,cluster:proxy.admin.route,cluster:proxy.admin.ops \ + -a Get,List,Update,Delete,Pub \ + -d Allow +``` + +(Keep Role C accounts to a minimum; every use is recorded in the auth audit +log with the `[PROXY-ADMIN-AUDIT]` prefix.) + +#### Environment restriction (recommended) + +Restrict admin access to the operations network via the `-i` sourceIp option: + +```bash +sh mqadmin updateAcl -n \ + -s user:rip2-ro \ + -r cluster:proxy.admin.client \ + -a Get,List \ + -d Allow \ + -i 10.0.0.0/8 +``` + +### 11.3 Fail-Closed Mode + +By default the admin server follows the cluster-wide authentication/authorization +switches (same behavior as the data plane). Fail-closed mode makes the admin +surface refuse to serve anything it cannot fully authorize: + +```properties +# proxy.json / -D grpcAdminServerAuthEnable=true +grpcAdminServerAuthEnable: true +``` + +With `grpcAdminServerAuthEnable=true`, the admin surface requires the cluster +authentication **and** authorization switches to be enabled. If authentication is +off, requests are rejected with `UNAUTHENTICATED`; if authorization is off, they +are rejected with `FAILED_PRECONDITION`. This guarantees the `proxy.admin.*` ACL +is genuinely enforced — use it when the admin port cannot be network-isolated. + +### 11.4 Enabling / Disabling the Surface + +The surface is **off by default** (`grpcAdminServerEnable=false`). To enable it: + +```properties +grpcAdminServerEnable: true # start the admin gRPC server on grpcAdminServerPort +``` + +To keep it disabled (the default), leave `grpcAdminServerEnable: false`, or set +`grpcAdminServerPort` to 0 / negative — the admin gRPC server is then not started +at all. + +### 11.5 Audit + +Every served admin RPC logs subject (Console login user / AK), method, resource, +action and source IP to the auth audit logger; denied requests are logged by the +ACL 2.0 engine itself. This satisfies the four-tuple audit requirement +(Console user + AK + resource + operation). diff --git a/pom.xml b/pom.xml index 9cb13dde1fe..e2a96df4a67 100644 --- a/pom.xml +++ b/pom.xml @@ -127,9 +127,8 @@ 6.0.53 1.0-beta-4 1.4.2 - 2.1.2 1.53.0 - 3.20.1 + 3.24.4 1.2.10 0.9.11 2.9.3 @@ -199,6 +198,7 @@ example container controller + rocketmq-proto proxy tieredstore @@ -663,7 +663,7 @@ ${project.groupId} rocketmq-proto - ${rocketmq-proto.version} + ${project.version} * diff --git a/proxy/BUILD.bazel b/proxy/BUILD.bazel index 8ddaa52382b..f5883a2107c 100644 --- a/proxy/BUILD.bazel +++ b/proxy/BUILD.bazel @@ -57,7 +57,7 @@ java_library( "@maven//:io_opentelemetry_opentelemetry_sdk_common", "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", "@maven//:org_apache_commons_commons_lang3", - "@maven//:org_apache_rocketmq_rocketmq_proto", + "//rocketmq-proto:rocketmq-proto", "@maven//:org_checkerframework_checker_qual", "@maven//:at_yawk_lz4_lz4_java", "@maven//:org_slf4j_slf4j_api", @@ -103,7 +103,7 @@ java_library( "@maven//:io_opentelemetry_opentelemetry_exporter_otlp", "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus", "@maven//:io_opentelemetry_opentelemetry_sdk", - "@maven//:org_apache_rocketmq_rocketmq_proto", + "//rocketmq-proto:rocketmq-proto", "@maven//:org_checkerframework_checker_qual", "@maven//:org_slf4j_slf4j_api", "@maven//:org_springframework_spring_core", diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java index 1b38a19ae6a..5c56f0fe55c 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java @@ -39,7 +39,13 @@ import org.apache.rocketmq.proxy.config.ProxyConfig; import org.apache.rocketmq.proxy.grpc.GrpcServer; import org.apache.rocketmq.proxy.grpc.GrpcServerBuilder; +import org.apache.rocketmq.proxy.grpc.admin.ProxyAdminAuthInterceptor; +import org.apache.rocketmq.proxy.grpc.admin.ProxyAdminForwarder; +import org.apache.rocketmq.proxy.grpc.admin.ProxyAdminGrpcService; +import org.apache.rocketmq.proxy.grpc.v2.DefaultGrpcMessagingActivity; import org.apache.rocketmq.proxy.grpc.v2.GrpcMessagingApplication; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; import org.apache.rocketmq.proxy.metrics.ProxyMetricsManager; import org.apache.rocketmq.proxy.processor.DefaultMessagingProcessor; import org.apache.rocketmq.proxy.processor.MessagingProcessor; @@ -81,10 +87,13 @@ public static void main(String[] args) { TlsCertificateManager tlsCertificateManager = new TlsCertificateManager(); PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(tlsCertificateManager); - // create grpcServer + // create grpcServer (data plane). Capture the application reference so the + // admin server can reuse the SAME GrpcChannelManager / GrpcClientSettingsManager + // that the data plane uses to track online clients. + GrpcMessagingApplication dataPlaneApplication = createServiceProcessor(messagingProcessor); GrpcServer grpcServer = GrpcServerBuilder.newBuilder(executor, ConfigurationManager.getProxyConfig().getGrpcServerPort(), tlsCertificateManager) - .addService(createServiceProcessor(messagingProcessor)) + .addService(dataPlaneApplication) .addService(ChannelzService.newInstance(100)) .addService(ProtoReflectionService.newInstance()) .configInterceptor() @@ -92,6 +101,41 @@ public static void main(String[] args) { .build(); PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(grpcServer); + // Dedicated admin gRPC server (control plane), gated by grpcAdminServerEnable. Reuses + // the data plane's GrpcChannelManager so admin queries can see the data-plane clients. + Integer adminPort = ConfigurationManager.getProxyConfig().getGrpcAdminServerPort(); + if (ConfigurationManager.getProxyConfig().isGrpcAdminServerEnable() + && adminPort != null && adminPort > 0) { + DefaultGrpcMessagingActivity dataPlaneActivity = + (DefaultGrpcMessagingActivity) dataPlaneApplication.getGrpcMessagingActivity(); + GrpcChannelManager sharedChannelManager = dataPlaneActivity.getGrpcChannelManager(); + GrpcClientSettingsManager sharedSettingsManager = dataPlaneActivity.getGrpcClientSettingsManager(); + DefaultMessagingProcessor defaultProcessor = (DefaultMessagingProcessor) messagingProcessor; + + // Forwards client-targeted admin RPCs to the proxy that owns the client channel. + ProxyAdminForwarder adminForwarder = + new ProxyAdminForwarder(defaultProcessor.getServiceManager()); + PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(adminForwarder); + + ProxyAdminGrpcService adminService = new ProxyAdminGrpcService( + defaultProcessor.getServiceManager(), + messagingProcessor, + sharedChannelManager, + sharedSettingsManager, + adminForwarder); + PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(adminService); + GrpcServer adminGrpcServer = GrpcServerBuilder.newBuilder(executor, adminPort, tlsCertificateManager) + .addService(adminService) + // authentication has to see the channel id HeaderInterceptor derives from the + // transport, so it is ordered after the standard pipeline rather than appended + .configInterceptor(new ProxyAdminAuthInterceptor( + ConfigurationManager.getAuthConfig(), messagingProcessor)) + .shutdownTime(ConfigurationManager.getProxyConfig().getGrpcShutdownTimeSeconds(), TimeUnit.SECONDS) + .build(); + PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(adminGrpcServer); + log.info("admin gRPC server will start on port {}", adminPort); + } + RemotingProtocolServer remotingServer = new RemotingProtocolServer(messagingProcessor, tlsCertificateManager); PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(remotingServer); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java index a7896c11e07..b2659be97e8 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java @@ -89,6 +89,33 @@ public class ProxyConfig implements ConfigFile { */ private String proxyMode = ProxyMode.CLUSTER.name(); private Integer grpcServerPort = 8081; + + /** + * Dedicated gRPC port for the Proxy Admin service. When > 0 the proxy + * starts an independent admin gRPC server (separate ACL scope, isolated + * traffic) in addition to the data-plane gRPC server. + */ + private Integer grpcAdminServerPort = 8088; + /** + * global kill switch for the Proxy Admin surface. When false the + * admin gRPC server is not started at all, regardless of {@link #grpcAdminServerPort}. + * Defaults to false: the admin surface is opt-in and must be explicitly enabled. + */ + private boolean grpcAdminServerEnable = false; + /** + * When true the admin server enforces credential checks even if the cluster-wide + * authentication switch is off; requests without verifiable credentials are rejected + * (fail-closed mode). When false the admin server follows the cluster-wide + * authenticationEnabled/authorizationEnabled switches (same behavior as the data plane). + */ + private boolean grpcAdminServerAuthEnable = false; + /** + * Per-request timeout in milliseconds for the broker calls an admin RPC fans out to. + * A single RPC may query several brokers concurrently, so this bounds each hop rather + * than the whole call. + */ + private long grpcAdminServerRequestTimeoutMillis = 3000L; + private long grpcShutdownTimeSeconds = 30; private int grpcBossLoopNum = 1; private int grpcWorkerLoopNum = PROCESSOR_NUMBER * 2; @@ -483,6 +510,38 @@ public void setGrpcServerPort(Integer grpcServerPort) { this.grpcServerPort = grpcServerPort; } + public Integer getGrpcAdminServerPort() { + return grpcAdminServerPort; + } + + public void setGrpcAdminServerPort(Integer grpcAdminServerPort) { + this.grpcAdminServerPort = grpcAdminServerPort; + } + + public boolean isGrpcAdminServerEnable() { + return grpcAdminServerEnable; + } + + public void setGrpcAdminServerEnable(boolean grpcAdminServerEnable) { + this.grpcAdminServerEnable = grpcAdminServerEnable; + } + + public boolean isGrpcAdminServerAuthEnable() { + return grpcAdminServerAuthEnable; + } + + public void setGrpcAdminServerAuthEnable(boolean grpcAdminServerAuthEnable) { + this.grpcAdminServerAuthEnable = grpcAdminServerAuthEnable; + } + + public long getGrpcAdminServerRequestTimeoutMillis() { + return grpcAdminServerRequestTimeoutMillis; + } + + public void setGrpcAdminServerRequestTimeoutMillis(long grpcAdminServerRequestTimeoutMillis) { + this.grpcAdminServerRequestTimeoutMillis = grpcAdminServerRequestTimeoutMillis; + } + public long getGrpcShutdownTimeSeconds() { return grpcShutdownTimeSeconds; } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java index 35d035b9938..0619d09ba5b 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.proxy.grpc; import io.grpc.BindableService; +import io.grpc.ServerBuilder; import io.grpc.ServerInterceptor; import io.grpc.ServerServiceDefinition; import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; @@ -141,6 +142,13 @@ public GrpcServerBuilder addService(ServerServiceDefinition service) { return this; } + /** + * Registers an interceptor that runs before the pipeline installed by + * {@link #configInterceptor(ServerInterceptor...)}, because gRPC invokes server interceptors in + * reverse registration order. An interceptor that needs the normalized headers (the + * transport-derived channel id, the cleared authorization subject) must NOT be appended here; + * pass it to {@link #configInterceptor(ServerInterceptor...)} instead. + */ public GrpcServerBuilder appendInterceptor(ServerInterceptor interceptor) { this.serverBuilder.intercept(interceptor); return this; @@ -174,11 +182,36 @@ private EventLoopGroup newEventLoopGroup(int threads) { return enableEpoll ? new EpollEventLoopGroup(threads) : new NioEventLoopGroup(threads); } - public GrpcServerBuilder configInterceptor() { - this.serverBuilder - .intercept(new GlobalExceptionInterceptor()) - .intercept(new ContextInterceptor()) - .intercept(new HeaderInterceptor()); + /** + * Installs the standard pipeline, optionally with interceptors that must observe its normalized + * headers. + * + * @param postHeaderInterceptors interceptors to run after {@link HeaderInterceptor}, typically + * authentication + */ + public GrpcServerBuilder configInterceptor(ServerInterceptor... postHeaderInterceptors) { + configureInterceptors(this.serverBuilder, postHeaderInterceptors); return this; } + + /** + * Orders the standard pipeline and the interceptors that depend on it. gRPC invokes server + * interceptors in reverse registration order, so {@code postHeaderInterceptors} are registered + * first precisely so that they run last. + * + *

Authentication depends on this order. {@link HeaderInterceptor} replaces the inbound + * {@code x-mq-channel-id} with the channel id derived from the transport and clears the + * authorization subject. Authentication results are cached per channel id, so authenticating on + * the client-supplied value would let a request on one connection reuse the cached success of + * another connection and skip signature verification. + */ + static void configureInterceptors(ServerBuilder serverBuilder, + ServerInterceptor... postHeaderInterceptors) { + for (ServerInterceptor interceptor : postHeaderInterceptors) { + serverBuilder.intercept(interceptor); + } + serverBuilder.intercept(new GlobalExceptionInterceptor()); + serverBuilder.intercept(new ContextInterceptor()); + serverBuilder.intercept(new HeaderInterceptor()); + } } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/admin/AdminModelConverter.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/admin/AdminModelConverter.java new file mode 100644 index 00000000000..2b7bb918d6f --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/admin/AdminModelConverter.java @@ -0,0 +1,372 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.admin; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.DescribeGroupAccumulationResponse.GroupAccumulation; +import apache.rocketmq.v2.DescribeTopicStatusResponse; +import apache.rocketmq.v2.GetTopicRouteResponse; +import apache.rocketmq.v2.MessageQueueItem; +import apache.rocketmq.v2.MessageType; +import apache.rocketmq.v2.QueryTimeSpanResponse; +import apache.rocketmq.v2.Resource; +import apache.rocketmq.v2.Status; +import com.alibaba.fastjson2.JSON; +import com.google.protobuf.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.filter.ExpressionType; +import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; +import org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper; +import org.apache.rocketmq.remoting.protocol.body.Connection; +import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; + +/** + * Bridge between the broker's internal wire types ({@code org.apache.rocketmq.remoting.*}) and the + * gRPC admin contract ({@code apache.rocketmq.v2.*}, generated from rocketmq-apis). + * + *

This is the only class that knows both worlds: {@link ProxyAdminGrpcService} stays + * protocol-pure and the admin gateway stays remoting-pure, so neither layer leaks into the other. + * Converting a whole message body is deliberately NOT done here — the data plane already has + * {@code GrpcConverter.buildMessage(MessageExt)}, which is reused instead of being duplicated. + */ +final class AdminModelConverter { + + private AdminModelConverter() { + } + + static Status ok() { + return Status.newBuilder().setCode(Code.OK).build(); + } + + // ------------------------------------------------------------------------- + // topic + // ------------------------------------------------------------------------- + + static DescribeTopicStatusResponse toTopicStatus(TopicConfig topicConfig, String topic) { + DescribeTopicStatusResponse.Builder builder = DescribeTopicStatusResponse.newBuilder() + .setStatus(ok()) + .setTopicMessageType(toMessageType(topicConfig == null ? null : topicConfig.getTopicMessageType())); + if (topicConfig != null) { + // a topic creation timestamp is not recorded by the broker, so create_timestamp and + // tags stay unset rather than being filled with a value that would only look real + builder.setDescription("readQueueNums=" + topicConfig.getReadQueueNums() + + ", writeQueueNums=" + topicConfig.getWriteQueueNums() + + ", perm=" + topicConfig.getPerm()); + } else { + builder.setDescription("topic config not available for " + topic); + } + return builder.build(); + } + + static MessageType toMessageType(TopicMessageType topicMessageType) { + if (topicMessageType == null) { + return MessageType.MESSAGE_TYPE_UNSPECIFIED; + } + switch (topicMessageType) { + case NORMAL: + return MessageType.NORMAL; + case FIFO: + return MessageType.FIFO; + case DELAY: + return MessageType.DELAY; + case TRANSACTION: + return MessageType.TRANSACTION; + case LITE: + return MessageType.LITE; + case UNSPECIFIED: + case MIXED: + case PRIORITY: + default: + // MessageType.UNRECOGNIZED cannot be set on a builder, so anything this proxy does + // not understand degrades to UNSPECIFIED instead of failing the whole RPC + return MessageType.MESSAGE_TYPE_UNSPECIFIED; + } + } + + static GetTopicRouteResponse toTopicRoute(TopicRouteData topicRouteData) { + return GetTopicRouteResponse.newBuilder() + .setStatus(ok()) + .setTopicRouteData(topicRouteData == null ? "{}" : JSON.toJSONString(topicRouteData)) + .build(); + } + + // ------------------------------------------------------------------------- + // consume time span + // ------------------------------------------------------------------------- + + /** + * The broker already computes min/max/consume timestamps and the delay per queue + * (RequestCode.QUERY_CONSUME_TIME_SPAN), so the response maps 1:1 and no offset table lookup + * is needed here. + */ + static QueryTimeSpanResponse toQueryTimeSpan(List queueTimeSpans) { + QueryTimeSpanResponse.Builder builder = QueryTimeSpanResponse.newBuilder().setStatus(ok()); + if (queueTimeSpans == null) { + return builder.build(); + } + for (QueueTimeSpan span : queueTimeSpans) { + if (span == null || span.getMessageQueue() == null) { + continue; + } + long delay = span.getDelayTime(); + builder.addQueueTimeSpanList(QueryTimeSpanResponse.QueueTimeSpan.newBuilder() + .setMessageQueue(toMessageQueue(span.getMessageQueue())) + .setMinTimestamp(span.getMinTimeStamp()) + .setMaxTimestamp(span.getMaxTimeStamp()) + .setConsumeTimestamp(span.getConsumeTimeStamp()) + .setDelayTimeMs(Math.max(delay, 0L)) + .build()); + } + return builder.build(); + } + + static apache.rocketmq.v2.MessageQueue toMessageQueue(org.apache.rocketmq.common.message.MessageQueue mq) { + return apache.rocketmq.v2.MessageQueue.newBuilder() + .setTopic(Resource.newBuilder().setName(mq.getTopic()).build()) + .setBroker(apache.rocketmq.v2.Broker.newBuilder().setName(mq.getBrokerName()).build()) + .setId(mq.getQueueId()) + .build(); + } + + // ------------------------------------------------------------------------- + // accumulation + // ------------------------------------------------------------------------- + + /** + * Aggregates consume stats collected from every broker of the cluster. + * + *

Per the contract {@code accumulation = inflight + ready}, where inflight counts messages + * already delivered but not yet acknowledged ({@code pullOffset - consumerOffset}) and ready + * counts messages still waiting ({@code brokerOffset - pullOffset}). Pop retry topics are + * folded into their normal topic so a caller asking about a topic sees its whole backlog; the + * pull retry topic is counted in the group total but kept out of the per-topic map because it + * is not a user-visible topic. + * + * @param statsByBroker consume stats per broker address; a null topic filter means the broker + * returned every topic the group has offsets for + * @param group consumer group, needed to recognise its retry topics + */ + static AccumulationResult toAccumulation(Map statsByBroker, String group) { + long totalDiff = 0L; + long totalInflight = 0L; + long earliestLastConsume = Long.MAX_VALUE; + Map byTopic = new HashMap<>(); + + String pullRetryTopic = MixAll.getRetryTopic(group); + for (ConsumeStats stats : statsByBroker.values()) { + if (stats == null || stats.getOffsetTable() == null) { + continue; + } + for (Map.Entry entry : + stats.getOffsetTable().entrySet()) { + org.apache.rocketmq.common.message.MessageQueue mq = entry.getKey(); + OffsetWrapper wrapper = entry.getValue(); + if (mq == null || wrapper == null) { + continue; + } + long diff = Math.max(wrapper.getBrokerOffset() - wrapper.getConsumerOffset(), 0L); + long inflight = Math.max(wrapper.getPullOffset() - wrapper.getConsumerOffset(), 0L); + if (inflight > diff) { + inflight = diff; + } + totalDiff += diff; + totalInflight += inflight; + if (diff > 0 && wrapper.getLastTimestamp() > 0) { + earliestLastConsume = Math.min(earliestLastConsume, wrapper.getLastTimestamp()); + } + + String topic = mq.getTopic(); + if (pullRetryTopic.equals(topic)) { + continue; + } + String userTopic = KeyBuilder.parseNormalTopic(topic, group); + long[] acc = byTopic.computeIfAbsent(userTopic, k -> new long[3]); + acc[0] += diff; + acc[1] += inflight; + if (diff > 0 && wrapper.getLastTimestamp() > 0 && (acc[2] == 0L || wrapper.getLastTimestamp() < acc[2])) { + acc[2] = wrapper.getLastTimestamp(); + } + } + } + + AccumulationResult result = new AccumulationResult(buildAccumulation(totalDiff, totalInflight, earliestLastConsume)); + for (Map.Entry entry : byTopic.entrySet()) { + long[] acc = entry.getValue(); + result.byTopic.put(entry.getKey(), buildAccumulation(acc[0], acc[1], acc[2])); + } + return result; + } + + private static GroupAccumulation buildAccumulation(long accumulation, long inflight, long lastConsumeTimestamp) { + GroupAccumulation.Builder builder = GroupAccumulation.newBuilder() + .setAccumulation(accumulation) + .setInflightMessages(inflight) + .setReadyMessages(Math.max(accumulation - inflight, 0L)); + if (lastConsumeTimestamp > 0 && lastConsumeTimestamp != Long.MAX_VALUE) { + builder.setLastConsumeTimestamp(lastConsumeTimestamp); + if (accumulation > 0) { + long delayMillis = Math.max(System.currentTimeMillis() - lastConsumeTimestamp, 0L); + builder.setDeliverDelayTime(Duration.newBuilder() + .setSeconds(delayMillis / 1000L) + .setNanos((int) (delayMillis % 1000L) * 1_000_000) + .build()); + } + } + return builder.build(); + } + + static final class AccumulationResult { + final GroupAccumulation total; + final Map byTopic = new HashMap<>(); + + private AccumulationResult(GroupAccumulation total) { + this.total = total; + } + } + + // ------------------------------------------------------------------------- + // clients + // ------------------------------------------------------------------------- + + /** + * Builds a {@code ClientInfo} from a broker-side connection. The broker sees consumers that + * registered through any proxy of the cluster, which is why the admin surface prefers this + * over scanning only the local gRPC channels. + */ + static apache.rocketmq.v2.ClientInfo toClientInfo(Connection connection, + apache.rocketmq.v2.MessageModel messageModel) { + apache.rocketmq.v2.ClientInfo.Builder builder = apache.rocketmq.v2.ClientInfo.newBuilder() + .setClientId(connection.getClientId() == null ? "" : connection.getClientId()); + if (connection.getLanguage() != null) { + builder.setLanguage(connection.getLanguage().name()); + } + builder.setVersion(String.valueOf(connection.getVersion())); + String clientAddr = connection.getClientAddr(); + if (clientAddr != null && !clientAddr.isEmpty()) { + // clientAddr is "ip:port"; egress_ip is documented as the address observed by the server + int idx = clientAddr.lastIndexOf('@'); + String addr = idx >= 0 ? clientAddr.substring(idx + 1) : clientAddr; + int colon = addr.lastIndexOf(':'); + builder.setEgressIp(colon > 0 ? addr.substring(0, colon) : addr); + builder.setHostname(idx >= 0 ? clientAddr.substring(0, idx) : ""); + } + if (messageModel != null) { + builder.setMessageModel(messageModel); + } + return builder.build(); + } + + static apache.rocketmq.v2.MessageModel toMessageModel( + org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel messageModel) { + if (messageModel == null) { + return apache.rocketmq.v2.MessageModel.MESSAGE_MODEL_UNSPECIFIED; + } + switch (messageModel) { + case BROADCASTING: + return apache.rocketmq.v2.MessageModel.BROADCASTING; + case CLUSTERING: + default: + return apache.rocketmq.v2.MessageModel.CLUSTERING; + } + } + + static apache.rocketmq.v2.FilterExpression toFilterExpression(String expressionType, String expression) { + apache.rocketmq.v2.FilterType type = ExpressionType.SQL92.equals(expressionType) + ? apache.rocketmq.v2.FilterType.SQL : apache.rocketmq.v2.FilterType.TAG; + return apache.rocketmq.v2.FilterExpression.newBuilder() + .setType(type) + .setExpression(expression == null ? "" : expression) + .build(); + } + + /** + * Full mapping of a remoting client's running info. A gRPC v2 client can only report its + * thread stack (the telemetry contract has no reply message carrying the other three fields), + * so this is reached only for remoting clients. + */ + static apache.rocketmq.v2.ConsumerRunningInfo toConsumerRunningInfo( + org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo runningInfo) { + apache.rocketmq.v2.ConsumerRunningInfo.Builder builder = apache.rocketmq.v2.ConsumerRunningInfo.newBuilder(); + if (runningInfo == null) { + return builder.build(); + } + if (runningInfo.getProperties() != null) { + for (String name : runningInfo.getProperties().stringPropertyNames()) { + builder.putProperties(name, runningInfo.getProperties().getProperty(name)); + } + } + if (runningInfo.getSubscriptionSet() != null) { + for (org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData data : + runningInfo.getSubscriptionSet()) { + builder.putSubscriptions(data.getTopic(), toFilterExpression(data.getExpressionType(), data.getSubString())); + } + } + if (runningInfo.getMqTable() != null) { + for (Map.Entry entry : runningInfo.getMqTable().entrySet()) { + org.apache.rocketmq.remoting.protocol.body.ProcessQueueInfo info = entry.getValue(); + if (info == null) { + continue; + } + builder.addMessageQueueTable(MessageQueueItem.newBuilder() + .setMessageQueue(toMessageQueue(entry.getKey())) + .setProcessQueueInfo(apache.rocketmq.v2.ProcessQueueInfo.newBuilder() + .setCommitOffset(info.getCommitOffset()) + .setCachedMsgMinOffset(info.getCachedMsgMinOffset()) + .setCachedMsgMaxOffset(info.getCachedMsgMaxOffset()) + .setCachedMsgCount(info.getCachedMsgCount()) + .setCachedMsgSizeInMib(info.getCachedMsgSizeInMiB()) + .setTransactionMsgMinOffset(info.getTransactionMsgMinOffset()) + .setTransactionMsgMaxOffset(info.getTransactionMsgMaxOffset()) + .setTransactionMsgCount(info.getTransactionMsgCount()) + .setLocked(info.isLocked()) + .setTryUnlockTimes(info.getTryUnlockTimes()) + .setLastLockTimestamp(info.getLastLockTimestamp()) + .setDropped(info.isDroped()) + .setLastPullTimestamp(info.getLastPullTimestamp()) + .setLastConsumeTimestamp(info.getLastConsumeTimestamp()) + .build()) + .build()); + } + } + if (runningInfo.getStatusTable() != null) { + for (Map.Entry entry : + runningInfo.getStatusTable().entrySet()) { + org.apache.rocketmq.remoting.protocol.body.ConsumeStatus status = entry.getValue(); + if (status == null) { + continue; + } + builder.putConsumeStatusTable(entry.getKey(), apache.rocketmq.v2.ConsumeStatus.newBuilder() + .setReceiveRt(status.getPullRT()) + .setReceiveTps(status.getPullTPS()) + .setConsumeRt(status.getConsumeRT()) + .setConsumeOkTps(status.getConsumeOKTPS()) + .setConsumeFailedTps(status.getConsumeFailedTPS()) + .setConsumeFailedMsgs(status.getConsumeFailedMsgs()) + .build()); + } + } + return builder.build(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminAuthInterceptor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminAuthInterceptor.java new file mode 100644 index 00000000000..9f4d7052839 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminAuthInterceptor.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.admin; + +import io.grpc.Grpc; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.auth.authentication.context.DefaultAuthenticationContext; +import org.apache.rocketmq.auth.authentication.exception.AuthenticationException; +import org.apache.rocketmq.auth.authentication.factory.AuthenticationFactory; +import org.apache.rocketmq.auth.authentication.model.User; +import org.apache.rocketmq.auth.authorization.AuthorizationEvaluator; +import org.apache.rocketmq.auth.authentication.AuthenticationEvaluator; +import org.apache.rocketmq.auth.authorization.context.DefaultAuthorizationContext; +import org.apache.rocketmq.auth.authorization.exception.AuthorizationException; +import org.apache.rocketmq.auth.authorization.factory.AuthorizationFactory; +import org.apache.rocketmq.auth.authorization.model.Resource; +import org.apache.rocketmq.auth.config.AuthConfig; +import org.apache.rocketmq.common.action.Action; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.resource.ResourcePattern; +import org.apache.rocketmq.common.resource.ResourceType; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; + +/** + * authorization interceptor for the dedicated Proxy Admin gRPC server. + * + *

Every admin RPC is bound to a dedicated ACL 2.0 resource under the + * {@code proxy.admin.*} namespace with a distinct action, giving true + * read-only / high-privilege isolation on top of the standard ACL 2.0 policy + * engine: + * + * 

+ *   proxy.admin.client      ListConsumerConnection / ListSubscription / DescribeSubscription /
+ *                           DescribeGroupAccumulation / GetConsumerRunningInfo / QueryTimeSpan (GET/LIST)
+ *   proxy.admin.config      ChangeLogLevel (UPDATE)
+ *   proxy.admin.connection  PrintThreadStackTrace / VerifyMessage (UPDATE, high privilege)
+ *   proxy.admin.route       GetTopicRoute (GET)
+ *   proxy.admin.ops         broker-facing operations served by the Admin service
+ *                           (GET/LIST for queries; UPDATE/DELETE/PUB for mutations)
+ *
+ * + *

Resources are modeled as {@code CLUSTER}-typed literal resources + * (resource key {@code cluster:proxy.admin.}) because the ACL 2.0 + * resource model only defines cluster/namespace/topic/group types; a + * cluster-typed literal gives exact least-privilege matching without + * colliding with real cluster names. + * + *

Behavior modes: + *

    + *
  • Cluster auth disabled and {@code grpcAdminServerAuthEnable=false}: the + * admin surface is open (same semantics as the data plane).
    • + *
  • Cluster auth enabled: requests are authenticated from the standard + * {@code Authorization} gRPC metadata and authorized against the + * per-method {@code proxy.admin.*} resource, exactly like the data + * plane does for topic/group resources.
    • + *
  • {@code grpcAdminServerAuthEnable=true}: fail-closed mode. The admin + * surface must be able to both authenticate the caller and enforce the + * per-method {@code proxy.admin.*} ACL, so it requires the cluster-wide + * authentication AND authorization switches to be on. If either is off + * every request is refused rather than served without a real permission + * check: the authorization evaluator is itself gated by the cluster + * authorization switch, so with authorization off it would silently pass + * every call (including the destructive ones).
    • + *
+ */ +public class ProxyAdminAuthInterceptor implements ServerInterceptor { + + private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + private static final Logger logAudit = LoggerFactory.getLogger(LoggerName.ROCKETMQ_AUTH_AUDIT_LOGGER_NAME); + + public static final String RESOURCE_CLIENT = "proxy.admin.client"; + public static final String RESOURCE_CONFIG = "proxy.admin.config"; + public static final String RESOURCE_CONNECTION = "proxy.admin.connection"; + public static final String RESOURCE_ROUTE = "proxy.admin.route"; + public static final String RESOURCE_OPS = "proxy.admin.ops"; + + private static final Map METHOD_PERMISSIONS = new HashMap<>(); + + static { + // Admin service (rocketmq-apis v2 admin.proto) served on the dedicated admin server. + METHOD_PERMISSIONS.put("GetProxyRuntimeStats", new ResourceAction(RESOURCE_OPS, Action.GET)); + METHOD_PERMISSIONS.put("GetTopicRoute", new ResourceAction(RESOURCE_ROUTE, Action.GET)); + METHOD_PERMISSIONS.put("DescribeTopicStatus", new ResourceAction(RESOURCE_OPS, Action.GET)); + METHOD_PERMISSIONS.put("ListSubscription", new ResourceAction(RESOURCE_CLIENT, Action.LIST)); + METHOD_PERMISSIONS.put("DescribeSubscription", new ResourceAction(RESOURCE_CLIENT, Action.GET)); + METHOD_PERMISSIONS.put("ListConsumerConnection", new ResourceAction(RESOURCE_CLIENT, Action.LIST)); + METHOD_PERMISSIONS.put("DescribeGroupAccumulation", new ResourceAction(RESOURCE_CLIENT, Action.GET)); + METHOD_PERMISSIONS.put("GetConsumerRunningInfo", new ResourceAction(RESOURCE_CLIENT, Action.GET)); + METHOD_PERMISSIONS.put("QueryTimeSpan", new ResourceAction(RESOURCE_CLIENT, Action.GET)); + METHOD_PERMISSIONS.put("QueryMessage", new ResourceAction(RESOURCE_OPS, Action.GET)); + METHOD_PERMISSIONS.put("ChangeLogLevel", new ResourceAction(RESOURCE_CONFIG, Action.UPDATE)); + // High-privilege mutations: strictly separated from the read-only actions above. + METHOD_PERMISSIONS.put("DeleteSubscription", new ResourceAction(RESOURCE_OPS, Action.DELETE)); + METHOD_PERMISSIONS.put("ResetGroupOffset", new ResourceAction(RESOURCE_OPS, Action.UPDATE)); + METHOD_PERMISSIONS.put("AdminSendMessage", new ResourceAction(RESOURCE_OPS, Action.PUB)); + METHOD_PERMISSIONS.put("PrintThreadStackTrace", new ResourceAction(RESOURCE_CONNECTION, Action.UPDATE)); + METHOD_PERMISSIONS.put("VerifyMessage", new ResourceAction(RESOURCE_CONNECTION, Action.UPDATE)); + } + + private final AuthConfig authConfig; + private final AuthenticationEvaluator authenticationEvaluator; + private final AuthorizationEvaluator authorizationEvaluator; + + public ProxyAdminAuthInterceptor(AuthConfig authConfig, MessagingProcessor messagingProcessor) { + this.authConfig = authConfig; + this.authenticationEvaluator = AuthenticationFactory.getEvaluator(authConfig, + messagingProcessor::getMetadataService); + this.authorizationEvaluator = AuthorizationFactory.getEvaluator(authConfig, + messagingProcessor::getMetadataService); + } + + @Override + public ServerCall.Listener interceptCall(ServerCall call, Metadata headers, + ServerCallHandler next) { + String method = call.getMethodDescriptor().getBareMethodName(); + try { + ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + boolean requireAuth = proxyConfig != null && proxyConfig.isGrpcAdminServerAuthEnable(); + boolean authnEnabled = authConfig != null && authConfig.isAuthenticationEnabled(); + boolean authzEnabled = authConfig != null && authConfig.isAuthorizationEnabled(); + + // Open mode: identical semantics to the data plane when cluster auth is off. + if (!requireAuth && !authnEnabled && !authzEnabled) { + return next.startCall(call, headers); + } + + // Fail-closed: credentials are demanded but cannot be verified at all. + if (requireAuth && !authnEnabled) { + call.close(Status.UNAUTHENTICATED.withDescription( + "grpcAdminServerAuthEnable is on but cluster authenticationEnabled is off; " + + "enable authentication before using the admin surface in fail-closed mode"), new Metadata()); + return noopListener(); + } + + // Fail-closed: the admin surface is locked down, but the ACL engine that enforces the + // per-method proxy.admin.* permission is gated by the cluster authorization switch. + // With that switch off the authorization evaluator returns without checking anything, + // so requiring auth here without authorization on would let any authenticated identity + // reach every admin RPC, including the destructive ones. Refuse instead of serving + // an unauthorized call. + if (requireAuth && !authzEnabled) { + call.close(Status.FAILED_PRECONDITION.withDescription( + "grpcAdminServerAuthEnable is on but cluster authorizationEnabled is off; " + + "enable authorization so the proxy.admin.* ACL can be enforced before " + + "using the admin surface in fail-closed mode"), new Metadata()); + return noopListener(); + } + + String username = null; + if (authnEnabled || requireAuth) { + DefaultAuthenticationContext authenticationContext = buildAuthenticationContext(call, headers); + username = authenticationContext.getUsername(); + if (StringUtils.isBlank(username)) { + if (requireAuth) { + call.close(Status.UNAUTHENTICATED.withDescription("missing credentials for proxy admin"), + new Metadata()); + return noopListener(); + } + authenticationEvaluator.evaluate(authenticationContext); + } else { + authenticationEvaluator.evaluate(authenticationContext); + } + } + + ResourceAction resourceAction = METHOD_PERMISSIONS.get(method); + // After the fail-closed guards above, requireAuth implies authzEnabled, so the ACL is + // enforced exactly when the cluster authorization switch is on; there is no path where + // requireAuth alone would ask the (globally gated) evaluator to enforce and be ignored. + if (resourceAction != null && authzEnabled) { + if (StringUtils.isBlank(username)) { + call.close(Status.UNAUTHENTICATED.withDescription("missing credentials for proxy admin"), + new Metadata()); + return noopListener(); + } + DefaultAuthorizationContext authorizationContext = DefaultAuthorizationContext.of( + User.of(username), + Resource.of(ResourceType.CLUSTER, resourceAction.resource, ResourcePattern.LITERAL), + resourceAction.action, + resolveSourceIp(call)); + authorizationContext.setRpcCode(call.getMethodDescriptor().getFullMethodName()); + authorizationEvaluator.evaluate(Collections.singletonList(authorizationContext)); + } + + logAudit.info("[PROXY-ADMIN-AUDIT] subject = {} method = {} resource = {} action = {} sourceIp = {}", + StringUtils.isBlank(username) ? "anonymous" : username, method, + resourceAction == null ? "unmapped" : resourceAction.resource, + resourceAction == null ? "unknown" : resourceAction.action.getName(), + resolveSourceIp(call)); + return next.startCall(call, headers); + } catch (AuthenticationException e) { + log.warn("admin authentication failed. method:{}, cause:{}", method, e.getMessage()); + call.close(Status.UNAUTHENTICATED.withDescription(e.getMessage()), new Metadata()); + return noopListener(); + } catch (AuthorizationException e) { + log.warn("admin authorization denied. method:{}, cause:{}", method, e.getMessage()); + call.close(Status.PERMISSION_DENIED.withDescription(e.getMessage()), new Metadata()); + return noopListener(); + } catch (Throwable t) { + log.error("admin auth interceptor error. method:{}", method, t); + call.close(Status.INTERNAL.withDescription(t.getMessage()), new Metadata()); + return noopListener(); + } + } + + private DefaultAuthenticationContext buildAuthenticationContext(ServerCall call, + Metadata headers) { + // The builder only uses the message for its descriptor name; pass the shared + // Status default instance and overwrite rpcCode with the real admin method. + Object context = AuthenticationFactory.newContext(authConfig, headers, + apache.rocketmq.v2.Status.getDefaultInstance()); + if (!(context instanceof DefaultAuthenticationContext)) { + throw new AuthenticationException("unsupported authentication context type for proxy admin"); + } + DefaultAuthenticationContext authenticationContext = (DefaultAuthenticationContext) context; + authenticationContext.setRpcCode(call.getMethodDescriptor().getFullMethodName()); + return authenticationContext; + } + + private static String resolveSourceIp(ServerCall call) { + try { + InetSocketAddress remoteAddress = (InetSocketAddress) call.getAttributes() + .get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR); + if (remoteAddress != null && remoteAddress.getAddress() != null) { + return remoteAddress.getAddress().getHostAddress(); + } + } catch (Throwable ignore) { + // best-effort only + } + return ""; + } + + private static ServerCall.Listener noopListener() { + return new ServerCall.Listener() { + }; + } + + static ResourceAction resolveResourceAction(String method) { + return METHOD_PERMISSIONS.get(method); + } + + static final class ResourceAction { + final String resource; + final Action action; + + ResourceAction(String resource, Action action) { + this.resource = resource; + this.action = action; + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminForwarder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminForwarder.java new file mode 100644 index 00000000000..7b806791a2b --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminForwarder.java @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.admin; + +import apache.rocketmq.v2.AdminGrpc; +import io.grpc.Context; +import io.grpc.ManagedChannel; +import io.grpc.Metadata; +import io.grpc.Status; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import io.grpc.stub.MetadataUtils; +import io.grpc.stub.StreamObserver; +import io.netty.channel.Channel; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerManager; +import org.apache.rocketmq.common.constant.GrpcConstants; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.utils.StartAndShutdown; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.proxy.common.channel.ChannelHelper; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.proxy.processor.channel.RemoteChannel; +import org.apache.rocketmq.proxy.service.ServiceManager; +import org.apache.rocketmq.remoting.common.TlsMode; +import org.apache.rocketmq.remoting.netty.TlsSystemConfig; + +/** + * Forwards a Proxy Admin RPC to the peer proxy that actually owns the target client. + * + *

A gRPC client keeps a single telemetry stream, so only the proxy holding that stream can ask + * the client for a thread dump, a verify-message run or its running info. {@code + * GrpcChannelManager#getChannel(clientId)} is therefore local-only and returns {@code null} for a + * client connected elsewhere. {@link ConsumerManager#findChannel(String, String)} on the other hand + * also sees clients registered on peer proxies: {@code HeartbeatSyncer} replicates them as + * {@link RemoteChannel} instances whose {@link RemoteChannel#getRemoteProxyIp()} is the owning + * proxy's {@code ProxyConfig.localServeAddr}. That is exactly the routing hint this class uses. + * + *

Peer admin port assumption. The admin port is not published anywhere in the heartbeat + * sync payload ({@code HeartbeatSyncerData} carries {@code remotingListenPort} and + * {@code grpcServerPort}, not the admin port), so this class assumes the admin port is uniform + * across the cluster and dials the peer at {@code :}. A cluster that runs heterogeneous admin ports would need + * the port added to the heartbeat sync record first. + * + *

TLS. Every proxy gRPC server installs {@code ProxyAndTlsProtocolNegotiator}. In the + * default configuration ({@code tlsTestModeEnable=true}) each JVM generates a fresh + * {@code SelfSignedCertificate}, so no peer can possibly validate it; because + * {@code TlsSystemConfig.tlsMode} defaults to {@code permissive} the plaintext handshake is + * accepted and used. Otherwise an {@link InsecureTrustManagerFactory}-based client SslContext is + * used: there is no cluster-wide CA distribution for service-to-service calls, and the peer is + * addressed by bare IP, which no sane certificate carries as a SAN. + * + *

Auth. {@code ProxyAdminAuthInterceptor} derives the caller identity from the inbound + * {@code authorization} + {@code x-mq-date-time} metadata and there is no service-to-service + * identity in the open-source proxy. A forwarded call therefore impersonates the original caller: + * every inbound metadata key is copied verbatim onto the outbound call. The caller must already be + * authorized for the admin resource on the entry proxy, and the peer re-runs the very same + * authentication/authorization evaluation against those copied credentials. + * + *

Loop prevention. A forwarded call carries {@link GrpcConstants#ADMIN_FORWARDED}. The + * receiving proxy detects it with {@link #isInboundForwarded()} and must serve the call locally + * instead of forwarding again, which also stops a two-proxy ping-pong when both sides see the + * client as remote. + */ +public class ProxyAdminForwarder implements StartAndShutdown { + + private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + /** + * Wire value of {@link GrpcConstants#ADMIN_FORWARDED}. + */ + private static final String FORWARDED_VALUE = "true"; + + /** + * gRPC reserves the {@code grpc-} prefix; {@link Metadata.Key#of} rejects such names. + */ + private static final String RESERVED_KEY_PREFIX = "grpc-"; + + private static final long SHUTDOWN_GRACE_SECONDS = 5L; + + private final ServiceManager serviceManager; + + /** + * One channel + stub per remote proxy address ({@code :}). Channels are created + * lazily and are long-lived: a proxy set is small and stable, so re-handshaking per RPC would + * only add latency. + */ + private final ConcurrentMap peerConnections = new ConcurrentHashMap<>(); + + public ProxyAdminForwarder(ServiceManager serviceManager) { + this.serviceManager = serviceManager; + } + + /** + * True when this call already arrived forwarded from a peer proxy; such a call must never be + * forwarded again. + */ + public boolean isInboundForwarded() { + Metadata inbound = GrpcConstants.METADATA.get(Context.current()); + if (inbound == null) { + return false; + } + return FORWARDED_VALUE.equalsIgnoreCase(inbound.get(GrpcConstants.ADMIN_FORWARDED)); + } + + /** + * If the client identified by (group, clientId) is registered on another proxy, forward the RPC + * there and return true — the caller must then return immediately and let the peer own the + * observer. Returns false when the client is local, unknown, or forwarding is not possible, so + * the caller proceeds with its normal local handling. + * + *

This method never throws. Every "cannot forward" condition degrades to {@code false}; if + * the decision to forward was already taken and the outbound call then fails, the failure is + * propagated to {@code responseObserver#onError} and {@code true} is returned — see + * {@link #invokeOnPeer}. + * + * @param group consumer group the client is registered under + * @param clientId client id, as reported by the client itself + * @param responseObserver the observer of the inbound admin RPC; ownership moves to the peer + * when this method returns true + * @param invocation performs the actual unary call on the peer's stub, e.g. + * {@code (stub, observer) -> stub.printThreadStackTrace(request, observer)} + */ + public boolean forwardIfRemote(String group, String clientId, + StreamObserver responseObserver, + BiConsumer> invocation) { + if (StringUtils.isBlank(clientId)) { + return false; + } + + // Loop guard first: a forwarded call must be answered locally even if this proxy still + // sees the client as remote (stale heartbeat sync), otherwise the two proxies bounce the + // call until it dies on a deadline. + if (isInboundForwarded()) { + log.info("admin call already forwarded by a peer, serving locally. group:{}, clientId:{}", + group, clientId); + return false; + } + + ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + if (proxyConfig == null) { + log.warn("proxy config is not initialized, cannot forward. group:{}, clientId:{}", group, clientId); + return false; + } + Integer adminPort = proxyConfig.getGrpcAdminServerPort(); + if (adminPort == null || adminPort <= 0 || !proxyConfig.isGrpcAdminServerEnable()) { + log.warn("proxy admin server is not enabled on this cluster, cannot forward. " + + "group:{}, clientId:{}, adminPort:{}", group, clientId, adminPort); + return false; + } + + ClientChannelInfo channelInfo = findClientChannel(group, clientId); + if (channelInfo == null) { + return false; + } + Channel channel = channelInfo.getChannel(); + if (!ChannelHelper.isRemote(channel)) { + // A locally connected client: the caller can reach it through its own channel manager. + return false; + } + RemoteChannel remoteChannel = (RemoteChannel) channel; + String remoteProxyIp = remoteChannel.getRemoteProxyIp(); + if (StringUtils.isBlank(remoteProxyIp)) { + log.warn("remote channel carries no owning proxy address, cannot forward. " + + "group:{}, clientId:{}, channel:{}", group, clientId, remoteChannel); + return false; + } + // Self-forward guard: RemoteChannel.remoteProxyIp is the peer's localServeAddr, so a plain + // string comparison detects "this proxy" and prevents a forward to ourselves. + if (StringUtils.equals(remoteProxyIp, proxyConfig.getLocalServeAddr())) { + log.info("client is owned by this proxy, serving locally. group:{}, clientId:{}", group, clientId); + return false; + } + + return invokeOnPeer(remoteProxyIp, adminPort, group, clientId, responseObserver, invocation); + } + + /** + * Runs {@code invocation} against the peer stub and hands the inbound observer to the peer's + * response stream. + * + *

Error translation: this class only holds the caller's {@code StreamObserver}, not the concrete + * response builder, so it cannot synthesize a business-level {@code Status} message inside the + * response body. A failed forward is therefore propagated as a gRPC {@code onError} carrying the + * peer's status (or {@code Status.INTERNAL} when the call could not even be issued), and + * {@code true} is returned so the caller stops and lets that error reach the client. Callers + * that want an in-band error instead should pre-check with {@code isInboundForwarded()} / + * channel locality themselves. + */ + private boolean invokeOnPeer(String remoteProxyIp, int adminPort, String group, String clientId, + StreamObserver responseObserver, + BiConsumer> invocation) { + String target = remoteProxyIp + ":" + adminPort; + try { + PeerConnection peer = this.peerConnections.computeIfAbsent(target, + key -> createPeerConnection(remoteProxyIp, adminPort)); + AdminGrpc.AdminStub stub = peer.stub.withInterceptors( + MetadataUtils.newAttachHeadersInterceptor(buildOutboundMetadata())); + log.info("forwarding admin call to peer proxy. target:{}, group:{}, clientId:{}", + target, group, clientId); + invocation.accept(stub, responseObserver); + return true; + } catch (Throwable t) { + log.error("forward admin call to peer proxy failed. target:{}, group:{}, clientId:{}", + target, group, clientId, t); + responseObserver.onError(Status.INTERNAL + .withDescription("forward admin call to peer proxy " + target + " failed: " + t.getMessage()) + .withCause(t) + .asRuntimeException()); + return true; + } + } + + private ClientChannelInfo findClientChannel(String group, String clientId) { + ConsumerManager consumerManager = this.serviceManager == null + ? null : this.serviceManager.getConsumerManager(); + if (consumerManager == null) { + return null; + } + try { + return consumerManager.findChannel(group, clientId); + } catch (Throwable t) { + log.warn("find client channel failed. group:{}, clientId:{}", group, clientId, t); + return null; + } + } + + /** + * Builds the metadata for the outbound call: a verbatim copy of the inbound metadata (so the + * peer can authenticate/authorize the original caller, see the class javadoc) plus the + * loop-prevention marker. + */ + private Metadata buildOutboundMetadata() { + Metadata outbound = new Metadata(); + Metadata inbound = GrpcConstants.METADATA.get(Context.current()); + if (inbound != null) { + for (String name : inbound.keys()) { + if (!isCopyable(name)) { + continue; + } + try { + Metadata.Key key = Metadata.Key.of(name, Metadata.ASCII_STRING_MARSHALLER); + for (String value : inbound.getAll(key)) { + outbound.put(key, value); + } + } catch (Throwable t) { + // Defensive: an exotic inbound key must not abort the whole forward. + log.warn("skip inbound metadata key that cannot be forwarded. key:{}", name, t); + } + } + } + // discardAll first: Metadata#put appends, and a stale marker from the inbound copy would + // otherwise leave the header duplicated. + outbound.discardAll(GrpcConstants.ADMIN_FORWARDED); + outbound.put(GrpcConstants.ADMIN_FORWARDED, FORWARDED_VALUE); + return outbound; + } + + private static boolean isCopyable(String name) { + if (StringUtils.isBlank(name)) { + return false; + } + // Binary headers cannot be read back through ASCII_STRING_MARSHALLER; grpc-* names are + // reserved and rejected by Metadata.Key#of. + return !name.endsWith(Metadata.BINARY_HEADER_SUFFIX) && !name.startsWith(RESERVED_KEY_PREFIX); + } + + private PeerConnection createPeerConnection(String host, int port) { + NettyChannelBuilder builder = NettyChannelBuilder.forAddress(host, port); + if (usePlaintext()) { + builder.usePlaintext(); + } else { + builder.sslContext(insecureClientSslContext()); + } + ManagedChannel channel = builder.build(); + log.info("created proxy admin client channel to peer proxy. target:{}:{}", host, port); + return new PeerConnection(channel, AdminGrpc.newStub(channel)); + } + + private static boolean usePlaintext() { + // ENFORCING is the only tlsMode in which the peer's negotiator rejects a plaintext + // handshake, so it always wins over tlsTestModeEnable. + if (TlsMode.ENFORCING.equals(TlsSystemConfig.tlsMode)) { + return false; + } + ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + // tlsTestModeEnable=true mints a fresh SelfSignedCertificate per JVM; no peer can validate + // it, and permissive mode (the default) accepts plaintext, so skip TLS entirely. + return proxyConfig == null || proxyConfig.isTlsTestModeEnable(); + } + + private static SslContext insecureClientSslContext() { + try { + return GrpcSslContexts.configure( + GrpcSslContexts.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE)) + .build(); + } catch (Exception e) { + throw new IllegalStateException("build proxy admin client SslContext failed", e); + } + } + + @Override + public void start() throws Exception { + // Peer connections are created lazily on first forward; nothing to warm up. + } + + @Override + public void shutdown() throws Exception { + for (PeerConnection peer : this.peerConnections.values()) { + try { + peer.channel.shutdown(); + if (!peer.channel.awaitTermination(SHUTDOWN_GRACE_SECONDS, TimeUnit.SECONDS)) { + peer.channel.shutdownNow(); + } + } catch (Throwable t) { + log.warn("shutdown proxy admin client channel failed. channel:{}", peer.channel, t); + peer.channel.shutdownNow(); + } + } + this.peerConnections.clear(); + } + + private static final class PeerConnection { + private final ManagedChannel channel; + private final AdminGrpc.AdminStub stub; + + private PeerConnection(ManagedChannel channel, AdminGrpc.AdminStub stub) { + this.channel = channel; + this.stub = stub; + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminGrpcService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminGrpcService.java new file mode 100644 index 00000000000..09b0196f441 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminGrpcService.java @@ -0,0 +1,1735 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.admin; + +import apache.rocketmq.v2.AdminGrpc; +import apache.rocketmq.v2.AdminSendMessageRequest; +import apache.rocketmq.v2.AdminSendMessageResponse; +import apache.rocketmq.v2.ChangeLogLevelRequest; +import apache.rocketmq.v2.ChangeLogLevelResponse; +import apache.rocketmq.v2.ClientInfo; +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.ConsumerRunningInfo; +import apache.rocketmq.v2.DeleteSubscriptionRequest; +import apache.rocketmq.v2.DeleteSubscriptionResponse; +import apache.rocketmq.v2.DescribeGroupAccumulationRequest; +import apache.rocketmq.v2.DescribeGroupAccumulationResponse; +import apache.rocketmq.v2.DescribeSubscriptionRequest; +import apache.rocketmq.v2.DescribeSubscriptionResponse; +import apache.rocketmq.v2.DescribeTopicStatusRequest; +import apache.rocketmq.v2.DescribeTopicStatusResponse; +import apache.rocketmq.v2.GetConsumerRunningInfoRequest; +import apache.rocketmq.v2.GetConsumerRunningInfoResponse; +import apache.rocketmq.v2.GetProxyRuntimeStatsRequest; +import apache.rocketmq.v2.GetProxyRuntimeStatsResponse; +import apache.rocketmq.v2.GetTopicRouteRequest; +import apache.rocketmq.v2.GetTopicRouteResponse; +import apache.rocketmq.v2.ListConsumerConnectionRequest; +import apache.rocketmq.v2.ListConsumerConnectionResponse; +import apache.rocketmq.v2.ListMessageRequest; +import apache.rocketmq.v2.ListMessageResponse; +import apache.rocketmq.v2.ListSubscriptionRequest; +import apache.rocketmq.v2.ListSubscriptionResponse; +import apache.rocketmq.v2.PrintThreadStackTraceRequest; +import apache.rocketmq.v2.PrintThreadStackTraceResponse; +import apache.rocketmq.v2.QueryTimeSpanRequest; +import apache.rocketmq.v2.QueryTimeSpanResponse; +import apache.rocketmq.v2.ResetGroupOffsetRequest; +import apache.rocketmq.v2.ResetGroupOffsetResponse; +import apache.rocketmq.v2.Resource; +import apache.rocketmq.v2.Status; +import apache.rocketmq.v2.SubscriptionInfo; +import apache.rocketmq.v2.SystemProperties; +import apache.rocketmq.v2.VerifyMessageRequest; +import apache.rocketmq.v2.VerifyMessageResponse; +import com.google.protobuf.Timestamp; +import com.google.protobuf.util.Timestamps; +import io.grpc.stub.StreamObserver; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerGroupInfo; +import org.apache.rocketmq.broker.client.ConsumerManager; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.common.MQVersion; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.utils.StartAndShutdown; +import org.apache.rocketmq.common.utils.ThreadUtils; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.proxy.service.ServiceManager; +import org.apache.rocketmq.proxy.service.admin.AdminService; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayRequest; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayResult; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; +import org.apache.rocketmq.proxy.service.route.MessageQueueView; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; +import org.apache.rocketmq.remoting.protocol.body.CMResult; +import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; +import org.apache.rocketmq.remoting.protocol.body.Connection; +import org.apache.rocketmq.remoting.protocol.body.GroupList; +import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan; +import org.apache.rocketmq.remoting.protocol.body.TopicList; +import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.protocol.subscription.SimpleSubscriptionData; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; + +/** + * Proxy Admin gRPC service (control plane). + * + *

Design rules this class follows: + *

    + *
  • Never block the gRPC executor. Every broker hop goes through the asynchronous + * {@link AdminService} gateway and is fanned out to all relevant brokers concurrently.
    • + *
  • Cluster-wide view. Subscription and connection data is read from the brokers (which + * see consumers registered through every proxy) rather than from this proxy's local channel + * table, and consumers owned by a peer proxy are reached by forwarding the whole RPC.
    • + *
  • Honest responses. A field the open-source proxy genuinely cannot supply is left + * unset and explained in the status message, instead of being filled with a plausible value.
    • + *
  • Graded errors. Failures are mapped through {@link ResponseBuilder#buildStatus(Throwable)} + * so callers can tell "topic not found" from "internal error".
    • + *
+ * + *

The translation between broker wire types and the v2 contract lives in + * {@link AdminModelConverter}; whole-message conversion reuses the data plane's + * {@link GrpcConverter} instead of duplicating it. + */ +public class ProxyAdminGrpcService extends AdminGrpc.AdminImplBase implements StartAndShutdown { + + private static final Logger log = LoggerFactory.getLogger(ProxyAdminGrpcService.class); + + /** + * Bounds every wait this service performs that is not already bounded by the callee: the answer + * to a relayed telemetry command, and each broker hop of a fan-out. The gRPC nonce sweeper and + * the remoting timeout cover the healthy paths, but a hop that never completes at all (an + * unwritable channel, a saturated blocking-call pool) would otherwise leave the RPC hanging + * forever with no response at all. + */ + private final ScheduledExecutorService adminTimeoutScheduler = + ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("ProxyAdminTimeout_", true)); + + /** + * How much longer than the broker timeout a single hop is allowed to take. The broker call + * carries its own timeout, so this only catches a gateway future that never completes; the slack + * keeps a genuine broker timeout reporting its own cause instead of this generic deadline. + */ + private static final long BROKER_CALL_DEADLINE_SLACK_MILLIS = 1000L; + + private static final String ADMIN_SEND_PRODUCER_GROUP = "ADMIN_SEND_PRODUCER_GROUP"; + private static final int DEFAULT_MAX_MESSAGE_NUMS = 32; + + private final ServiceManager serviceManager; + private final MessagingProcessor messagingProcessor; + private final GrpcChannelManager grpcChannelManager; + private final GrpcClientSettingsManager grpcClientSettingsManager; + private final ProxyAdminForwarder forwarder; + + public ProxyAdminGrpcService(ServiceManager serviceManager, MessagingProcessor messagingProcessor, + GrpcChannelManager grpcChannelManager, GrpcClientSettingsManager grpcClientSettingsManager, + ProxyAdminForwarder forwarder) { + this.serviceManager = serviceManager; + this.messagingProcessor = messagingProcessor; + this.grpcChannelManager = grpcChannelManager; + this.grpcClientSettingsManager = grpcClientSettingsManager; + this.forwarder = forwarder; + } + + // ========================================================================= + // helpers + // ========================================================================= + + private AdminService admin() { + return this.serviceManager.getAdminService(); + } + + private long timeoutMillis() { + return ConfigurationManager.getProxyConfig().getGrpcAdminServerRequestTimeoutMillis(); + } + + private ProxyContext ctx() { + return ProxyContext.create(); + } + + private static Status ok() { + return AdminModelConverter.ok(); + } + + private static Status err(Throwable t) { + return ResponseBuilder.getInstance().buildStatus(t); + } + + private static Status err(Code code, String message) { + return ResponseBuilder.getInstance().buildStatus(code, message); + } + + /** Sends exactly one response. Splitting onNext/onCompleted across a try and a catch is what + * makes a StreamObserver throw IllegalStateException on a partially-succeeded call. */ + private static void respond(StreamObserver observer, T response) { + observer.onNext(response); + observer.onCompleted(); + } + + /** Master broker address of every broker group in the cluster. */ + private CompletableFuture> allMasterBrokerAddrs() { + return admin().getBrokerClusterInfo(timeoutMillis()).thenApply(clusterInfo -> { + List addrs = new ArrayList<>(); + if (clusterInfo == null || clusterInfo.getBrokerAddrTable() == null) { + return addrs; + } + for (BrokerData brokerData : clusterInfo.getBrokerAddrTable().values()) { + String master = brokerData.getBrokerAddrs() == null ? null + : brokerData.getBrokerAddrs().get(MixAll.MASTER_ID); + if (master != null && !master.isEmpty()) { + addrs.add(master); + } + } + return addrs; + }); + } + + /** + * Addresses of the brokers hosting a topic. Write operations must use the write selector: the + * read selector can omit brokers that only accept writes and vice versa. + */ + private List topicBrokerAddrs(String topic, boolean forWrite) throws Exception { + MessageQueueView view = serviceManager.getTopicRouteService().getAllMessageQueueView(ctx(), topic); + if (view == null) { + throw new IllegalStateException("topic route not found for " + topic); + } + List queues = forWrite + ? view.getWriteSelector().getQueues() : view.getReadSelector().getQueues(); + if (queues == null || queues.isEmpty()) { + throw new IllegalStateException("no " + (forWrite ? "writable" : "readable") + " queue for topic " + topic); + } + Set addrs = new LinkedHashSet<>(); + for (AddressableMessageQueue queue : queues) { + if (queue.getBrokerAddr() != null && !queue.getBrokerAddr().isEmpty()) { + addrs.add(queue.getBrokerAddr()); + } + } + if (addrs.isEmpty()) { + throw new IllegalStateException("broker address not found for topic " + topic); + } + return new ArrayList<>(addrs); + } + + /** + * Runs a query against every broker concurrently and keeps the successful answers. A broker + * that fails does not fail the whole call — a partially available cluster should still return + * the data it has. Only when every broker fails is the first error propagated, so the caller + * reports a real cause instead of an empty success. + */ + private CompletableFuture> queryAllBrokers(Collection brokerAddrs, + BiFunction> query) { + if (brokerAddrs == null || brokerAddrs.isEmpty()) { + return CompletableFuture.completedFuture(Collections.emptyMap()); + } + long timeout = timeoutMillis(); + Map> pending = new LinkedHashMap<>(); + List failures = Collections.synchronizedList(new ArrayList<>()); + for (String brokerAddr : brokerAddrs) { + CompletableFuture hop = withTimeout(query.apply(brokerAddr, timeout), + timeout + BROKER_CALL_DEADLINE_SLACK_MILLIS, "broker call to " + brokerAddr); + pending.put(brokerAddr, hop + .whenComplete((result, throwable) -> { + if (throwable != null) { + failures.add(throwable); + } + })); + } + return CompletableFuture.allOf(pending.values().toArray(new CompletableFuture[0])) + .handle((ignored, throwable) -> { + Map succeeded = new LinkedHashMap<>(); + for (Map.Entry> entry : pending.entrySet()) { + CompletableFuture future = entry.getValue(); + if (!future.isCompletedExceptionally()) { + T result = future.getNow(null); + if (result != null) { + succeeded.put(entry.getKey(), result); + } + } + } + return succeeded; + }) + .thenCompose(succeeded -> { + if (succeeded.isEmpty() && !failures.isEmpty()) { + CompletableFuture> failed = new CompletableFuture<>(); + failed.completeExceptionally(failures.get(0)); + return failed; + } + return CompletableFuture.completedFuture(succeeded); + }); + } + + private CompletableFuture withTimeout(CompletableFuture future, long timeoutMillis, + String what) { + CompletableFuture timeout = new CompletableFuture<>(); + ScheduledFuture timer = this.adminTimeoutScheduler.schedule( + () -> timeout.completeExceptionally(new IllegalStateException( + what + " timed out after " + timeoutMillis + "ms")), + timeoutMillis, TimeUnit.MILLISECONDS); + // one admin RPC fans out to every broker, so the timer is cancelled as soon as the answer + // arrives instead of leaving a dead task per hop queued until its deadline + return future.applyToEither(timeout, result -> result) + .whenComplete((result, throwable) -> timer.cancel(false)); + } + + private static Resource resource(String name) { + return Resource.newBuilder().setName(name == null ? "" : name).build(); + } + + /** + * Locates the client a client-targeted RPC is about. Returns the channel info when the client + * is known to this proxy's consumer manager (which, in cluster mode, also holds entries synced + * from peer proxies), or null. + */ + private ClientChannelInfo findClientChannel(String group, String clientId) { + if (clientId == null || clientId.isEmpty()) { + return null; + } + ConsumerManager consumerManager = serviceManager.getConsumerManager(); + if (consumerManager == null) { + return null; + } + if (group != null && !group.isEmpty()) { + ClientChannelInfo info = consumerManager.findChannel(group, clientId); + if (info != null) { + return info; + } + } + // the caller may not know the group; the local gRPC channel table is keyed by clientId alone + GrpcClientChannel channel = grpcChannelManager.getChannel(clientId); + if (channel == null) { + return null; + } + return new ClientChannelInfo(channel, clientId, null, 0); + } + + // ========================================================================= + // 1. ChangeLogLevel + // ========================================================================= + + @Override + public void changeLogLevel(ChangeLogLevelRequest request, StreamObserver responseObserver) { + String remark; + try { + org.apache.rocketmq.logging.org.slf4j.ILoggerFactory factory = + org.apache.rocketmq.logging.org.slf4j.LoggerFactory.getILoggerFactory(); + if (!(factory instanceof org.apache.rocketmq.logging.ch.qos.logback.classic.LoggerContext)) { + remark = "unsupported logging backend, cannot change log level at runtime"; + } else { + ChangeLogLevelRequest.Level level = request.getLevel(); + // Level.TRACE is enum value 0, so proto3 cannot distinguish "caller asked for TRACE" + // from "caller sent an empty request". Honour the contract but make the consequence + // visible instead of silently switching a production proxy to the most verbose level. + if (level == ChangeLogLevelRequest.Level.TRACE) { + log.warn("changeLogLevel requested TRACE, which is also the proto3 default for an " + + "unset level; make sure this is intentional"); + } + org.apache.rocketmq.logging.ch.qos.logback.classic.LoggerContext loggerContext = + (org.apache.rocketmq.logging.ch.qos.logback.classic.LoggerContext) factory; + org.apache.rocketmq.logging.ch.qos.logback.classic.Level logbackLevel = + org.apache.rocketmq.logging.ch.qos.logback.classic.Level.toLevel(level.name()); + loggerContext.getLogger( + org.apache.rocketmq.logging.ch.qos.logback.classic.Logger.ROOT_LOGGER_NAME) + .setLevel(logbackLevel); + remark = "log level changed to " + logbackLevel; + } + } catch (Throwable t) { + log.warn("changeLogLevel failed", t); + remark = "failed to change log level: " + t.getMessage(); + } + respond(responseObserver, ChangeLogLevelResponse.newBuilder().setRemark(remark).build()); + } + + // ========================================================================= + // 2. DescribeTopicStatus + // ========================================================================= + + @Override + public void describeTopicStatus(DescribeTopicStatusRequest request, + StreamObserver responseObserver) { + String topic = request.getTopic().getName(); + List brokerAddrs; + try { + brokerAddrs = topicBrokerAddrs(topic, false); + } catch (Throwable t) { + respond(responseObserver, DescribeTopicStatusResponse.newBuilder().setStatus(err(t)).build()); + return; + } + // a topic is normally configured identically on every broker hosting it, but asking only the + // first one hides a real inconsistency, so all of them are queried and compared + queryAllBrokers(brokerAddrs, (addr, timeout) -> admin().getTopicConfig(addr, topic, timeout)) + .whenComplete((configs, throwable) -> { + if (throwable != null) { + respond(responseObserver, DescribeTopicStatusResponse.newBuilder().setStatus(err(throwable)).build()); + return; + } + try { + respond(responseObserver, AdminModelConverter.toTopicStatus( + mergeTopicConfigs(configs.values()), topic)); + } catch (Throwable t) { + respond(responseObserver, DescribeTopicStatusResponse.newBuilder().setStatus(err(t)).build()); + } + }); + } + + private static TopicConfig mergeTopicConfigs(Collection configs) { + TopicConfig merged = null; + TopicMessageType messageType = null; + int readQueueNums = 0; + int writeQueueNums = 0; + boolean consistent = true; + for (TopicConfig config : configs) { + if (config == null) { + continue; + } + if (merged == null) { + merged = config; + messageType = config.getTopicMessageType(); + } else if (messageType != config.getTopicMessageType()) { + consistent = false; + } + readQueueNums += config.getReadQueueNums(); + writeQueueNums += config.getWriteQueueNums(); + } + if (merged == null) { + return null; + } + TopicConfig result = new TopicConfig(); + result.setTopicName(merged.getTopicName()); + result.setReadQueueNums(readQueueNums); + result.setWriteQueueNums(writeQueueNums); + result.setPerm(merged.getPerm()); + // MIXED is the honest answer when brokers disagree, and TopicConfig maps it back to + // MESSAGE_TYPE_UNSPECIFIED in the response rather than pretending one broker is right + result.setAttributes(new HashMap<>()); + result.getAttributes().put(org.apache.rocketmq.common.TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName(), + (consistent ? messageType : TopicMessageType.MIXED).name()); + return result; + } + + // ========================================================================= + // 3. ListSubscription + // ========================================================================= + + @Override + public void listSubscription(ListSubscriptionRequest request, StreamObserver responseObserver) { + boolean hasTopic = request.hasTopic() && !request.getTopic().getName().isEmpty(); + boolean hasGroup = request.hasGroup() && !request.getGroup().getName().isEmpty(); + if (!hasTopic && !hasGroup) { + respond(responseObserver, ListSubscriptionResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "at least one of topic or group must be set")).build()); + return; + } + String topic = hasTopic ? request.getTopic().getName() : null; + String group = hasGroup ? request.getGroup().getName() : null; + resolveGroups(topic, group) + .thenCompose(groups -> allMasterBrokerAddrs() + .thenCompose(brokers -> collectConnections(brokers, groups))) + .whenComplete((connections, throwable) -> { + if (throwable != null) { + respond(responseObserver, ListSubscriptionResponse.newBuilder().setStatus(err(throwable)).build()); + return; + } + try { + respond(responseObserver, buildListSubscriptionResponse(connections, topic)); + } catch (Throwable t) { + respond(responseObserver, ListSubscriptionResponse.newBuilder().setStatus(err(t)).build()); + } + }); + } + + private ListSubscriptionResponse buildListSubscriptionResponse(Map connections, + String topicFilter) { + ListSubscriptionResponse.Builder builder = ListSubscriptionResponse.newBuilder().setStatus(ok()); + // a group reported by several brokers, or by several clients, describes the same + // subscription; keyed by group+topic+expression so the answer does not scale with + // the number of online consumers + Map dedup = new LinkedHashMap<>(); + String pullRetryPrefix = MixAll.RETRY_GROUP_TOPIC_PREFIX; + for (Map.Entry entry : connections.entrySet()) { + String group = entry.getKey(); + ConsumerConnection connection = entry.getValue(); + if (connection == null || connection.getSubscriptionTable() == null) { + continue; + } + boolean online = connection.getConnectionSet() != null && !connection.getConnectionSet().isEmpty(); + apache.rocketmq.v2.MessageModel messageModel = AdminModelConverter.toMessageModel(connection.getMessageModel()); + for (Map.Entry sub : + connection.getSubscriptionTable().entrySet()) { + String subTopic = sub.getKey(); + if (subTopic == null || subTopic.startsWith(pullRetryPrefix) || KeyBuilder.isPopRetryTopicV2(subTopic)) { + continue; + } + if (topicFilter != null && !topicFilter.equals(subTopic)) { + continue; + } + org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData data = sub.getValue(); + SubscriptionInfo info = SubscriptionInfo.newBuilder() + .setGroup(resource(group)) + .setTopic(resource(subTopic)) + .setExpression(AdminModelConverter.toFilterExpression( + data == null ? null : data.getExpressionType(), data == null ? null : data.getSubString())) + .setOnline(online) + .setMessageModel(messageModel) + .build(); + dedup.put(group + "\u0001" + subTopic, info); + } + } + for (SubscriptionInfo info : dedup.values()) { + builder.addSubscriptionInfo(info); + } + return builder.build(); + } + + /** Groups to inspect: the requested one, or every group consuming the requested topic. */ + private CompletableFuture> resolveGroups(String topic, String group) { + if (group != null) { + return CompletableFuture.completedFuture(Collections.singleton(group)); + } + return allMasterBrokerAddrs().thenCompose(brokers -> + queryAllBrokers(brokers, (addr, timeout) -> admin().queryTopicConsumeByWho(addr, topic, timeout)) + .thenApply(byBroker -> { + Set groups = new LinkedHashSet<>(); + for (GroupList groupList : byBroker.values()) { + if (groupList != null && groupList.getGroupList() != null) { + groups.addAll(groupList.getGroupList()); + } + } + return groups; + })); + } + + /** Consumer connections of each group, merged across brokers. */ + private CompletableFuture> collectConnections(List brokers, + Collection groups) { + List> tasks = new ArrayList<>(); + Map merged = new LinkedHashMap<>(); + for (String group : groups) { + tasks.add(queryAllBrokers(brokers, + (addr, timeout) -> admin().getConsumerConnectionList(addr, group, timeout)) + .handle((byBroker, throwable) -> { + // seed with the proxy-side view: gRPC v2 consumers never reach the broker, and a + // broker that does not know the group must not erase what this proxy does know + ConsumerConnection acc = proxySideConnection(group); + if (throwable != null) { + log.info("broker has no consumer connection for group {}: {}", group, throwable.getMessage()); + } else { + for (ConsumerConnection connection : byBroker.values()) { + acc = mergeConsumerConnection(acc, connection); + } + } + if (acc != null) { + synchronized (merged) { + merged.put(group, acc); + } + } + return null; + })); + } + return CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])) + .thenApply(ignored -> merged); + } + + /** + * Consumers registered through this proxy cluster, synthesised into the same shape the broker + * reports so the two sources can be merged without the callers knowing where a client came from. + * + *

A gRPC v2 client keeps its channel on the proxy and is never registered in the broker's + * consumer manager, so a broker-only query returns nothing for it. {@code HeartbeatSyncer} + * replicates registrations between proxies, so this view also covers clients connected to a peer + * proxy (they appear as {@link org.apache.rocketmq.proxy.processor.channel.RemoteChannel}). + */ + private ConsumerConnection proxySideConnection(String group) { + ConsumerManager consumerManager = serviceManager.getConsumerManager(); + ConsumerGroupInfo groupInfo = consumerManager == null ? null : consumerManager.getConsumerGroupInfo(group); + if (groupInfo == null) { + return null; + } + ConsumerConnection connection = new ConsumerConnection(); + connection.setConsumeType(groupInfo.getConsumeType()); + connection.setMessageModel(groupInfo.getMessageModel()); + connection.setConsumeFromWhere(groupInfo.getConsumeFromWhere()); + if (groupInfo.getSubscriptionTable() != null) { + connection.getSubscriptionTable().putAll(groupInfo.getSubscriptionTable()); + } + for (ClientChannelInfo channelInfo : groupInfo.getChannelInfoTable().values()) { + if (channelInfo == null) { + continue; + } + Connection client = new Connection(); + client.setClientId(channelInfo.getClientId()); + client.setLanguage(channelInfo.getLanguage()); + client.setVersion(channelInfo.getVersion()); + client.setClientAddr(remoteAddressOf(channelInfo)); + connection.getConnectionSet().add(client); + } + return connection; + } + + /** Netty renders a socket address as "/ip:port"; drop the leading slash so it parses as an IP. */ + private static String remoteAddressOf(ClientChannelInfo channelInfo) { + if (channelInfo.getChannel() == null || channelInfo.getChannel().remoteAddress() == null) { + return ""; + } + String address = channelInfo.getChannel().remoteAddress().toString(); + return address.startsWith("/") ? address.substring(1) : address; + } + + private static ConsumerConnection mergeConsumerConnection(ConsumerConnection target, ConsumerConnection source) { + if (source == null) { + return target; + } + if (target == null) { + ConsumerConnection copy = new ConsumerConnection(); + copy.setConsumeType(source.getConsumeType()); + copy.setMessageModel(source.getMessageModel()); + copy.setConsumeFromWhere(source.getConsumeFromWhere()); + copy.getConnectionSet().addAll(source.getConnectionSet()); + copy.getSubscriptionTable().putAll(source.getSubscriptionTable()); + return copy; + } + for (Connection connection : source.getConnectionSet()) { + boolean seen = false; + for (Connection existing : target.getConnectionSet()) { + if (existing.getClientId() != null && existing.getClientId().equals(connection.getClientId())) { + seen = true; + break; + } + } + if (!seen) { + target.getConnectionSet().add(connection); + } + } + target.getSubscriptionTable().putAll(source.getSubscriptionTable()); + return target; + } + + // ========================================================================= + // 4. DescribeSubscription + // ========================================================================= + + @Override + public void describeSubscription(DescribeSubscriptionRequest request, + StreamObserver responseObserver) { + boolean hasTopic = request.hasTopic() && !request.getTopic().getName().isEmpty(); + boolean hasGroup = request.hasGroup() && !request.getGroup().getName().isEmpty(); + if (!hasTopic && !hasGroup) { + respond(responseObserver, DescribeSubscriptionResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "at least one of topic or group must be set")).build()); + return; + } + String topic = hasTopic ? request.getTopic().getName() : null; + String group = hasGroup ? request.getGroup().getName() : null; + resolveGroups(topic, group) + .thenCompose(groups -> allMasterBrokerAddrs() + .thenCompose(brokers -> collectConnections(brokers, groups))) + .whenComplete((connections, throwable) -> { + if (throwable != null) { + respond(responseObserver, DescribeSubscriptionResponse.newBuilder().setStatus(err(throwable)).build()); + return; + } + try { + respond(responseObserver, buildDescribeSubscriptionResponse(connections, topic)); + } catch (Throwable t) { + respond(responseObserver, DescribeSubscriptionResponse.newBuilder().setStatus(err(t)).build()); + } + }); + } + + /** + * One entry per connected client, which is what makes inconsistent subscriptions inside a group + * visible. Per-client settings are only held for clients connected to this proxy, so a client + * owned by a peer proxy is reported from the broker-side connection data with the group-level + * subscription rather than being dropped. + */ + private DescribeSubscriptionResponse buildDescribeSubscriptionResponse(Map connections, + String topicFilter) { + DescribeSubscriptionResponse.Builder builder = DescribeSubscriptionResponse.newBuilder().setStatus(ok()); + for (Map.Entry entry : connections.entrySet()) { + String group = entry.getKey(); + ConsumerConnection connection = entry.getValue(); + if (connection == null) { + continue; + } + apache.rocketmq.v2.MessageModel messageModel = AdminModelConverter.toMessageModel(connection.getMessageModel()); + for (Connection conn : connection.getConnectionSet()) { + ClientInfo clientInfo = AdminModelConverter.toClientInfo(conn, messageModel); + apache.rocketmq.v2.Settings settings = + grpcClientSettingsManager.getRawClientSettings(conn.getClientId()); + List infos = settings != null && settings.hasSubscription() + ? subscriptionsFromSettings(group, settings, topicFilter, messageModel) + : subscriptionsFromConnection(group, connection, topicFilter); + for (SubscriptionInfo info : infos) { + builder.addClientSubscriptionInfo( + DescribeSubscriptionResponse.ClientSubscriptionInfo.newBuilder() + .setClientInfo(clientInfo) + .setSubscriptionInfo(info) + .build()); + } + } + } + return builder.build(); + } + + private List subscriptionsFromSettings(String group, apache.rocketmq.v2.Settings settings, + String topicFilter, apache.rocketmq.v2.MessageModel messageModel) { + List result = new ArrayList<>(); + for (apache.rocketmq.v2.SubscriptionEntry entry : settings.getSubscription().getSubscriptionsList()) { + String entryTopic = entry.hasTopic() ? entry.getTopic().getName() : ""; + if (topicFilter != null && !topicFilter.equals(entryTopic)) { + continue; + } + SubscriptionInfo.Builder info = SubscriptionInfo.newBuilder() + .setGroup(resource(group)) + .setTopic(resource(entryTopic)) + .setOnline(true) + .setMessageModel(messageModel); + if (entry.hasExpression()) { + info.setExpression(entry.getExpression()); + } + result.add(info.build()); + } + return result; + } + + private List subscriptionsFromConnection(String group, ConsumerConnection connection, + String topicFilter) { + List result = new ArrayList<>(); + if (connection.getSubscriptionTable() == null) { + return result; + } + apache.rocketmq.v2.MessageModel messageModel = AdminModelConverter.toMessageModel(connection.getMessageModel()); + for (Map.Entry sub : + connection.getSubscriptionTable().entrySet()) { + if (topicFilter != null && !topicFilter.equals(sub.getKey())) { + continue; + } + org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData data = sub.getValue(); + result.add(SubscriptionInfo.newBuilder() + .setGroup(resource(group)) + .setTopic(resource(sub.getKey())) + .setExpression(AdminModelConverter.toFilterExpression( + data == null ? null : data.getExpressionType(), data == null ? null : data.getSubString())) + .setOnline(true) + .setMessageModel(messageModel) + .build()); + } + return result; + } + + // ========================================================================= + // 5. DeleteSubscription + // ========================================================================= + + /** + * Deletes ONE subscription relationship (group + topic + filter expression), which is what the + * contract asks for. It deliberately does NOT delete the consumer group: an earlier version + * answered this request with {@code deleteSubscriptionGroup} on every broker hosting the topic, + * which destroyed the group's other subscriptions along with its offsets. + * + *

The broker has no request code for a granular delete, so this is a read-modify-write on + * {@link SubscriptionGroupConfig#getSubscriptionDataSet()} via UPDATE_AND_CREATE_SUBSCRIPTIONGROUP. + * + *

In open source this is effectively a no-op, and that is expected rather than a bug. + * Nothing in the codebase ever writes {@code subscriptionDataSet}: the broker only reads it + * ({@code ConsumerLagCalculator}, to attribute lag to topics), so on a stock deployment the set + * is {@code null} and there is no persisted relationship to remove. Durable per-relationship + * subscriptions are a downstream concern: distributions that need this RPC back it with an + * external subscription store that populates the same field. Live subscriptions held in the + * broker's {@code ConsumerManager} are deliberately not touched: they are rebuilt from client + * heartbeats, so removing them there would be undone within one heartbeat interval and would + * only look like a successful delete. + * + *

Callers therefore observe {@link Code#NOT_FOUND} with an explanation, and the consumer group + * and its offsets are left untouched. Making this RPC functional requires a persisted + * subscription store (or a contract change), which is out of scope for the proxy. + */ + @Override + public void deleteSubscription(DeleteSubscriptionRequest request, + StreamObserver responseObserver) { + String topic = request.getTopic().getName(); + String group = request.hasGroup() ? request.getGroup().getName() : ""; + if (topic.isEmpty() || group.isEmpty()) { + respond(responseObserver, DeleteSubscriptionResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "topic and group are required")).build()); + return; + } + boolean hasExpression = request.hasExpression() && !request.getExpression().getExpression().isEmpty(); + String expression = hasExpression ? request.getExpression().getExpression() : null; + boolean sql = hasExpression && request.getExpression().getType() == apache.rocketmq.v2.FilterType.SQL; + List brokerAddrs; + try { + brokerAddrs = topicBrokerAddrs(topic, true); + } catch (Throwable t) { + respond(responseObserver, DeleteSubscriptionResponse.newBuilder().setStatus(err(t)).build()); + return; + } + log.info("deleteSubscription group={} topic={} expression={} brokers={}", group, topic, expression, brokerAddrs); + + long timeout = timeoutMillis(); + List> tasks = new ArrayList<>(); + List failures = Collections.synchronizedList(new ArrayList<>()); + for (String brokerAddr : brokerAddrs) { + tasks.add(admin().getSubscriptionGroupConfig(brokerAddr, group, timeout) + .thenCompose(config -> { + if (config == null) { + return CompletableFuture.completedFuture(false); + } + Set dataSet = config.getSubscriptionDataSet(); + if (dataSet == null || dataSet.isEmpty()) { + // the normal path on a stock deployment: nothing persists relationships here, + // so this falls through to the NOT_FOUND explained in the method javadoc + return CompletableFuture.completedFuture(false); + } + Set remaining = new HashSet<>(); + boolean removed = false; + for (SimpleSubscriptionData data : dataSet) { + if (matches(data, topic, expression, sql)) { + removed = true; + } else { + remaining.add(data); + } + } + if (!removed) { + return CompletableFuture.completedFuture(false); + } + config.setSubscriptionDataSet(remaining); + return admin().updateSubscriptionGroupConfig(brokerAddr, config, timeout).thenApply(v -> true); + }) + .exceptionally(t -> { + failures.add(t); + log.warn("deleteSubscription failed on broker {} group={} topic={}", brokerAddr, group, topic, t); + return false; + })); + } + CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])).whenComplete((ignored, throwable) -> { + boolean anyRemoved = false; + for (CompletableFuture task : tasks) { + if (Boolean.TRUE.equals(task.getNow(false))) { + anyRemoved = true; + } + } + if (anyRemoved) { + respond(responseObserver, DeleteSubscriptionResponse.newBuilder().setStatus(ok()).build()); + } else if (!failures.isEmpty()) { + respond(responseObserver, DeleteSubscriptionResponse.newBuilder().setStatus(err(failures.get(0))).build()); + } else { + // Nothing matched. Say why precisely instead of reporting success for a no-op, and + // make clear the consumer group itself was left alone: the previous implementation + // answered this request by deleting the whole group on every broker. + respond(responseObserver, DeleteSubscriptionResponse.newBuilder() + .setStatus(err(Code.NOT_FOUND, + "no subscription of group " + group + " on topic " + topic + + " is recorded on any broker. Open-source RocketMQ does not persist " + + "per-topic subscription relationships (SubscriptionGroupConfig." + + "subscriptionDataSet is only ever read by the broker, so it stays empty " + + "unless an external subscription store writes it), so there is nothing to " + + "delete. The consumer group and its offsets were left untouched.")) + .build()); + } + }); + } + + private static boolean matches(SimpleSubscriptionData data, String topic, String expression, boolean sql) { + if (data == null || !topic.equals(data.getTopic())) { + return false; + } + if (expression == null) { + return true; + } + if (!expression.equals(data.getExpression())) { + return false; + } + String type = sql ? org.apache.rocketmq.common.filter.ExpressionType.SQL92 + : org.apache.rocketmq.common.filter.ExpressionType.TAG; + return data.getExpressionType() == null || type.equals(data.getExpressionType()); + } + + // ========================================================================= + // 6. DescribeGroupAccumulation + // ========================================================================= + + @Override + public void describeGroupAccumulation(DescribeGroupAccumulationRequest request, + StreamObserver responseObserver) { + String group = request.getGroup().getName(); + if (group.isEmpty()) { + respond(responseObserver, DescribeGroupAccumulationResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "group is required")).build()); + return; + } + Set requested = new LinkedHashSet<>(); + for (Resource resource : request.getTopicsList()) { + if (!resource.getName().isEmpty()) { + requested.add(resource.getName()); + } + } + // The contract says an empty topic list means "the whole group". Resolving that by treating + // the group name as a topic name cannot work, so the topics the group actually consumes are + // asked from the brokers instead. + CompletableFuture> topics = requested.isEmpty() + ? resolveTopicsOfGroup(group) + : CompletableFuture.completedFuture(requested); + + topics.thenCompose(topicSet -> allMasterBrokerAddrs() + .thenCompose(brokers -> queryAllBrokers(brokers, + (addr, timeout) -> admin().getConsumeStats(addr, group, "", timeout))) + .thenApply(byBroker -> AdminModelConverter.toAccumulation( + filterToTopics(byBroker, group, topicSet), group))) + .whenComplete((result, throwable) -> { + if (throwable != null) { + respond(responseObserver, DescribeGroupAccumulationResponse.newBuilder() + .setStatus(err(throwable)).build()); + return; + } + try { + DescribeGroupAccumulationResponse.Builder builder = DescribeGroupAccumulationResponse.newBuilder() + .setStatus(ok()) + .setAccumulation(result.total); + for (Map.Entry e : + result.byTopic.entrySet()) { + builder.putTopicAccumulation(e.getKey(), e.getValue()); + } + respond(responseObserver, builder.build()); + } catch (Throwable t) { + respond(responseObserver, DescribeGroupAccumulationResponse.newBuilder().setStatus(err(t)).build()); + } + }); + } + + /** Topics a group consumes, including its pop retry topics, asked from the brokers. */ + private CompletableFuture> resolveTopicsOfGroup(String group) { + return allMasterBrokerAddrs().thenCompose(brokers -> + queryAllBrokers(brokers, (addr, timeout) -> admin().queryTopicsByConsumer(addr, group, timeout)) + .thenApply(byBroker -> { + Set topics = new LinkedHashSet<>(); + for (TopicList topicList : byBroker.values()) { + if (topicList == null || topicList.getTopicList() == null) { + continue; + } + for (String topic : topicList.getTopicList()) { + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + continue; + } + topics.add(topic); + } + } + return topics; + })); + } + + /** + * Keeps only the queues belonging to the requested topics. When the caller asked for the whole + * group everything is kept, so a caller scoping the query to one topic does not get the backlog + * of unrelated topics mixed in. + */ + private Map filterToTopics(Map byBroker, String group, + Set requestedTopics) { + if (requestedTopics.isEmpty()) { + return byBroker; + } + Set wanted = new HashSet<>(); + for (String topic : requestedTopics) { + wanted.add(topic); + // a topic's backlog is spread over its own queues and the group's retry queues for it + wanted.add(KeyBuilder.buildPopRetryTopic(topic, group)); + wanted.add(KeyBuilder.buildPopRetryTopicV2(topic, group)); + } + wanted.add(MixAll.getRetryTopic(group)); + Map filtered = new LinkedHashMap<>(); + for (Map.Entry entry : byBroker.entrySet()) { + ConsumeStats stats = entry.getValue(); + if (stats == null || stats.getOffsetTable() == null) { + continue; + } + ConsumeStats copy = new ConsumeStats(); + for (Map.Entry e : + stats.getOffsetTable().entrySet()) { + if (e.getKey() != null && wanted.contains(e.getKey().getTopic())) { + copy.getOffsetTable().put(e.getKey(), e.getValue()); + } + } + filtered.put(entry.getKey(), copy); + } + return filtered; + } + + // ========================================================================= + // 7. ListConsumerConnection + // ========================================================================= + + @Override + public void listConsumerConnection(ListConsumerConnectionRequest request, + StreamObserver responseObserver) { + String group = request.getGroup().getName(); + if (group.isEmpty()) { + respond(responseObserver, ListConsumerConnectionResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "group is required")).build()); + return; + } + String topicFilter = request.hasTopic() && !request.getTopic().getName().isEmpty() + ? request.getTopic().getName() : null; + allMasterBrokerAddrs() + .thenCompose(brokers -> collectConnections(brokers, Collections.singletonList(group))) + .whenComplete((byGroup, throwable) -> { + if (throwable != null) { + // an offline group is a normal answer for a listing RPC, not a server error + log.info("listConsumerConnection group={} unavailable: {}", group, throwable.getMessage()); + respond(responseObserver, ListConsumerConnectionResponse.newBuilder() + .setStatus(ok()).build()); + return; + } + try { + ConsumerConnection merged = byGroup.get(group); + ListConsumerConnectionResponse.Builder builder = + ListConsumerConnectionResponse.newBuilder().setStatus(ok()); + if (merged != null) { + if (topicFilter != null && (merged.getSubscriptionTable() == null + || !merged.getSubscriptionTable().containsKey(topicFilter))) { + respond(responseObserver, builder.build()); + return; + } + apache.rocketmq.v2.MessageModel messageModel = + AdminModelConverter.toMessageModel(merged.getMessageModel()); + for (Connection connection : merged.getConnectionSet()) { + builder.addClientInfo(AdminModelConverter.toClientInfo(connection, messageModel)); + } + } + respond(responseObserver, builder.build()); + } catch (Throwable t) { + respond(responseObserver, ListConsumerConnectionResponse.newBuilder() + .setStatus(err(t)).build()); + } + }); + } + + // ========================================================================= + // 8. ResetGroupOffset + // ========================================================================= + + @Override + public void resetGroupOffset(ResetGroupOffsetRequest request, StreamObserver responseObserver) { + String group = request.getGroup().getName(); + String topic = request.getTopic().getName(); + if (group.isEmpty() || topic.isEmpty()) { + respond(responseObserver, ResetGroupOffsetResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "group and topic are required")).build()); + return; + } + // An unset protobuf Timestamp is 0, which would silently mean "replay everything since 1970". + if (!request.hasResetTimestamp() || request.getResetTimestamp().getSeconds() <= 0) { + respond(responseObserver, ResetGroupOffsetResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "reset_timestamp is required and must be a positive Unix time")) + .build()); + return; + } + long resetTimestamp = TimeUnit.SECONDS.toMillis(request.getResetTimestamp().getSeconds()) + + TimeUnit.NANOSECONDS.toMillis(request.getResetTimestamp().getNanos()); + List brokerAddrs; + try { + brokerAddrs = topicBrokerAddrs(topic, true); + } catch (Throwable t) { + respond(responseObserver, ResetGroupOffsetResponse.newBuilder().setStatus(err(t)).build()); + return; + } + log.info("resetGroupOffset group={} topic={} timestamp={} brokers={}", group, topic, resetTimestamp, brokerAddrs); + queryAllBrokers(brokerAddrs, (addr, timeout) -> + admin().resetOffset(addr, topic, group, resetTimestamp, true, timeout)) + .whenComplete((ignored, throwable) -> { + if (throwable != null) { + respond(responseObserver, ResetGroupOffsetResponse.newBuilder().setStatus(err(throwable)).build()); + } else { + respond(responseObserver, ResetGroupOffsetResponse.newBuilder().setStatus(ok()).build()); + } + }); + } + + // ========================================================================= + // 9. QueryMessage + // ========================================================================= + + @Override + public void queryMessage(ListMessageRequest request, StreamObserver responseObserver) { + String topic = request.getTopic().getName(); + if (topic.isEmpty()) { + respond(responseObserver, ListMessageResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "topic is required")).build()); + return; + } + int maxNums = request.getMaxMessageNums() > 0 ? request.getMaxMessageNums() : DEFAULT_MAX_MESSAGE_NUMS; + long begin = request.hasBeginTimestamp() ? TimeUnit.SECONDS.toMillis(request.getBeginTimestamp().getSeconds()) : 0L; + long end = request.hasEndTimestamp() ? TimeUnit.SECONDS.toMillis(request.getEndTimestamp().getSeconds()) + : Long.MAX_VALUE; + + List brokerAddrs; + try { + brokerAddrs = topicBrokerAddrs(topic, false); + } catch (Throwable t) { + respond(responseObserver, ListMessageResponse.newBuilder().setStatus(err(t)).build()); + return; + } + + String key; + boolean uniqueKey; + switch (request.getSearchKeyCase()) { + case MESSAGE_ID: + // a v2 message_id is the client-generated unique key, not the broker's offset-encoded + // id, so it must be looked up through the unique-key index on every broker that + // hosts the topic — decoding it as an offset would query unrelated data + key = request.getMessageId(); + uniqueKey = true; + break; + case MESSAGE_KEY: + key = request.getMessageKey(); + uniqueKey = false; + break; + case SUBSCRIPTION: + case LITE_TOPIC: + case SEARCHKEY_NOT_SET: + default: + respond(responseObserver, ListMessageResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, + "querying by " + request.getSearchKeyCase() + " is not supported; " + + "message_id or message_key is required")) + .build()); + return; + } + if (key == null || key.isEmpty()) { + respond(responseObserver, ListMessageResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "the search key must not be empty")).build()); + return; + } + + final String queryKey = key; + final boolean queryUniqueKey = uniqueKey; + queryAllBrokers(brokerAddrs, (addr, timeout) -> + admin().queryMessage(addr, topic, queryKey, maxNums, begin, end, queryUniqueKey, true, timeout)) + .whenComplete((byBroker, throwable) -> { + if (throwable != null) { + respond(responseObserver, ListMessageResponse.newBuilder().setStatus(err(throwable)).build()); + return; + } + ListMessageResponse.Builder builder = ListMessageResponse.newBuilder().setStatus(ok()); + if (request.hasScrollId()) { + builder.setScrollId(request.getScrollId()); + } + int count = 0; + outer: + for (List messages : byBroker.values()) { + if (messages == null) { + continue; + } + for (MessageExt messageExt : messages) { + if (messageExt == null) { + continue; + } + if (count >= maxNums) { + break outer; + } + // reuse the data plane converter so born/store timestamps, hosts, delivery + // time, message group and user properties are all carried over + builder.addMessages(GrpcConverter.getInstance().buildMessage(messageExt)); + count++; + } + } + if (count == 0) { + builder.setStatus(err(Code.MESSAGE_NOT_FOUND, + "no message found for " + (queryUniqueKey ? "message_id " : "message_key ") + queryKey)); + } + respond(responseObserver, builder.build()); + }); + } + + // ========================================================================= + // 10. PrintThreadStackTrace + // ========================================================================= + + @Override + public void printThreadStackTrace(PrintThreadStackTraceRequest request, + StreamObserver responseObserver) { + String group = request.getGroup().getName(); + String clientId = request.getClientId(); + if (forwarder.forwardIfRemote(group, clientId, responseObserver, + (stub, observer) -> stub.printThreadStackTrace(request, observer))) { + return; + } + relayConsumerRunningInfo(group, clientId, true) + .whenComplete((result, throwable) -> { + if (throwable != null) { + respond(responseObserver, PrintThreadStackTraceResponse.newBuilder() + .setStatus(err(throwable)).build()); + return; + } + org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo runningInfo = result.getResult(); + String jstack = runningInfo == null ? null : runningInfo.getJstack(); + if (result.getCode() != ResponseCode.SUCCESS || jstack == null || jstack.isEmpty()) { + respond(responseObserver, PrintThreadStackTraceResponse.newBuilder() + .setStatus(err(Code.NOT_FOUND, + "client " + clientId + " did not return a thread stack: " + result.getRemark())) + .build()); + return; + } + respond(responseObserver, PrintThreadStackTraceResponse.newBuilder() + .setStatus(ok()) + .setThreadStackTrace(jstack) + .build()); + }); + } + + // ========================================================================= + // 11. VerifyMessage + // ========================================================================= + + @Override + public void verifyMessage(VerifyMessageRequest request, StreamObserver responseObserver) { + String group = request.getGroup().getName(); + String clientId = request.getClientId(); + String topic = request.getTopic().getName(); + String messageId = request.getMessageId(); + if (group.isEmpty() || clientId.isEmpty() || topic.isEmpty() || messageId.isEmpty()) { + respond(responseObserver, VerifyMessageResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "group, client_id, topic and message_id are required")).build()); + return; + } + if (forwarder.forwardIfRemote(group, clientId, responseObserver, + (stub, observer) -> stub.verifyMessage(request, observer))) { + return; + } + // the real message has to be fetched first: asking a client to consume an empty shell would + // prove nothing about its consumer logic + List brokerAddrs; + try { + brokerAddrs = topicBrokerAddrs(topic, false); + } catch (Throwable t) { + respond(responseObserver, VerifyMessageResponse.newBuilder().setStatus(err(t)).build()); + return; + } + // The message is fetched with its body still compressed: it is handed to the client + // verbatim, so inflating it here would leave the body and the compression flag the client + // reads disagreeing with each other. + queryAllBrokers(brokerAddrs, (addr, timeout) -> + admin().queryMessage(addr, topic, messageId, 1, 0L, Long.MAX_VALUE, true, false, timeout)) + .whenComplete((byBroker, throwable) -> { + if (throwable != null) { + respond(responseObserver, VerifyMessageResponse.newBuilder().setStatus(err(throwable)).build()); + return; + } + MessageExt messageExt = null; + for (List messages : byBroker.values()) { + if (messages != null && !messages.isEmpty() && messages.get(0) != null) { + messageExt = messages.get(0); + break; + } + } + if (messageExt == null) { + // "no such message" is a normal answer, not a proxy fault + respond(responseObserver, VerifyMessageResponse.newBuilder() + .setStatus(err(Code.MESSAGE_NOT_FOUND, + "message " + messageId + " not found on topic " + topic)) + .build()); + return; + } + relayConsumeMessageDirectly(group, clientId, messageExt) + .whenComplete((result, relayThrowable) -> { + if (relayThrowable != null) { + respond(responseObserver, VerifyMessageResponse.newBuilder() + .setStatus(err(relayThrowable)).build()); + return; + } + ConsumeMessageDirectlyResult directlyResult = result.getResult(); + CMResult consumeResult = directlyResult == null ? null : directlyResult.getConsumeResult(); + if (consumeResult == CMResult.CR_SUCCESS) { + respond(responseObserver, VerifyMessageResponse.newBuilder().setStatus(ok()).build()); + } else { + respond(responseObserver, VerifyMessageResponse.newBuilder() + .setStatus(err(Code.MESSAGE_CORRUPTED, + "client " + clientId + " failed to consume message " + messageId + ": " + + (consumeResult == null ? result.getRemark() : consumeResult.name()) + + (directlyResult != null && directlyResult.getRemark() != null + ? " (" + directlyResult.getRemark() + ")" : ""))) + .build()); + } + }); + }); + } + + // ========================================================================= + // 12. AdminSendMessage + // ========================================================================= + + @Override + public void adminSendMessage(AdminSendMessageRequest request, StreamObserver responseObserver) { + String topic = request.getTopic().getName(); + if (topic.isEmpty()) { + respond(responseObserver, AdminSendMessageResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "topic is required")).build()); + return; + } + org.apache.rocketmq.common.message.Message message; + try { + message = buildAdminMessage(request); + } catch (Throwable t) { + // building the message validates caller-supplied properties, so a rejection has to be + // answered with a Status; letting it escape this method would reply with a bare gRPC + // error carrying no Status at all + respond(responseObserver, AdminSendMessageResponse.newBuilder().setStatus(err(t)).build()); + return; + } + // ext_info is request-scoped and unknown keys must be ignored, so it is only logged here + if (!request.getExtInfoMap().isEmpty()) { + log.info("adminSendMessage ext_info={}", request.getExtInfoMap()); + } + + List messages = new ArrayList<>(); + messages.add(message); + final String shardingKey = message.getProperty(MessageConst.PROPERTY_SHARDING_KEY); + long timeout = timeoutMillis(); + CompletableFuture> sendFuture; + try { + sendFuture = messagingProcessor.sendMessage(ctx(), + (queueContext, messageQueueView) -> selectQueue(messageQueueView, shardingKey), + ADMIN_SEND_PRODUCER_GROUP, 0, messages, timeout); + } catch (Throwable t) { + respond(responseObserver, AdminSendMessageResponse.newBuilder().setStatus(err(t)).build()); + return; + } + withTimeout(sendFuture, timeout * 2, "adminSendMessage") + .whenComplete((sendResults, throwable) -> { + if (throwable != null) { + respond(responseObserver, AdminSendMessageResponse.newBuilder().setStatus(err(throwable)).build()); + return; + } + String messageId = sendResults != null && !sendResults.isEmpty() && sendResults.get(0) != null + ? sendResults.get(0).getMsgId() : ""; + respond(responseObserver, AdminSendMessageResponse.newBuilder() + .setStatus(ok()) + .setMessageId(messageId == null ? "" : messageId) + .build()); + }); + } + + /** + * Builds the message an admin send request describes, mirroring what the data plane's + * {@code SendMessageActivity} does for a producer. + * + *

System properties are written with {@link MessageAccessor#putProperty}, never with + * {@code Message#putUserProperty}: the latter rejects every name in + * {@link MessageConst#STRING_HASH_SET}, so using it for a system property such as the delivery + * timestamp throws instead of sending the message. Caller-supplied user properties are checked + * against that same set first and refused as {@link Code#ILLEGAL_MESSAGE_PROPERTY_KEY}, so a + * request that tries to forge a system property gets a graded answer rather than an internal + * error. + */ + private static org.apache.rocketmq.common.message.Message buildAdminMessage(AdminSendMessageRequest request) { + org.apache.rocketmq.common.message.Message message = new org.apache.rocketmq.common.message.Message( + request.getTopic().getName(), request.getBody().toByteArray()); + if (request.hasTag() && !request.getTag().isEmpty()) { + message.setTags(request.getTag()); + } + if (request.hasKey() && !request.getKey().isEmpty()) { + message.setKeys(request.getKey()); + } + for (Map.Entry property : request.getUserPropertiesMap().entrySet()) { + if (MessageConst.STRING_HASH_SET.contains(property.getKey())) { + throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_PROPERTY_KEY, + "property is used by system: " + property.getKey()); + } + MessageAccessor.putProperty(message, property.getKey(), property.getValue()); + } + if (!request.hasSystemProperties()) { + return message; + } + SystemProperties systemProperties = request.getSystemProperties(); + if (systemProperties.hasDeliveryTimestamp()) { + Timestamp deliveryTimestamp = systemProperties.getDeliveryTimestamp(); + if (!Timestamps.isValid(deliveryTimestamp)) { + throw new GrpcProxyException(Code.ILLEGAL_DELIVERY_TIME, + "delivery_timestamp is not a valid timestamp"); + } + long deliverAt = Timestamps.toMillis(deliveryTimestamp); + if (deliverAt <= System.currentTimeMillis()) { + throw new GrpcProxyException(Code.ILLEGAL_DELIVERY_TIME, + "delivery_timestamp must be in the future"); + } + MessageAccessor.putProperty(message, MessageConst.PROPERTY_TIMER_DELIVER_MS, String.valueOf(deliverAt)); + } + if (!systemProperties.getMessageGroup().isEmpty()) { + MessageAccessor.putProperty(message, MessageConst.PROPERTY_SHARDING_KEY, + systemProperties.getMessageGroup()); + } + for (String key : systemProperties.getKeysList()) { + if (!key.isEmpty()) { + message.setKeys(key); + break; + } + } + if (systemProperties.hasTag() && !systemProperties.getTag().isEmpty()) { + message.setTags(systemProperties.getTag()); + } + return message; + } + + /** + * A FIFO message must keep going to the queue its message group hashes to; anything else is + * spread over the writable queues instead of always landing on queue 0. + */ + private static AddressableMessageQueue selectQueue(MessageQueueView messageQueueView, String shardingKey) { + List queues = messageQueueView.getWriteSelector().getQueues(); + if (queues == null || queues.isEmpty()) { + throw new IllegalStateException("no writable queue available"); + } + if (shardingKey != null && !shardingKey.isEmpty()) { + int index = Math.floorMod(shardingKey.hashCode(), queues.size()); + return queues.get(index); + } + return queues.get(ThreadLocalRandom.current().nextInt(queues.size())); + } + + // ========================================================================= + // 13. GetConsumerRunningInfo + // ========================================================================= + + @Override + public void getConsumerRunningInfo(GetConsumerRunningInfoRequest request, + StreamObserver responseObserver) { + String group = request.getGroup().getName(); + String clientId = request.getClientId(); + if (clientId.isEmpty()) { + respond(responseObserver, GetConsumerRunningInfoResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "client_id is required")).build()); + return; + } + if (forwarder.forwardIfRemote(group, clientId, responseObserver, + (stub, observer) -> stub.getConsumerRunningInfo(request, observer))) { + return; + } + relayConsumerRunningInfo(group, clientId, false) + .whenComplete((result, throwable) -> { + if (throwable == null && result.getCode() == ResponseCode.SUCCESS && result.getResult() != null) { + respond(responseObserver, GetConsumerRunningInfoResponse.newBuilder() + .setStatus(ok()) + .setConsumerRunningInfo(AdminModelConverter.toConsumerRunningInfo(result.getResult())) + .build()); + return; + } + // A gRPC v2 client cannot report running info: the telemetry contract has no reply + // message carrying properties, process queue snapshots or consume statistics. Only + // remoting clients can. Rather than returning an empty shell that looks like a + // healthy consumer, fall back to what the proxy does know and say so. + ConsumerRunningInfo.Builder builder = ConsumerRunningInfo.newBuilder(); + apache.rocketmq.v2.Settings settings = grpcClientSettingsManager.getRawClientSettings(clientId); + if (settings == null || !settings.hasSubscription()) { + respond(responseObserver, GetConsumerRunningInfoResponse.newBuilder() + .setStatus(err(throwable != null ? throwable + : new IllegalStateException("client " + clientId + " is not connected to this cluster"))) + .build()); + return; + } + for (apache.rocketmq.v2.SubscriptionEntry entry : settings.getSubscription().getSubscriptionsList()) { + builder.putSubscriptions(entry.hasTopic() ? entry.getTopic().getName() : "", + entry.hasExpression() ? entry.getExpression() + : apache.rocketmq.v2.FilterExpression.newBuilder() + .setType(apache.rocketmq.v2.FilterType.TAG).setExpression("*").build()); + } + // The client was found and its subscriptions are returned, so this is a success with + // an explanatory message rather than NOT_FOUND, which would claim the client does not + // exist and hide the fact that usable data came back. + respond(responseObserver, GetConsumerRunningInfoResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, + "only subscriptions are available: a gRPC v2 client does not report properties, " + + "process queue snapshots or consume statistics over the telemetry channel; " + + "use a remoting client for the full running info")) + .setConsumerRunningInfo(builder.build()) + .build()); + }); + } + + // ========================================================================= + // 14. GetTopicRoute + // ========================================================================= + + @Override + public void getTopicRoute(GetTopicRouteRequest request, StreamObserver responseObserver) { + String topic = request.getTopic().getName(); + if (topic.isEmpty()) { + respond(responseObserver, GetTopicRouteResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "topic is required")).build()); + return; + } + admin().getTopicRouteData(topic).whenComplete((topicRouteData, throwable) -> { + if (throwable != null) { + respond(responseObserver, GetTopicRouteResponse.newBuilder().setStatus(err(throwable)).build()); + return; + } + // topic_route_data is documented as the serialized route table, so the request's + // network_type / protocol_type / client_address hints have no bearing on it: they only + // matter when the response carries protocol-specific access points, which this one does not + respond(responseObserver, AdminModelConverter.toTopicRoute((TopicRouteData) topicRouteData)); + }); + } + + // ========================================================================= + // 15. QueryTimeSpan + // ========================================================================= + + @Override + public void queryTimeSpan(QueryTimeSpanRequest request, StreamObserver responseObserver) { + String group = request.getGroup().getName(); + if (group.isEmpty()) { + respond(responseObserver, QueryTimeSpanResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "group is required")).build()); + return; + } + Set topics = new LinkedHashSet<>(); + for (Resource resource : request.getTopicsList()) { + if (!resource.getName().isEmpty()) { + topics.add(resource.getName()); + } + } + if (topics.isEmpty()) { + respond(responseObserver, QueryTimeSpanResponse.newBuilder() + .setStatus(err(Code.BAD_REQUEST, "at least one topic is required")).build()); + return; + } + // every requested topic is queried, on every broker hosting it; the broker computes the + // per-queue min/max/consume timestamps and delay itself (QUERY_CONSUME_TIME_SPAN) + List>> tasks = new ArrayList<>(); + List failures = Collections.synchronizedList(new ArrayList<>()); + for (String topic : topics) { + List brokerAddrs; + try { + brokerAddrs = topicBrokerAddrs(topic, false); + } catch (Throwable t) { + failures.add(t); + log.warn("queryTimeSpan cannot resolve brokers for topic {} group {}", topic, group, t); + continue; + } + tasks.add(queryAllBrokers(brokerAddrs, (addr, timeout) -> + admin().queryConsumeTimeSpan(addr, topic, group, timeout)) + .thenApply(byBroker -> { + List spans = new ArrayList<>(); + for (List perBroker : byBroker.values()) { + if (perBroker != null) { + spans.addAll(perBroker); + } + } + return spans; + }) + .exceptionally(t -> { + failures.add(t); + return Collections.emptyList(); + })); + } + CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])).whenComplete((ignored, throwable) -> { + List all = new ArrayList<>(); + for (CompletableFuture> task : tasks) { + List spans = task.getNow(null); + if (spans != null) { + all.addAll(spans); + } + } + if (all.isEmpty() && !failures.isEmpty()) { + respond(responseObserver, QueryTimeSpanResponse.newBuilder().setStatus(err(failures.get(0))).build()); + return; + } + try { + respond(responseObserver, AdminModelConverter.toQueryTimeSpan(all)); + } catch (Throwable t) { + respond(responseObserver, QueryTimeSpanResponse.newBuilder().setStatus(err(t)).build()); + } + }); + } + + // ========================================================================= + // 16. GetProxyRuntimeStats + // ========================================================================= + + @Override + public void getProxyRuntimeStats(GetProxyRuntimeStatsRequest request, + StreamObserver responseObserver) { + try { + long producers = 0L; + long consumers = 0L; + Collection channels = grpcChannelManager.getClientChannels(); + for (GrpcClientChannel channel : channels) { + apache.rocketmq.v2.Settings settings = grpcClientSettingsManager.getRawClientSettings(channel.getClientId()); + if (settings == null) { + continue; + } + switch (settings.getClientType()) { + case PRODUCER: + producers++; + break; + case PUSH_CONSUMER: + case SIMPLE_CONSUMER: + case PULL_CONSUMER: + case LITE_PUSH_CONSUMER: + case LITE_SIMPLE_CONSUMER: + consumers++; + break; + default: + break; + } + } + // in_tps / out_tps stay unset on purpose: the open-source proxy keeps no per-process + // throughput counter, and reporting 0 would be indistinguishable from an idle proxy + respond(responseObserver, GetProxyRuntimeStatsResponse.newBuilder() + .setStatus(ok()) + .setProxyName(ConfigurationManager.getProxyConfig().getProxyName()) + .setVersion(MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION)) + .setConnections(channels.size()) + .setProducers(producers) + .setConsumers(consumers) + .build()); + } catch (Throwable t) { + log.warn("getProxyRuntimeStats failed", t); + respond(responseObserver, GetProxyRuntimeStatsResponse.newBuilder().setStatus(err(t)).build()); + } + } + + // ========================================================================= + // client relay + // ========================================================================= + + /** + * Asks a connected client for its running info through the proxy's in-process relay. The result + * future is carried on the {@link ProxyRelayRequest} so that + * {@code ProxyChannel.writeAndFlush} can hand it to the channel implementation, which completes + * it when the client answers. + */ + private CompletableFuture> + relayConsumerRunningInfo(String group, String clientId, boolean jstack) { + CompletableFuture> future = + new CompletableFuture<>(); + ClientChannelInfo channelInfo = findClientChannel(group, clientId); + if (channelInfo == null || channelInfo.getChannel() == null) { + future.complete(new ProxyRelayResult<>(ResponseCode.SYSTEM_ERROR, + "client " + clientId + " is not connected to this cluster", null)); + return future; + } + if (!channelInfo.getChannel().isActive()) { + future.complete(new ProxyRelayResult<>(ResponseCode.SYSTEM_ERROR, + "client " + clientId + " channel is inactive", null)); + return future; + } + try { + GetConsumerRunningInfoRequestHeader header = new GetConsumerRunningInfoRequestHeader(); + header.setConsumerGroup(group); + header.setClientId(clientId); + header.setJstackEnable(jstack); + ProxyRelayRequest relayRequest = ProxyRelayRequest.createRequestCommand( + RequestCode.GET_CONSUMER_RUNNING_INFO, header, future); + channelInfo.getChannel().writeAndFlush(relayRequest); + } catch (Throwable t) { + future.completeExceptionally(t); + return future; + } + return withTimeout(future, relayTimeoutMillis(), "getConsumerRunningInfo for client " + clientId); + } + + private CompletableFuture> relayConsumeMessageDirectly( + String group, String clientId, MessageExt messageExt) { + CompletableFuture> future = new CompletableFuture<>(); + ClientChannelInfo channelInfo = findClientChannel(group, clientId); + if (channelInfo == null || channelInfo.getChannel() == null) { + future.complete(new ProxyRelayResult<>(ResponseCode.SYSTEM_ERROR, + "client " + clientId + " is not connected to this cluster", null)); + return future; + } + try { + ConsumeMessageDirectlyResultRequestHeader header = new ConsumeMessageDirectlyResultRequestHeader(); + header.setConsumerGroup(group); + header.setClientId(clientId); + header.setTopic(messageExt.getTopic()); + header.setMsgId(messageExt.getMsgId()); + // the header requires a broker name although the write never reaches a broker; the + // message's own store queue is the most meaningful value available here + header.setBrokerName(messageExt.getBrokerName() == null ? "" : messageExt.getBrokerName()); + ProxyRelayRequest relayRequest = ProxyRelayRequest.createRequestCommand( + RequestCode.CONSUME_MESSAGE_DIRECTLY, header, future); + // storeSize is what the broker recorded on disk and MessageDecoder.encode allocates + // exactly that many bytes; zero it so the size is recomputed from what is actually being + // written, or a body that no longer matches the stored one overflows the buffer + messageExt.setStoreSize(0); + relayRequest.setBody(org.apache.rocketmq.common.message.MessageDecoder.encode(messageExt, false)); + channelInfo.getChannel().writeAndFlush(relayRequest); + } catch (Throwable t) { + future.completeExceptionally(t); + return future; + } + return withTimeout(future, relayTimeoutMillis(), "verifyMessage for client " + clientId); + } + + private long relayTimeoutMillis() { + return TimeUnit.SECONDS.toMillis( + ConfigurationManager.getProxyConfig().getGrpcProxyRelayRequestTimeoutInSeconds()) + 1000L; + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + this.adminTimeoutScheduler.shutdownNow(); + } + + @Override + public String toString() { + return "ProxyAdminGrpcService{proxyName=" + ConfigurationManager.getProxyConfig().getProxyName() + + ", adminPort=" + ConfigurationManager.getProxyConfig().getGrpcAdminServerPort() + "}"; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessagingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessagingActivity.java index 88099207b93..fc602c99eaa 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessagingActivity.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessagingActivity.java @@ -97,6 +97,14 @@ protected void init(MessagingProcessor messagingProcessor) { this.appendStartAndShutdown(this.grpcClientSettingsManager); } + public GrpcChannelManager getGrpcChannelManager() { + return this.grpcChannelManager; + } + + public GrpcClientSettingsManager getGrpcClientSettingsManager() { + return this.grpcClientSettingsManager; + } + @Override public CompletableFuture queryRoute(ProxyContext ctx, QueryRouteRequest request) { return this.routeActivity.queryRoute(ctx, request); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java index 3429ad54e27..a2b1150bf2d 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java @@ -78,6 +78,10 @@ public class GrpcMessagingApplication extends MessagingServiceGrpc.MessagingServ private final GrpcMessagingActivity grpcMessagingActivity; + public GrpcMessagingActivity getGrpcMessagingActivity() { + return this.grpcMessagingActivity; + } + protected final RequestPipeline requestPipeline; protected ThreadPoolExecutor routeThreadPoolExecutor; diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java index a18cf7600c1..306b5ee486c 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.proxy.grpc.v2.channel; +import java.util.Collection; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -69,6 +70,10 @@ public GrpcClientChannel getChannel(String clientId) { return clientIdChannelMap.get(clientId); } + public Collection getClientChannels() { + return clientIdChannelMap.values(); + } + public GrpcClientChannel removeChannel(String clientId) { return this.clientIdChannelMap.remove(clientId); } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java index 0135818fb3b..7c42b7f47d1 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java @@ -52,6 +52,7 @@ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; import org.apache.rocketmq.proxy.service.transaction.TransactionData; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult; import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader; @@ -233,6 +234,21 @@ protected CompletableFuture processGetConsumerRunningInfo(RemotingCommand GetConsumerRunningInfoRequestHeader header, CompletableFuture> responseFuture) { if (Objects.isNull(header) || !header.isJstackEnable()) { + // The gRPC v2 telemetry protocol only offers PrintThreadStackTraceCommand for this + // request: there is no command that reports consumer running info without a stack dump. + // Returning without touching responseFuture used to leave the caller hanging until + // GrpcChannelManager#scanExpireResultFuture fired SYSTEM_BUSY "call remote timeout", + // which both wastes grpcProxyRelayRequestTimeoutInSeconds and misreports the cause as a + // timeout. REQUEST_CODE_NOT_SUPPORTED is the accurate code here: the request itself is + // understood and the peer is healthy, the protocol simply cannot serve this variant. + // (SYSTEM_ERROR/SYSTEM_BUSY would advertise a retryable proxy-side fault, which is + // wrong and would make callers loop.) + if (responseFuture != null) { + responseFuture.complete(new ProxyRelayResult<>(ResponseCode.REQUEST_CODE_NOT_SUPPORTED, + "gRPC v2 protocol cannot report consumer running info without jstack, " + + "retry with jstackEnable=true", + null)); + } return CompletableFuture.completedFuture(null); } this.writeTelemetryCommand(TelemetryCommand.newBuilder() diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcConverter.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcConverter.java index 87d20ebca1b..695ac137d9c 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcConverter.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcConverter.java @@ -109,10 +109,11 @@ public Message buildMessage(MessageExt messageExt) { protected Map buildUserAttributes(MessageExt messageExt) { Map userAttributes = new HashMap<>(); Map properties = messageExt.getProperties(); - - for (Map.Entry property : properties.entrySet()) { - if (!MessageConst.STRING_HASH_SET.contains(property.getKey())) { - userAttributes.put(property.getKey(), property.getValue()); + if (properties != null) { + for (Map.Entry property : properties.entrySet()) { + if (!MessageConst.STRING_HASH_SET.contains(property.getKey())) { + userAttributes.put(property.getKey(), property.getValue()); + } } } @@ -161,7 +162,9 @@ protected SystemProperties buildSystemProperties(MessageExt messageExt) { } // message_type - TopicMessageType topicMessageType = TopicMessageType.parseFromMessageProperty(messageExt.getProperties()); + Map properties = messageExt.getProperties(); + TopicMessageType topicMessageType = TopicMessageType.parseFromMessageProperty( + properties != null ? properties : new HashMap()); systemPropertiesBuilder.setMessageType(convertToGrpcMessageType(topicMessageType)); // born_timestamp (millis) diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java index a56bc42596b..a9ebe83c178 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java @@ -385,6 +385,10 @@ public MetadataService getMetadataService() { return this.serviceManager.getMetadataService(); } + public ServiceManager getServiceManager() { + return this.serviceManager; + } + @Override public void addReceiptHandle(ProxyContext ctx, Channel channel, String group, String msgID, MessageReceiptHandle messageReceiptHandle) { diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java index 8b1c20c0bdb..77c61a2dfdc 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java @@ -154,6 +154,7 @@ protected void init() { this.appendShutdown(scheduledExecutorService::shutdown); this.appendStartAndShutdown(this.messagingClientAPIFactory); this.appendStartAndShutdown(this.operationClientAPIFactory); + this.appendStartAndShutdown(this.adminService); this.appendStartAndShutdown(this.transactionClientAPIFactory); this.appendStartAndShutdown(this.liteSubscriptionAPIFactory); this.appendStartAndShutdown(this.topicRouteService); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java index 8f5073bb3aa..6e7827c4ebf 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java @@ -88,6 +88,7 @@ public LocalServiceManager(BrokerController brokerController, RPCHook rpcHook) { protected void init() { this.appendStartAndShutdown(this.mqClientAPIFactory); this.appendStartAndShutdown(this.topicRouteService); + this.appendStartAndShutdown(this.adminService); this.appendStartAndShutdown(new LocalServiceManagerStartAndShutdown()); } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/AdminService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/AdminService.java index a9e6686b438..4b8f3f9889b 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/AdminService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/AdminService.java @@ -14,13 +14,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.rocketmq.proxy.service.admin; import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.utils.StartAndShutdown; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; +import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; +import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; +import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; +import org.apache.rocketmq.remoting.protocol.body.GroupList; +import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan; +import org.apache.rocketmq.remoting.protocol.body.TopicList; import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; + +public interface AdminService extends StartAndShutdown { + + @Override + default void start() { + } -public interface AdminService { + @Override + default void shutdown() { + } boolean topicExist(String topic); @@ -29,4 +51,68 @@ boolean createTopicOnTopicBrokerIfNotExist(String createTopic, String sampleTopi boolean createTopicOnBroker(String topic, int wQueueNum, int rQueueNum, List curBrokerDataList, List sampleBrokerDataList, boolean examineTopic, int retryCheckCount) throws Exception; + + // ========================================================================= + // Admin gateway: broker-facing queries and mutations. + // + // Every call is delegated to the proxy's OWN managed broker client + // (rocketmq-proxy's MQClientAPIFactory), so the admin surface never opens a + // direct link to a broker. All methods are asynchronous: the gRPC admin + // handlers run on the shared gRPC executor and must not block it, and most + // of them fan out to several brokers at once. + // + // A blank {@code topic} on the statistics methods means "every topic of the + // group", which the broker resolves from its own consumer offset table. + // ========================================================================= + + CompletableFuture getBrokerClusterInfo(long timeoutMillis); + + CompletableFuture getTopicRouteData(String topic); + + CompletableFuture getTopicConfig(String brokerAddr, String topic, long timeoutMillis); + + CompletableFuture getTopicStats(String brokerAddr, String topic, long timeoutMillis); + + CompletableFuture getConsumeStats(String brokerAddr, String group, String topic, long timeoutMillis); + + CompletableFuture> queryConsumeTimeSpan(String brokerAddr, String topic, String group, + long timeoutMillis); + + CompletableFuture> resetOffset(String brokerAddr, String topic, String group, + long timestamp, boolean isForce, long timeoutMillis); + + CompletableFuture getConsumerConnectionList(String brokerAddr, String group, + long timeoutMillis); + + CompletableFuture queryTopicConsumeByWho(String brokerAddr, String topic, long timeoutMillis); + + CompletableFuture queryTopicsByConsumer(String brokerAddr, String group, long timeoutMillis); + + CompletableFuture getSubscriptionGroupConfig(String brokerAddr, String group, + long timeoutMillis); + + CompletableFuture updateSubscriptionGroupConfig(String brokerAddr, SubscriptionGroupConfig config, + long timeoutMillis); + + CompletableFuture deleteSubscriptionGroup(String brokerAddr, String group, boolean cleanOffset, + long timeoutMillis); + + /** + * Query messages of a topic. When {@code uniqueKey} is true the key is treated as the + * client-generated unique message id (what the gRPC protocol exposes as {@code message_id}); + * otherwise it is matched against the message keys index. + * + * @param decompressBody inflate a compressed body before returning. Callers that render the + * message (QueryMessage) want it inflated; callers that hand the message + * back to a client as-is (VerifyMessage) must not, so the body keeps + * matching the compression flag the client will act on. + */ + CompletableFuture> queryMessage(String brokerAddr, String topic, String key, int maxNum, + long beginTimestamp, long endTimestamp, boolean uniqueKey, boolean decompressBody, long timeoutMillis); + + CompletableFuture viewMessage(String brokerAddr, String topic, long physicalOffset, + long timeoutMillis); + + CompletableFuture searchOffsetByTimestamp(String brokerAddr, MessageQueue messageQueue, long timestamp, + long timeoutMillis); } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java index f3c68eab5c4..af668ac04f0 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminService.java @@ -20,24 +20,72 @@ import java.time.Duration; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.rocketmq.client.impl.admin.MqClientAdminImpl; +import org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt; +import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; -import org.apache.rocketmq.remoting.protocol.route.BrokerData; -import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; -import org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt; -import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; import org.apache.rocketmq.proxy.service.route.TopicRouteHelper; +import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; +import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; +import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; +import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; +import org.apache.rocketmq.remoting.protocol.body.GroupList; +import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan; +import org.apache.rocketmq.remoting.protocol.body.TopicList; +import org.apache.rocketmq.remoting.protocol.header.DeleteSubscriptionGroupRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetConsumerConnectionListRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetTopicStatsInfoRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryConsumeTimeSpanRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryTopicConsumeByWhoRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryTopicsByConsumerRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; public class DefaultAdminService implements AdminService { private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + private static final long ROUTE_TIMEOUT_MILLIS = Duration.ofSeconds(3).toMillis(); + private final MQClientAPIFactory mqClientAPIFactory; + /** + * A few broker requests only have a blocking client method (topic config, subscription group + * config, search offset, nameserver route). The admin gateway promises non-blocking calls so + * that a handler fanning out to N brokers never parks a gRPC executor thread, so those are + * lifted onto this pool. Daemon threads: the pool lives as long as the proxy process and must + * not keep it alive on shutdown. + */ + private final ExecutorService blockingCallExecutor = Executors.newFixedThreadPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), new ThreadFactory() { + private final AtomicInteger seq = new AtomicInteger(); + + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r, "AdminBlockingCall_" + seq.getAndIncrement()); + thread.setDaemon(true); + return thread; + } + }); + public DefaultAdminService(MQClientAPIFactory mqClientAPIFactory) { this.mqClientAPIFactory = mqClientAPIFactory; } @@ -116,7 +164,7 @@ public boolean createTopicOnBroker(String topic, int wQueueNum, int rQueueNum, L } try { - this.getClient().createTopic(addr, TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC, topicConfig, Duration.ofSeconds(3).toMillis()); + this.getClient().createTopic(addr, TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC, topicConfig, ROUTE_TIMEOUT_MILLIS); } catch (Exception e) { log.error("create topic on broker failed. topic:{}, broker:{}", topicConfig, addr, e); } @@ -137,10 +185,179 @@ public boolean createTopicOnBroker(String topic, int wQueueNum, int rQueueNum, L } protected TopicRouteData getTopicRouteDataDirectlyFromNameServer(String topic) throws Exception { - return this.getClient().getTopicRouteInfoFromNameServer(topic, Duration.ofSeconds(3).toMillis()); + return this.getClient().getTopicRouteInfoFromNameServer(topic, ROUTE_TIMEOUT_MILLIS); } protected MQClientAPIExt getClient() { return this.mqClientAPIFactory.getClient(); } + + protected MqClientAdminImpl getAdmin() { + return this.getClient().getMqClientAdmin(); + } + + @Override + public void start() { + } + + @Override + public void shutdown() { + this.blockingCallExecutor.shutdownNow(); + } + + private CompletableFuture supplyBlocking(java.util.function.Supplier call) { + return CompletableFuture.supplyAsync(call, blockingCallExecutor); + } + + // ========================================================================= + // Admin gateway: broker-facing queries and mutations. + // Every call goes through the proxy's OWN managed broker client. + // ========================================================================= + + @Override + public CompletableFuture getBrokerClusterInfo(long timeoutMillis) { + // a null address makes the remoting client talk to the nameserver + return this.getAdmin().getBrokerClusterInfo(null, timeoutMillis); + } + + @Override + public CompletableFuture getTopicRouteData(String topic) { + return supplyBlocking(() -> { + try { + return this.getTopicRouteDataDirectlyFromNameServer(topic); + } catch (Exception e) { + throw new java.util.concurrent.CompletionException(e); + } + }); + } + + @Override + public CompletableFuture getTopicConfig(String brokerAddr, String topic, long timeoutMillis) { + return supplyBlocking(() -> { + try { + return this.getClient().getTopicConfig(brokerAddr, topic, timeoutMillis); + } catch (Exception e) { + throw new java.util.concurrent.CompletionException(e); + } + }); + } + + @Override + public CompletableFuture getTopicStats(String brokerAddr, String topic, long timeoutMillis) { + GetTopicStatsInfoRequestHeader header = new GetTopicStatsInfoRequestHeader(); + header.setTopic(topic); + return this.getAdmin().getTopicStatsInfo(brokerAddr, header, timeoutMillis); + } + + @Override + public CompletableFuture getConsumeStats(String brokerAddr, String group, String topic, + long timeoutMillis) { + GetConsumeStatsRequestHeader header = new GetConsumeStatsRequestHeader(); + header.setConsumerGroup(group); + // a blank topic tells the broker to collect stats over every topic the group has offsets for + header.setTopic(topic == null ? "" : topic); + return this.getAdmin().getConsumeStats(brokerAddr, header, timeoutMillis); + } + + @Override + public CompletableFuture> queryConsumeTimeSpan(String brokerAddr, String topic, String group, + long timeoutMillis) { + QueryConsumeTimeSpanRequestHeader header = new QueryConsumeTimeSpanRequestHeader(); + header.setTopic(topic); + header.setGroup(group); + return this.getAdmin().queryConsumeTimeSpan(brokerAddr, header, timeoutMillis); + } + + @Override + public CompletableFuture> resetOffset(String brokerAddr, String topic, String group, + long timestamp, boolean isForce, long timeoutMillis) { + ResetOffsetRequestHeader header = new ResetOffsetRequestHeader(); + header.setTopic(topic); + header.setGroup(group); + header.setTimestamp(timestamp); + header.setForce(isForce); + return this.getAdmin().invokeBrokerToResetOffset(brokerAddr, header, timeoutMillis); + } + + @Override + public CompletableFuture getConsumerConnectionList(String brokerAddr, String group, + long timeoutMillis) { + GetConsumerConnectionListRequestHeader header = new GetConsumerConnectionListRequestHeader(); + header.setConsumerGroup(group); + return this.getAdmin().getConsumerConnectionList(brokerAddr, header, timeoutMillis); + } + + @Override + public CompletableFuture queryTopicConsumeByWho(String brokerAddr, String topic, long timeoutMillis) { + QueryTopicConsumeByWhoRequestHeader header = new QueryTopicConsumeByWhoRequestHeader(); + header.setTopic(topic); + return this.getAdmin().queryTopicConsumeByWho(brokerAddr, header, timeoutMillis); + } + + @Override + public CompletableFuture queryTopicsByConsumer(String brokerAddr, String group, long timeoutMillis) { + QueryTopicsByConsumerRequestHeader header = new QueryTopicsByConsumerRequestHeader(); + header.setGroup(group); + return this.getAdmin().queryTopicsByConsumer(brokerAddr, header, timeoutMillis); + } + + @Override + public CompletableFuture getSubscriptionGroupConfig(String brokerAddr, String group, + long timeoutMillis) { + return supplyBlocking(() -> { + try { + return this.getClient().getSubscriptionGroupConfig(brokerAddr, group, timeoutMillis); + } catch (Exception e) { + throw new java.util.concurrent.CompletionException(e); + } + }); + } + + @Override + public CompletableFuture updateSubscriptionGroupConfig(String brokerAddr, SubscriptionGroupConfig config, + long timeoutMillis) { + return this.getAdmin().updateOrCreateSubscriptionGroup(brokerAddr, config, timeoutMillis); + } + + @Override + public CompletableFuture deleteSubscriptionGroup(String brokerAddr, String group, boolean cleanOffset, + long timeoutMillis) { + DeleteSubscriptionGroupRequestHeader header = new DeleteSubscriptionGroupRequestHeader(); + header.setGroupName(group); + header.setCleanOffset(cleanOffset); + return this.getAdmin().deleteSubscriptionGroup(brokerAddr, header, timeoutMillis); + } + + @Override + public CompletableFuture> queryMessage(String brokerAddr, String topic, String key, int maxNum, + long beginTimestamp, long endTimestamp, boolean uniqueKey, boolean decompressBody, long timeoutMillis) { + QueryMessageRequestHeader header = new QueryMessageRequestHeader(); + header.setTopic(topic); + header.setKey(key); + header.setMaxNum(maxNum); + header.setBeginTimestamp(beginTimestamp); + header.setEndTimestamp(endTimestamp); + return this.getAdmin().queryMessage(brokerAddr, uniqueKey, decompressBody, header, timeoutMillis); + } + + @Override + public CompletableFuture viewMessage(String brokerAddr, String topic, long physicalOffset, + long timeoutMillis) { + ViewMessageRequestHeader header = new ViewMessageRequestHeader(); + header.setTopic(topic); + header.setOffset(physicalOffset); + return this.getAdmin().viewMessage(brokerAddr, header, timeoutMillis); + } + + @Override + public CompletableFuture searchOffsetByTimestamp(String brokerAddr, MessageQueue messageQueue, + long timestamp, long timeoutMillis) { + return supplyBlocking(() -> { + try { + return this.getClient().searchOffset(brokerAddr, messageQueue, timestamp, timeoutMillis); + } catch (Exception e) { + throw new java.util.concurrent.CompletionException(e); + } + }); + } } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ClusterProxyRelayService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ClusterProxyRelayService.java index 71ce222a8c0..6aed6ff6d90 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ClusterProxyRelayService.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ClusterProxyRelayService.java @@ -26,7 +26,20 @@ import org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader; /** - * not implement yet + * Relay service used when the proxy runs in CLUSTER mode, i.e. there is no co-located broker + * whose {@code NettyRemotingAbstract} could receive the relayed answer (that is what + * {@link LocalProxyRelayService} does). + * + *

In CLUSTER mode the proxy process itself is the caller: it writes a + * {@link ProxyRelayRequest} into a {@link ProxyChannel} and waits on the future carried by that + * request. This class therefore hands back a real, non-null relay future (returning {@code null} + * used to make {@code GrpcChannelManager#addResponseFuture} store a null and the caller never + * observe any result) and bridges it to the caller's future, including the exceptional path and + * the {@code GrpcChannelManager#scanExpireResultFuture} timeout completion. + * + *

A plain {@link RemotingCommand} (not a {@link ProxyRelayRequest}) has no in-process waiter; + * the relay future is still returned so that the channel implementation can register it for the + * client answer, but nobody observes the outcome. */ public class ClusterProxyRelayService extends AbstractProxyRelayService { @@ -38,13 +51,41 @@ public ClusterProxyRelayService(TransactionService transactionService) { public CompletableFuture> processGetConsumerRunningInfo( ProxyContext context, RemotingCommand command, GetConsumerRunningInfoRequestHeader header) { - return null; + CompletableFuture> relayFuture = new CompletableFuture<>(); + bridgeToCaller(command, relayFuture); + return relayFuture; } @Override public CompletableFuture> processConsumeMessageDirectly( ProxyContext context, RemotingCommand command, ConsumeMessageDirectlyResultRequestHeader header) { - return null; + CompletableFuture> relayFuture = new CompletableFuture<>(); + bridgeToCaller(command, relayFuture); + return relayFuture; + } + + /** + * Propagates the relay outcome to the in-process caller that attached its future to the + * command. Both normal and exceptional completions are forwarded so the caller never has to + * fall back on its own timeout. + */ + private static void bridgeToCaller(RemotingCommand command, + CompletableFuture> relayFuture) { + if (!(command instanceof ProxyRelayRequest)) { + return; + } + CompletableFuture> callerFuture = + ((ProxyRelayRequest) command).typedResponseFuture(); + if (callerFuture == null) { + return; + } + relayFuture.whenComplete((relayResult, throwable) -> { + if (throwable != null) { + callerFuture.completeExceptionally(throwable); + } else { + callerFuture.complete(relayResult); + } + }); } } diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyRelayRequest.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyRelayRequest.java new file mode 100644 index 00000000000..982cd2e54d9 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyRelayRequest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.relay; + +import com.alibaba.fastjson2.annotation.JSONField; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; + +/** + * A {@link RemotingCommand} that travels through the in-process relay pipeline + * ({@code ProxyChannel#writeAndFlush}) instead of a real Netty socket, and therefore has to + * carry the caller's {@link CompletableFuture} along with the wire fields. + * + *

Why this exists: {@code ProxyChannel#writeAndFlush(Object)} can only return a Netty + * {@code ChannelFuture}, so an in-process caller (CLUSTER-mode proxy admin) that writes a + * {@code GET_CONSUMER_RUNNING_INFO} / {@code CONSUME_MESSAGE_DIRECTLY} command into a + * {@code ProxyChannel} has no way to observe the client's answer. The relay future created by + * {@link ProxyRelayService} is consumed by the concrete channel implementation (it is handed to + * {@code GrpcChannelManager#addResponseFuture}) and never surfaces again. Attaching the caller's + * future to the command lets {@link ClusterProxyRelayService} bridge the two. + * + *

Payload typing. The future is stored type-erased. The request code fully determines + * the payload type and is the only contract between the caller and the relay: + *

    + *
  • {@code RequestCode.GET_CONSUMER_RUNNING_INFO} → a future of {@link ProxyRelayResult} + * whose payload is a {@link ConsumerRunningInfo}
    • + *
  • {@code RequestCode.CONSUME_MESSAGE_DIRECTLY} → a future of {@link ProxyRelayResult} + * whose payload is a {@link ConsumeMessageDirectlyResult}
    • + *
+ * + *

This command is strictly in-process: it is never encoded onto a socket, so the non-wire + * field below is excluded from the fastjson2 view of the command. + */ +public class ProxyRelayRequest extends RemotingCommand { + + /** + * The in-process caller's future, kept type-erased (see the class javadoc for the code to + * payload mapping). Never serialized. + */ + @JSONField(serialize = false) + private final transient CompletableFuture responseFuture; + + protected ProxyRelayRequest(CompletableFuture responseFuture) { + this.responseFuture = responseFuture; + } + + /** + * Mirrors {@link RemotingCommand#createRequestCommand(int, CommandCustomHeader)}. The only + * difference is that the private {@code customHeader} field of the superclass is populated + * through the public {@link #writeCustomHeader(CommandCustomHeader)} hook, so that + * {@code ProxyChannel#writeAndFlush} can read it back with {@code readCustomHeader()}. + * + * @param code {@code RequestCode.GET_CONSUMER_RUNNING_INFO} or + * {@code RequestCode.CONSUME_MESSAGE_DIRECTLY} + * @param customHeader the matching request header + * @param responseFuture the caller's future, typed as documented on the class + */ + public static ProxyRelayRequest createRequestCommand(int code, CommandCustomHeader customHeader, + CompletableFuture responseFuture) { + ProxyRelayRequest cmd = new ProxyRelayRequest(responseFuture); + cmd.setCode(code); + cmd.writeCustomHeader(customHeader); + setCmdVersion(cmd); + return cmd; + } + + /** + * Type-erased view of the caller's future. Prefer {@link #typedResponseFuture()} unless the + * caller only needs to know whether a future was attached at all. + */ + public CompletableFuture getResponseFuture() { + return responseFuture; + } + + /** + * Typed view of the caller's future, e.g. + * {@code CompletableFuture>}. + * + * @param the payload type implied by the request code + */ + public CompletableFuture> typedResponseFuture() { + return castResponseFuture(this.responseFuture); + } + + /** + * The single unavoidable unchecked cast of this design. + * + *

It is safe because both ends of the future live inside the same proxy process and are + * written by the same feature: the caller creates + * {@code CompletableFuture>} (or the + * {@link ConsumeMessageDirectlyResult} twin), pairs it with the matching request code in + * {@link #createRequestCommand}, and the relay only ever completes it with the payload of the + * same {@link ProxyRelayService} method that the request code selected. The command never + * crosses a process boundary, so no untrusted producer can violate the pairing. + */ + @SuppressWarnings("unchecked") + private static CompletableFuture> castResponseFuture(CompletableFuture raw) { + return (CompletableFuture>) raw; + } +} diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/GrpcServerInterceptorOrderTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/GrpcServerInterceptorOrderTest.java new file mode 100644 index 00000000000..2c1044342cb --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/GrpcServerInterceptorOrderTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc; + +import io.grpc.Attributes; +import io.grpc.Metadata; +import io.grpc.ServerBuilder; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import java.util.List; +import org.apache.rocketmq.common.constant.GrpcConstants; +import org.apache.rocketmq.proxy.grpc.constant.AttributeKeys; +import org.apache.rocketmq.proxy.grpc.interceptor.HeaderInterceptor; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * The admin server authenticates in an interceptor, and authentication reads the channel id from the + * request metadata. Since authentication results are cached per channel id, it must never run on the + * client-supplied value: a second connection could otherwise present the same id and reuse the first + * connection's successful authentication. These tests pin the two properties that together prevent + * that — the ordering of the pipeline, and the header normalization it performs. + */ +public class GrpcServerInterceptorOrderTest { + + /** + * gRPC invokes server interceptors in reverse registration order, so an interceptor that must + * run after {@link HeaderInterceptor} has to be registered before it. + */ + @Test + public void postHeaderInterceptorsAreOrderedAfterHeaderInterceptorTest() { + ServerInterceptor authInterceptor = mock(ServerInterceptor.class); + @SuppressWarnings("unchecked") + ServerBuilder serverBuilder = mock(ServerBuilder.class); + + GrpcServerBuilder.configureInterceptors(serverBuilder, authInterceptor); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ServerInterceptor.class); + verify(serverBuilder, atLeastOnce()).intercept(captor.capture()); + List registrationOrder = captor.getAllValues(); + + int authIndex = registrationOrder.indexOf(authInterceptor); + int headerIndex = -1; + for (int i = 0; i < registrationOrder.size(); i++) { + if (registrationOrder.get(i) instanceof HeaderInterceptor) { + headerIndex = i; + } + } + + assertSame(authInterceptor, registrationOrder.get(0)); + assertTrue("HeaderInterceptor must be registered", headerIndex >= 0); + // registered earlier means executed later, so authentication observes normalized headers + assertTrue("authentication must be registered before HeaderInterceptor so that it runs after it", + authIndex < headerIndex); + } + + /** + * The normalization the ordering above depends on: whatever channel id the client puts on the + * wire is discarded in favour of the one derived from the transport. + */ + @Test + public void headerInterceptorReplacesClientSuppliedChannelIdTest() { + Metadata headers = new Metadata(); + headers.put(GrpcConstants.CHANNEL_ID, "forged-by-client"); + + ServerCall call = mock(ServerCall.class); + when(call.getAttributes()).thenReturn(Attributes.newBuilder() + .set(AttributeKeys.CHANNEL_ID, "transport-channel-id") + .build()); + ServerCallHandler next = mock(ServerCallHandler.class); + + new HeaderInterceptor().interceptCall(call, headers, next); + + assertEquals("transport-channel-id", headers.get(GrpcConstants.CHANNEL_ID)); + verify(next).startCall(any(), any()); + } +} diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/admin/AdminModelConverterTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/admin/AdminModelConverterTest.java new file mode 100644 index 00000000000..40608cab5c7 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/admin/AdminModelConverterTest.java @@ -0,0 +1,533 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.admin; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.DescribeGroupAccumulationResponse.GroupAccumulation; +import apache.rocketmq.v2.DescribeTopicStatusResponse; +import apache.rocketmq.v2.FilterType; +import apache.rocketmq.v2.GetTopicRouteResponse; +import apache.rocketmq.v2.MessageQueueItem; +import apache.rocketmq.v2.MessageType; +import apache.rocketmq.v2.QueryTimeSpanResponse; +import com.alibaba.fastjson2.JSON; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.filter.ExpressionType; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.protocol.LanguageCode; +import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; +import org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper; +import org.apache.rocketmq.remoting.protocol.body.Connection; +import org.apache.rocketmq.remoting.protocol.body.ConsumeStatus; +import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.remoting.protocol.body.ProcessQueueInfo; +import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan; +import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class AdminModelConverterTest { + + private static final String TOPIC = "topicA"; + private static final String GROUP = "groupA"; + private static final String BROKER_A = "broker-a"; + private static final String BROKER_B = "broker-b"; + private static final String ADDR_A = "127.0.0.1:10911"; + private static final String ADDR_B = "127.0.0.2:10911"; + + // ------------------------------------------------------------------ helpers + + private static OffsetWrapper offsetWrapper(long brokerOffset, long consumerOffset, long pullOffset, + long lastTimestamp) { + OffsetWrapper wrapper = new OffsetWrapper(); + wrapper.setBrokerOffset(brokerOffset); + wrapper.setConsumerOffset(consumerOffset); + wrapper.setPullOffset(pullOffset); + wrapper.setLastTimestamp(lastTimestamp); + return wrapper; + } + + private static ConsumeStats consumeStats(String topic, String brokerName, int queueId, OffsetWrapper wrapper) { + ConsumeStats stats = new ConsumeStats(); + stats.getOffsetTable().put(new MessageQueue(topic, brokerName, queueId), wrapper); + return stats; + } + + private static Map statsByBroker(ConsumeStats... statsList) { + Map map = new LinkedHashMap<>(); + for (int i = 0; i < statsList.length; i++) { + map.put(i == 0 ? ADDR_A : ADDR_B, statsList[i]); + } + return map; + } + + private static QueueTimeSpan queueTimeSpan(String topic, String brokerName, int queueId, long min, long max, + long consume, long delay) { + QueueTimeSpan span = new QueueTimeSpan(); + span.setMessageQueue(new MessageQueue(topic, brokerName, queueId)); + span.setMinTimeStamp(min); + span.setMaxTimeStamp(max); + span.setConsumeTimeStamp(consume); + span.setDelayTime(delay); + return span; + } + + // ------------------------------------------------------------------ toAccumulation + + @Test + public void toAccumulationSplitsInflightFromReadyTest() { + // brokerOffset=100 consumerOffset=60 pullOffset=75: + // accumulation = 100-60 = 40, inflight = 75-60 = 15, ready = 40-15 = 25 + ConsumeStats stats = consumeStats(TOPIC, BROKER_A, 0, offsetWrapper(100L, 60L, 75L, 0L)); + + AdminModelConverter.AccumulationResult result = + AdminModelConverter.toAccumulation(statsByBroker(stats), GROUP); + + assertEquals(40L, result.total.getAccumulation()); + assertEquals(15L, result.total.getInflightMessages()); + assertEquals(25L, result.total.getReadyMessages()); + GroupAccumulation byTopic = result.byTopic.get(TOPIC); + assertNotNull(byTopic); + assertEquals(40L, byTopic.getAccumulation()); + assertEquals(15L, byTopic.getInflightMessages()); + assertEquals(25L, byTopic.getReadyMessages()); + } + + @Test + public void toAccumulationClampsNegativeDiffsToZeroTest() { + // consumer ahead of the broker (offset moved back, stats skew): never report negative lag + ConsumeStats stats = consumeStats(TOPIC, BROKER_A, 0, offsetWrapper(50L, 60L, 40L, 0L)); + + AdminModelConverter.AccumulationResult result = + AdminModelConverter.toAccumulation(statsByBroker(stats), GROUP); + + assertEquals(0L, result.total.getAccumulation()); + assertEquals(0L, result.total.getInflightMessages()); + assertEquals(0L, result.total.getReadyMessages()); + assertEquals(0L, result.byTopic.get(TOPIC).getAccumulation()); + } + + @Test + public void toAccumulationCapsInflightAtAccumulationTest() { + // pullOffset beyond brokerOffset: inflight must not exceed accumulation, ready stays >= 0 + ConsumeStats stats = consumeStats(TOPIC, BROKER_A, 0, offsetWrapper(100L, 90L, 120L, 0L)); + + AdminModelConverter.AccumulationResult result = + AdminModelConverter.toAccumulation(statsByBroker(stats), GROUP); + + assertEquals(10L, result.total.getAccumulation()); + assertEquals(10L, result.total.getInflightMessages()); + assertEquals(0L, result.total.getReadyMessages()); + } + + @Test + public void toAccumulationFoldsPopRetryTopicIntoNormalTopicTest() { + ConsumeStats normal = consumeStats(TOPIC, BROKER_A, 0, offsetWrapper(100L, 60L, 60L, 0L)); + ConsumeStats popRetryV1 = consumeStats(KeyBuilder.buildPopRetryTopicV1(TOPIC, GROUP), BROKER_A, 0, + offsetWrapper(10L, 5L, 5L, 0L)); + ConsumeStats popRetryV2 = consumeStats(KeyBuilder.buildPopRetryTopicV2(TOPIC, GROUP), BROKER_B, 0, + offsetWrapper(20L, 5L, 5L, 0L)); + Map byBroker = new LinkedHashMap<>(); + byBroker.put(ADDR_A, merge(normal, popRetryV1)); + byBroker.put(ADDR_B, popRetryV2); + + AdminModelConverter.AccumulationResult result = AdminModelConverter.toAccumulation(byBroker, GROUP); + + // 40 (normal) + 5 (pop retry v1) + 15 (pop retry v2) + assertEquals(60L, result.total.getAccumulation()); + assertEquals(1, result.byTopic.size()); + assertFalse(result.byTopic.containsKey(KeyBuilder.buildPopRetryTopicV1(TOPIC, GROUP))); + assertFalse(result.byTopic.containsKey(KeyBuilder.buildPopRetryTopicV2(TOPIC, GROUP))); + assertEquals(60L, result.byTopic.get(TOPIC).getAccumulation()); + } + + @Test + public void toAccumulationCountsPullRetryInTotalButNotInByTopicTest() { + String pullRetryTopic = MixAll.getRetryTopic(GROUP); + ConsumeStats pullRetry = consumeStats(pullRetryTopic, BROKER_A, 0, offsetWrapper(30L, 10L, 10L, 0L)); + ConsumeStats normal = consumeStats(TOPIC, BROKER_A, 1, offsetWrapper(100L, 60L, 60L, 0L)); + + AdminModelConverter.AccumulationResult result = + AdminModelConverter.toAccumulation(statsByBroker(merge(pullRetry, normal)), GROUP); + + // the pull retry backlog is real lag of the group, so it belongs to the total ... + assertEquals(60L, result.total.getAccumulation()); + // ... but %RETRY% is not a user-visible topic, so it must not show up per topic + assertEquals(1, result.byTopic.size()); + assertFalse(result.byTopic.containsKey(pullRetryTopic)); + assertEquals(40L, result.byTopic.get(TOPIC).getAccumulation()); + } + + @Test + public void toAccumulationSetsLastConsumeTimestampAndDelayTest() { + long lastTimestamp = System.currentTimeMillis() - 5000L; + ConsumeStats stats = consumeStats(TOPIC, BROKER_A, 0, offsetWrapper(100L, 60L, 60L, lastTimestamp)); + + AdminModelConverter.AccumulationResult result = + AdminModelConverter.toAccumulation(statsByBroker(stats), GROUP); + + assertEquals(lastTimestamp, result.total.getLastConsumeTimestamp()); + assertTrue(result.total.hasDeliverDelayTime()); + assertTrue(result.total.getDeliverDelayTime().getSeconds() >= 4L); + } + + @Test + public void toAccumulationIgnoresTimestampWhenCaughtUpTest() { + ConsumeStats stats = consumeStats(TOPIC, BROKER_A, 0, + offsetWrapper(60L, 60L, 60L, System.currentTimeMillis())); + + AdminModelConverter.AccumulationResult result = + AdminModelConverter.toAccumulation(statsByBroker(stats), GROUP); + + assertEquals(0L, result.total.getAccumulation()); + assertEquals(0L, result.total.getLastConsumeTimestamp()); + assertFalse(result.total.hasDeliverDelayTime()); + } + + @Test + public void toAccumulationEmptyInputIsZeroTest() { + AdminModelConverter.AccumulationResult result = + AdminModelConverter.toAccumulation(Collections.emptyMap(), GROUP); + + assertEquals(0L, result.total.getAccumulation()); + assertEquals(0L, result.total.getInflightMessages()); + assertEquals(0L, result.total.getReadyMessages()); + assertTrue(result.byTopic.isEmpty()); + } + + private static ConsumeStats merge(ConsumeStats first, ConsumeStats second) { + ConsumeStats merged = new ConsumeStats(); + merged.getOffsetTable().putAll(first.getOffsetTable()); + merged.getOffsetTable().putAll(second.getOffsetTable()); + return merged; + } + + // ------------------------------------------------------------------ toQueryTimeSpan + + @Test + public void toQueryTimeSpanMapsAllFieldsTest() { + List spans = Collections.singletonList( + queueTimeSpan(TOPIC, BROKER_A, 3, 100L, 200L, 150L, 42L)); + + QueryTimeSpanResponse response = AdminModelConverter.toQueryTimeSpan(spans); + + assertEquals(Code.OK, response.getStatus().getCode()); + assertEquals(1, response.getQueueTimeSpanListCount()); + QueryTimeSpanResponse.QueueTimeSpan span = response.getQueueTimeSpanList(0); + assertEquals(TOPIC, span.getMessageQueue().getTopic().getName()); + assertEquals(BROKER_A, span.getMessageQueue().getBroker().getName()); + assertEquals(3, span.getMessageQueue().getId()); + assertEquals(100L, span.getMinTimestamp()); + assertEquals(200L, span.getMaxTimestamp()); + assertEquals(150L, span.getConsumeTimestamp()); + assertEquals(42L, span.getDelayTimeMs()); + } + + @Test + public void toQueryTimeSpanClampsNegativeDelayToZeroTest() { + List spans = Collections.singletonList( + queueTimeSpan(TOPIC, BROKER_A, 0, 100L, 200L, 150L, -5L)); + + QueryTimeSpanResponse response = AdminModelConverter.toQueryTimeSpan(spans); + + assertEquals(0L, response.getQueueTimeSpanList(0).getDelayTimeMs()); + } + + @Test + public void toQueryTimeSpanSkipsNullAndHandlesNullListTest() { + // a null list is tolerated and yields an OK, empty response + QueryTimeSpanResponse nullResponse = AdminModelConverter.toQueryTimeSpan(null); + assertEquals(Code.OK, nullResponse.getStatus().getCode()); + assertEquals(0, nullResponse.getQueueTimeSpanListCount()); + + // null / queue-less entries are skipped, only the valid span is mapped + List spans = new ArrayList<>(); + spans.add(null); + spans.add(new QueueTimeSpan()); + spans.add(queueTimeSpan(TOPIC, BROKER_A, 0, 1L, 2L, 3L, 4L)); + QueryTimeSpanResponse response = AdminModelConverter.toQueryTimeSpan(spans); + assertEquals(Code.OK, response.getStatus().getCode()); + assertEquals(1, response.getQueueTimeSpanListCount()); + } + + // ------------------------------------------------------------------ toMessageType / toTopicStatus + + @Test + public void toMessageTypeMapsKnownAndDegradesUnknownTest() { + assertEquals(MessageType.NORMAL, AdminModelConverter.toMessageType(TopicMessageType.NORMAL)); + assertEquals(MessageType.FIFO, AdminModelConverter.toMessageType(TopicMessageType.FIFO)); + assertEquals(MessageType.DELAY, AdminModelConverter.toMessageType(TopicMessageType.DELAY)); + assertEquals(MessageType.TRANSACTION, AdminModelConverter.toMessageType(TopicMessageType.TRANSACTION)); + assertEquals(MessageType.LITE, AdminModelConverter.toMessageType(TopicMessageType.LITE)); + // MessageType.UNRECOGNIZED cannot be set on a builder, so unknowns must degrade, not blow up + for (TopicMessageType type : Arrays.asList(TopicMessageType.MIXED, TopicMessageType.PRIORITY, + TopicMessageType.UNSPECIFIED, null)) { + MessageType mapped = AdminModelConverter.toMessageType(type); + assertEquals(MessageType.MESSAGE_TYPE_UNSPECIFIED, mapped); + assertNotEquals(MessageType.UNRECOGNIZED, mapped); + } + } + + @Test + public void toTopicStatusReadsMessageTypeFromAttributesTest() { + TopicConfig config = new TopicConfig(); + config.setTopicName(TOPIC); + config.setReadQueueNums(8); + config.setWriteQueueNums(16); + config.setPerm(6); + config.setTopicMessageType(TopicMessageType.FIFO); + + DescribeTopicStatusResponse response = AdminModelConverter.toTopicStatus(config, TOPIC); + + assertEquals(Code.OK, response.getStatus().getCode()); + assertEquals(MessageType.FIFO, response.getTopicMessageType()); + assertTrue(response.getDescription().contains("readQueueNums=8")); + assertTrue(response.getDescription().contains("writeQueueNums=16")); + assertTrue(response.getDescription().contains("perm=6")); + } + + @Test + public void toTopicStatusFallsBackWhenConfigThinOrMissingTest() { + // an empty config (no message-type attribute) defaults to NORMAL + DescribeTopicStatusResponse normal = AdminModelConverter.toTopicStatus(new TopicConfig(), TOPIC); + assertEquals(MessageType.NORMAL, normal.getTopicMessageType()); + + // a null config stays honest: UNSPECIFIED type + an explanatory description + DescribeTopicStatusResponse missing = AdminModelConverter.toTopicStatus(null, TOPIC); + assertEquals(Code.OK, missing.getStatus().getCode()); + assertEquals(MessageType.MESSAGE_TYPE_UNSPECIFIED, missing.getTopicMessageType()); + assertTrue(missing.getDescription().contains("topic config not available for " + TOPIC)); + } + + // ------------------------------------------------------------------ toTopicRoute / toMessageQueue + + @Test + public void toTopicRouteSerializesRouteDataOrEmptyJsonTest() { + TopicRouteData routeData = new TopicRouteData(); + routeData.setOrderTopicConf("orderConf"); + GetTopicRouteResponse response = AdminModelConverter.toTopicRoute(routeData); + assertEquals(Code.OK, response.getStatus().getCode()); + assertEquals(JSON.toJSONString(routeData), response.getTopicRouteData()); + + // a null route serializes to empty JSON rather than the literal "null" + GetTopicRouteResponse nullResponse = AdminModelConverter.toTopicRoute(null); + assertEquals(Code.OK, nullResponse.getStatus().getCode()); + assertEquals("{}", nullResponse.getTopicRouteData()); + } + + @Test + public void toMessageQueueSetsBrokerAndIdTest() { + apache.rocketmq.v2.MessageQueue v2 = + AdminModelConverter.toMessageQueue(new MessageQueue(TOPIC, BROKER_A, 3)); + + assertEquals(TOPIC, v2.getTopic().getName()); + assertEquals(BROKER_A, v2.getBroker().getName()); + assertEquals(3, v2.getId()); + } + + // ------------------------------------------------------------------ clients + + @Test + public void toClientInfoParsesHostnameAndPlainAddrTest() { + // "hostname@ip:port" splits into hostname + egress ip; message model carried through + Connection withHost = new Connection(); + withHost.setClientId("CID-1"); + withHost.setClientAddr("consumer-host@192.168.1.5:43210"); + withHost.setLanguage(LanguageCode.JAVA); + withHost.setVersion(355); + apache.rocketmq.v2.ClientInfo hostInfo = AdminModelConverter.toClientInfo(withHost, + apache.rocketmq.v2.MessageModel.CLUSTERING); + assertEquals("CID-1", hostInfo.getClientId()); + assertEquals("JAVA", hostInfo.getLanguage()); + assertEquals("355", hostInfo.getVersion()); + assertEquals("192.168.1.5", hostInfo.getEgressIp()); + assertEquals("consumer-host", hostInfo.getHostname()); + assertEquals(apache.rocketmq.v2.MessageModel.CLUSTERING, hostInfo.getMessageModel()); + + // a plain "ip:port" yields no hostname; null message model degrades to UNSPECIFIED + Connection plain = new Connection(); + plain.setClientId("CID-2"); + plain.setClientAddr("192.168.1.5:43210"); + apache.rocketmq.v2.ClientInfo plainInfo = AdminModelConverter.toClientInfo(plain, null); + assertEquals("192.168.1.5", plainInfo.getEgressIp()); + assertEquals("", plainInfo.getHostname()); + assertEquals(apache.rocketmq.v2.MessageModel.MESSAGE_MODEL_UNSPECIFIED, plainInfo.getMessageModel()); + } + + @Test + public void toMessageModelMapsValuesTest() { + assertEquals(apache.rocketmq.v2.MessageModel.BROADCASTING, + AdminModelConverter.toMessageModel(MessageModel.BROADCASTING)); + assertEquals(apache.rocketmq.v2.MessageModel.CLUSTERING, + AdminModelConverter.toMessageModel(MessageModel.CLUSTERING)); + assertEquals(apache.rocketmq.v2.MessageModel.MESSAGE_MODEL_UNSPECIFIED, + AdminModelConverter.toMessageModel(null)); + } + + @Test + public void toFilterExpressionMapsSqlAndTagTest() { + apache.rocketmq.v2.FilterExpression sql = + AdminModelConverter.toFilterExpression(ExpressionType.SQL92, "a > 1"); + assertEquals(FilterType.SQL, sql.getType()); + assertEquals("a > 1", sql.getExpression()); + + apache.rocketmq.v2.FilterExpression tag = AdminModelConverter.toFilterExpression(ExpressionType.TAG, "tagA"); + assertEquals(FilterType.TAG, tag.getType()); + assertEquals("tagA", tag.getExpression()); + + apache.rocketmq.v2.FilterExpression nullExpression = AdminModelConverter.toFilterExpression(null, null); + assertEquals(FilterType.TAG, nullExpression.getType()); + assertEquals("", nullExpression.getExpression()); + } + + @Test + public void toConsumerRunningInfoMapsAllSectionsTest() { + ConsumerRunningInfo runningInfo = new ConsumerRunningInfo(); + Properties properties = runningInfo.getProperties(); + properties.setProperty(ConsumerRunningInfo.PROP_CLIENT_VERSION, "V5_0_0"); + properties.setProperty(ConsumerRunningInfo.PROP_CONSUME_ORDERLY, "false"); + + SubscriptionData tagSub = new SubscriptionData(); + tagSub.setTopic(TOPIC); + tagSub.setSubString("tagA"); + tagSub.setExpressionType(ExpressionType.TAG); + SubscriptionData sqlSub = new SubscriptionData(); + sqlSub.setTopic("topicB"); + sqlSub.setSubString("a > 1"); + sqlSub.setExpressionType(ExpressionType.SQL92); + runningInfo.getSubscriptionSet().add(tagSub); + runningInfo.getSubscriptionSet().add(sqlSub); + + ProcessQueueInfo processQueueInfo = new ProcessQueueInfo(); + processQueueInfo.setCommitOffset(10L); + processQueueInfo.setCachedMsgMinOffset(5L); + processQueueInfo.setCachedMsgMaxOffset(15L); + processQueueInfo.setCachedMsgCount(7); + processQueueInfo.setCachedMsgSizeInMiB(2); + processQueueInfo.setTransactionMsgMinOffset(1L); + processQueueInfo.setTransactionMsgMaxOffset(3L); + processQueueInfo.setTransactionMsgCount(4); + processQueueInfo.setLocked(true); + processQueueInfo.setTryUnlockTimes(9L); + processQueueInfo.setLastLockTimestamp(111L); + processQueueInfo.setDroped(false); + processQueueInfo.setLastPullTimestamp(222L); + processQueueInfo.setLastConsumeTimestamp(333L); + runningInfo.getMqTable().put(new MessageQueue(TOPIC, BROKER_A, 0), processQueueInfo); + + ConsumeStatus consumeStatus = new ConsumeStatus(); + consumeStatus.setPullRT(1.5D); + consumeStatus.setPullTPS(2.5D); + consumeStatus.setConsumeRT(3.5D); + consumeStatus.setConsumeOKTPS(4.5D); + consumeStatus.setConsumeFailedTPS(5.5D); + consumeStatus.setConsumeFailedMsgs(6L); + runningInfo.getStatusTable().put(TOPIC, consumeStatus); + + apache.rocketmq.v2.ConsumerRunningInfo v2 = AdminModelConverter.toConsumerRunningInfo(runningInfo); + + // 1. properties + assertEquals("V5_0_0", v2.getPropertiesMap().get(ConsumerRunningInfo.PROP_CLIENT_VERSION)); + assertEquals("false", v2.getPropertiesMap().get(ConsumerRunningInfo.PROP_CONSUME_ORDERLY)); + // 2. subscriptions + assertEquals(2, v2.getSubscriptionsCount()); + assertEquals(FilterType.TAG, v2.getSubscriptionsMap().get(TOPIC).getType()); + assertEquals("tagA", v2.getSubscriptionsMap().get(TOPIC).getExpression()); + assertEquals(FilterType.SQL, v2.getSubscriptionsMap().get("topicB").getType()); + assertEquals("a > 1", v2.getSubscriptionsMap().get("topicB").getExpression()); + // 3. message queue table + assertEquals(1, v2.getMessageQueueTableCount()); + MessageQueueItem item = v2.getMessageQueueTable(0); + assertEquals(TOPIC, item.getMessageQueue().getTopic().getName()); + assertEquals(BROKER_A, item.getMessageQueue().getBroker().getName()); + assertEquals(0, item.getMessageQueue().getId()); + apache.rocketmq.v2.ProcessQueueInfo pqi = item.getProcessQueueInfo(); + assertEquals(10L, pqi.getCommitOffset()); + assertEquals(5L, pqi.getCachedMsgMinOffset()); + assertEquals(15L, pqi.getCachedMsgMaxOffset()); + assertEquals(7, pqi.getCachedMsgCount()); + assertEquals(2, pqi.getCachedMsgSizeInMib()); + assertEquals(1L, pqi.getTransactionMsgMinOffset()); + assertEquals(3L, pqi.getTransactionMsgMaxOffset()); + assertEquals(4, pqi.getTransactionMsgCount()); + assertTrue(pqi.getLocked()); + assertEquals(9L, pqi.getTryUnlockTimes()); + assertEquals(111L, pqi.getLastLockTimestamp()); + assertFalse(pqi.getDropped()); + assertEquals(222L, pqi.getLastPullTimestamp()); + assertEquals(333L, pqi.getLastConsumeTimestamp()); + // 4. consume status table + assertEquals(1, v2.getConsumeStatusTableCount()); + apache.rocketmq.v2.ConsumeStatus status = v2.getConsumeStatusTableMap().get(TOPIC); + assertNotNull(status); + assertEquals(1.5D, status.getReceiveRt(), 0.0001D); + assertEquals(2.5D, status.getReceiveTps(), 0.0001D); + assertEquals(3.5D, status.getConsumeRt(), 0.0001D); + assertEquals(4.5D, status.getConsumeOkTps(), 0.0001D); + assertEquals(5.5D, status.getConsumeFailedTps(), 0.0001D); + assertEquals(6L, status.getConsumeFailedMsgs()); + } + + @Test + public void toConsumerRunningInfoNullReturnsEmptyTest() { + apache.rocketmq.v2.ConsumerRunningInfo v2 = AdminModelConverter.toConsumerRunningInfo(null); + + assertEquals(0, v2.getPropertiesCount()); + assertEquals(0, v2.getSubscriptionsCount()); + assertEquals(0, v2.getMessageQueueTableCount()); + assertEquals(0, v2.getConsumeStatusTableCount()); + } + + // ------------------------------------------------------------------ misc + + @Test + public void toAccumulationSkipsNullStatsEntriesTest() { + Map byBroker = new HashMap<>(); + byBroker.put(ADDR_A, null); + ConsumeStats withNulls = new ConsumeStats(); + // the default offsetTable is a ConcurrentHashMap which forbids null values, so install a + // plain map to exercise the converter's null guard (deserialized stats can contain one) + Map offsetTable = new HashMap<>(); + offsetTable.put(new MessageQueue(TOPIC, BROKER_A, 0), null); + offsetTable.put(new MessageQueue(TOPIC, BROKER_A, 1), offsetWrapper(10L, 4L, 4L, 0L)); + withNulls.setOffsetTable(offsetTable); + byBroker.put(ADDR_B, withNulls); + + AdminModelConverter.AccumulationResult result = AdminModelConverter.toAccumulation(byBroker, GROUP); + + assertEquals(6L, result.total.getAccumulation()); + assertEquals(6L, result.byTopic.get(TOPIC).getAccumulation()); + } +} diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminAuthInterceptorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminAuthInterceptorTest.java new file mode 100644 index 00000000000..c44038a7272 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminAuthInterceptorTest.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.admin; + +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.Status; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import org.apache.rocketmq.auth.config.AuthConfig; +import org.apache.rocketmq.common.action.Action; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.InitConfigTest; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ProxyAdminAuthInterceptorTest extends InitConfigTest { + + @Mock + private MessagingProcessor messagingProcessor; + + private static final MethodDescriptor.Marshaller BYTE_MARSHALLER = + new MethodDescriptor.Marshaller() { + @Override + public InputStream stream(byte[] value) { + return new ByteArrayInputStream(value == null ? new byte[0] : value); + } + + @Override + public byte[] parse(InputStream stream) { + return new byte[0]; + } + }; + + private static MethodDescriptor method(String name) { + return MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.UNARY) + .setFullMethodName("apache.rocketmq.v2.Admin/" + name) + .setRequestMarshaller(BYTE_MARSHALLER) + .setResponseMarshaller(BYTE_MARSHALLER) + .build(); + } + + @SuppressWarnings("unchecked") + private static ServerCall serverCall(String methodName) { + ServerCall call = mock(ServerCall.class); + when(call.getMethodDescriptor()).thenReturn(method(methodName)); + return call; + } + + // --------------------------------------------------------------------- + // per-method resource / action mapping (read-only vs high-privilege isolation) + // --------------------------------------------------------------------- + + @Test + public void everyAdminMethodMapsToExpectedResourceAndAction() { + // the full per-method ACL table: resource module + action. Read-only RPCs use GET/LIST, + // high-privilege mutations use UPDATE/DELETE/PUB, and each is scoped to its own resource. + Map expected = new LinkedHashMap<>(); + expected.put("GetProxyRuntimeStats", ra(ProxyAdminAuthInterceptor.RESOURCE_OPS, Action.GET)); + expected.put("GetTopicRoute", ra(ProxyAdminAuthInterceptor.RESOURCE_ROUTE, Action.GET)); + expected.put("DescribeTopicStatus", ra(ProxyAdminAuthInterceptor.RESOURCE_OPS, Action.GET)); + expected.put("ListSubscription", ra(ProxyAdminAuthInterceptor.RESOURCE_CLIENT, Action.LIST)); + expected.put("DescribeSubscription", ra(ProxyAdminAuthInterceptor.RESOURCE_CLIENT, Action.GET)); + expected.put("ListConsumerConnection", ra(ProxyAdminAuthInterceptor.RESOURCE_CLIENT, Action.LIST)); + expected.put("DescribeGroupAccumulation", ra(ProxyAdminAuthInterceptor.RESOURCE_CLIENT, Action.GET)); + expected.put("GetConsumerRunningInfo", ra(ProxyAdminAuthInterceptor.RESOURCE_CLIENT, Action.GET)); + expected.put("QueryTimeSpan", ra(ProxyAdminAuthInterceptor.RESOURCE_CLIENT, Action.GET)); + expected.put("QueryMessage", ra(ProxyAdminAuthInterceptor.RESOURCE_OPS, Action.GET)); + expected.put("ChangeLogLevel", ra(ProxyAdminAuthInterceptor.RESOURCE_CONFIG, Action.UPDATE)); + expected.put("DeleteSubscription", ra(ProxyAdminAuthInterceptor.RESOURCE_OPS, Action.DELETE)); + expected.put("ResetGroupOffset", ra(ProxyAdminAuthInterceptor.RESOURCE_OPS, Action.UPDATE)); + expected.put("AdminSendMessage", ra(ProxyAdminAuthInterceptor.RESOURCE_OPS, Action.PUB)); + expected.put("PrintThreadStackTrace", ra(ProxyAdminAuthInterceptor.RESOURCE_CONNECTION, Action.UPDATE)); + expected.put("VerifyMessage", ra(ProxyAdminAuthInterceptor.RESOURCE_CONNECTION, Action.UPDATE)); + + Set resources = new HashSet<>(); + for (Map.Entry e : expected.entrySet()) { + ProxyAdminAuthInterceptor.ResourceAction actual = + ProxyAdminAuthInterceptor.resolveResourceAction(e.getKey()); + assertNotNull("missing mapping for " + e.getKey(), actual); + assertEquals("resource for " + e.getKey(), e.getValue().resource, actual.resource); + assertEquals("action for " + e.getKey(), e.getValue().action, actual.action); + resources.add(actual.resource); + } + // five distinct modules, all under the proxy.admin.* namespace + assertEquals(5, resources.size()); + for (String resource : resources) { + assertTrue(resource.startsWith("proxy.admin.")); + } + } + + private static ProxyAdminAuthInterceptor.ResourceAction ra(String resource, Action action) { + return new ProxyAdminAuthInterceptor.ResourceAction(resource, action); + } + + // --------------------------------------------------------------------- + // behavior modes + // --------------------------------------------------------------------- + + @Test + @SuppressWarnings("unchecked") + public void openModePassesThroughWhenClusterAuthDisabled() { + AuthConfig authConfig = new AuthConfig(); + authConfig.setAuthenticationEnabled(false); + authConfig.setAuthorizationEnabled(false); + ConfigurationManager.getProxyConfig().setGrpcAdminServerAuthEnable(false); + + ProxyAdminAuthInterceptor interceptor = new ProxyAdminAuthInterceptor(authConfig, messagingProcessor); + ServerCall call = serverCall("ListConsumerConnection"); + ServerCallHandler next = mock(ServerCallHandler.class); + + interceptor.interceptCall(call, new Metadata(), next); + verify(next).startCall(any(), any()); + verify(call, never()).close(any(), any()); + } + + @Test + @SuppressWarnings("unchecked") + public void failClosedRejectsWhenRequireAuthButClusterAuthDisabled() { + AuthConfig authConfig = new AuthConfig(); + authConfig.setAuthenticationEnabled(false); + authConfig.setAuthorizationEnabled(false); + ConfigurationManager.getProxyConfig().setGrpcAdminServerAuthEnable(true); + try { + ProxyAdminAuthInterceptor interceptor = new ProxyAdminAuthInterceptor(authConfig, messagingProcessor); + ServerCall call = serverCall("ListClients"); + ServerCallHandler next = mock(ServerCallHandler.class); + + interceptor.interceptCall(call, new Metadata(), next); + verify(next, never()).startCall(any(), any()); + org.mockito.ArgumentCaptor statusCaptor = org.mockito.ArgumentCaptor.forClass(Status.class); + verify(call).close(statusCaptor.capture(), any(Metadata.class)); + assertEquals(Status.Code.UNAUTHENTICATED, statusCaptor.getValue().getCode()); + assertTrue(statusCaptor.getValue().getDescription().contains("grpcAdminServerAuthEnable")); + } finally { + ConfigurationManager.getProxyConfig().setGrpcAdminServerAuthEnable(false); + } + } + + @Test + @SuppressWarnings("unchecked") + public void failClosedRejectsWhenRequireAuthButAuthorizationDisabled() { + // authentication on but authorization off: the ACL evaluator is globally gated off, so + // enforcement would silently pass. Fail-closed must refuse instead of serving an + // unauthorized (potentially destructive) admin RPC to any authenticated identity. + AuthConfig authConfig = new AuthConfig(); + authConfig.setAuthenticationEnabled(true); + authConfig.setAuthorizationEnabled(false); + ConfigurationManager.getProxyConfig().setGrpcAdminServerAuthEnable(true); + try { + ProxyAdminAuthInterceptor interceptor = new ProxyAdminAuthInterceptor(authConfig, messagingProcessor); + ServerCall call = serverCall("DeleteSubscription"); + ServerCallHandler next = mock(ServerCallHandler.class); + + interceptor.interceptCall(call, new Metadata(), next); + verify(next, never()).startCall(any(), any()); + org.mockito.ArgumentCaptor statusCaptor = org.mockito.ArgumentCaptor.forClass(Status.class); + verify(call).close(statusCaptor.capture(), any(Metadata.class)); + assertEquals(Status.Code.FAILED_PRECONDITION, statusCaptor.getValue().getCode()); + assertTrue(statusCaptor.getValue().getDescription().contains("authorizationEnabled")); + } finally { + ConfigurationManager.getProxyConfig().setGrpcAdminServerAuthEnable(false); + } + } + + @Test + @SuppressWarnings("unchecked") + public void failClosedRejectsAnonymousWhenRequireAuth() { + AuthConfig authConfig = new AuthConfig(); + authConfig.setAuthenticationEnabled(true); + authConfig.setAuthorizationEnabled(true); + ConfigurationManager.getProxyConfig().setGrpcAdminServerAuthEnable(true); + try { + ProxyAdminAuthInterceptor interceptor = new ProxyAdminAuthInterceptor(authConfig, messagingProcessor); + ServerCall call = serverCall("PrintThreadStackTrace"); + ServerCallHandler next = mock(ServerCallHandler.class); + + // empty metadata: no credentials at all + ServerCall.Listener listener = interceptor.interceptCall(call, new Metadata(), next); + assertNotNull(listener); + verify(next, never()).startCall(any(), any()); + org.mockito.ArgumentCaptor statusCaptor = org.mockito.ArgumentCaptor.forClass(Status.class); + verify(call).close(statusCaptor.capture(), any(Metadata.class)); + assertEquals(Status.Code.UNAUTHENTICATED, statusCaptor.getValue().getCode()); + } finally { + ConfigurationManager.getProxyConfig().setGrpcAdminServerAuthEnable(false); + } + } + + @Test + @SuppressWarnings("unchecked") + public void authenticationFailureClosesUnauthenticated() { + // cluster authentication on + no credentials -> evaluator throws AuthenticationException. + // AuthenticationFactory caches evaluators per configName, so use a unique name to get + // a fresh evaluator bound to THIS config (otherwise a stale one from another test wins). + AuthConfig authConfig = new AuthConfig(); + authConfig.setConfigName("proxy-admin-auth-failure-test-" + System.nanoTime()); + authConfig.setAuthenticationEnabled(true); + authConfig.setAuthorizationEnabled(false); + ConfigurationManager.getProxyConfig().setGrpcAdminServerAuthEnable(false); + + ProxyAdminAuthInterceptor interceptor = new ProxyAdminAuthInterceptor(authConfig, messagingProcessor); + ServerCall call = serverCall("ListConsumerConnection"); + ServerCallHandler next = mock(ServerCallHandler.class); + + interceptor.interceptCall(call, new Metadata(), next); + verify(next, never()).startCall(any(), any()); + org.mockito.ArgumentCaptor statusCaptor = org.mockito.ArgumentCaptor.forClass(Status.class); + verify(call).close(statusCaptor.capture(), any(Metadata.class)); + assertEquals(Status.Code.UNAUTHENTICATED, statusCaptor.getValue().getCode()); + assertNotNull(statusCaptor.getValue().getDescription()); + } + + @Test + @SuppressWarnings("unchecked") + public void authorizationEnabledRejectsMissingCredentials() { + // authorization on without authentication -> mapped method demands credentials first + AuthConfig authConfig = new AuthConfig(); + authConfig.setConfigName("proxy-admin-authz-only-test-" + System.nanoTime()); + authConfig.setAuthenticationEnabled(false); + authConfig.setAuthorizationEnabled(true); + ConfigurationManager.getProxyConfig().setGrpcAdminServerAuthEnable(false); + + ProxyAdminAuthInterceptor interceptor = new ProxyAdminAuthInterceptor(authConfig, messagingProcessor); + ServerCall call = serverCall("GetTopicRoute"); + ServerCallHandler next = mock(ServerCallHandler.class); + + interceptor.interceptCall(call, new Metadata(), next); + verify(next, never()).startCall(any(), any()); + org.mockito.ArgumentCaptor statusCaptor = org.mockito.ArgumentCaptor.forClass(Status.class); + verify(call).close(statusCaptor.capture(), any(Metadata.class)); + assertEquals(Status.Code.UNAUTHENTICATED, statusCaptor.getValue().getCode()); + assertTrue(statusCaptor.getValue().getDescription().contains("missing credentials")); + } +} diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminGrpcServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminGrpcServiceTest.java new file mode 100644 index 00000000000..52362b2884e --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/admin/ProxyAdminGrpcServiceTest.java @@ -0,0 +1,1495 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.admin; + +import apache.rocketmq.v2.AdminSendMessageRequest; +import apache.rocketmq.v2.AdminSendMessageResponse; +import apache.rocketmq.v2.ChangeLogLevelRequest; +import apache.rocketmq.v2.ChangeLogLevelResponse; +import apache.rocketmq.v2.ClientType; +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.DeleteSubscriptionRequest; +import apache.rocketmq.v2.DeleteSubscriptionResponse; +import apache.rocketmq.v2.DescribeGroupAccumulationRequest; +import apache.rocketmq.v2.DescribeGroupAccumulationResponse; +import apache.rocketmq.v2.DescribeSubscriptionRequest; +import apache.rocketmq.v2.DescribeSubscriptionResponse; +import apache.rocketmq.v2.DescribeTopicStatusRequest; +import apache.rocketmq.v2.DescribeTopicStatusResponse; +import apache.rocketmq.v2.FilterExpression; +import apache.rocketmq.v2.FilterType; +import apache.rocketmq.v2.GetConsumerRunningInfoRequest; +import apache.rocketmq.v2.GetConsumerRunningInfoResponse; +import apache.rocketmq.v2.GetProxyRuntimeStatsRequest; +import apache.rocketmq.v2.GetProxyRuntimeStatsResponse; +import apache.rocketmq.v2.GetTopicRouteRequest; +import apache.rocketmq.v2.GetTopicRouteResponse; +import apache.rocketmq.v2.ListConsumerConnectionRequest; +import apache.rocketmq.v2.ListConsumerConnectionResponse; +import apache.rocketmq.v2.ListMessageRequest; +import apache.rocketmq.v2.ListMessageResponse; +import apache.rocketmq.v2.ListSubscriptionRequest; +import apache.rocketmq.v2.ListSubscriptionResponse; +import apache.rocketmq.v2.MessageType; +import apache.rocketmq.v2.PrintThreadStackTraceRequest; +import apache.rocketmq.v2.PrintThreadStackTraceResponse; +import apache.rocketmq.v2.QueryTimeSpanRequest; +import apache.rocketmq.v2.QueryTimeSpanResponse; +import apache.rocketmq.v2.ResetGroupOffsetRequest; +import apache.rocketmq.v2.ResetGroupOffsetResponse; +import apache.rocketmq.v2.Resource; +import apache.rocketmq.v2.Settings; +import apache.rocketmq.v2.Subscription; +import apache.rocketmq.v2.SubscriptionEntry; +import apache.rocketmq.v2.SubscriptionInfo; +import apache.rocketmq.v2.SystemProperties; +import apache.rocketmq.v2.VerifyMessageRequest; +import apache.rocketmq.v2.VerifyMessageResponse; +import com.alibaba.fastjson2.JSON; +import com.google.protobuf.ByteString; +import com.google.protobuf.Timestamp; +import io.grpc.stub.StreamObserver; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerManager; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.common.MQVersion; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.filter.ExpressionType; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.InitConfigTest; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.proxy.service.ServiceManager; +import org.apache.rocketmq.proxy.service.admin.AdminService; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayRequest; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayResult; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; +import org.apache.rocketmq.proxy.service.route.MessageQueueSelector; +import org.apache.rocketmq.proxy.service.route.MessageQueueView; +import org.apache.rocketmq.proxy.service.route.TopicRouteService; +import org.apache.rocketmq.remoting.protocol.LanguageCode; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; +import org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper; +import org.apache.rocketmq.remoting.protocol.body.CMResult; +import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; +import org.apache.rocketmq.remoting.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.remoting.protocol.body.Connection; +import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; +import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.remoting.protocol.body.GroupList; +import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan; +import org.apache.rocketmq.remoting.protocol.body.TopicList; +import org.apache.rocketmq.remoting.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader; +import org.apache.rocketmq.remoting.protocol.heartbeat.ConsumeType; +import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.protocol.subscription.SimpleSubscriptionData; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ProxyAdminGrpcServiceTest extends InitConfigTest { + + @Mock + private ServiceManager serviceManager; + @Mock + private MessagingProcessor messagingProcessor; + @Mock + private GrpcChannelManager grpcChannelManager; + @Mock + private GrpcClientSettingsManager grpcClientSettingsManager; + @Mock + private ProxyAdminForwarder forwarder; + @Mock + private AdminService adminService; + @Mock + private TopicRouteService topicRouteService; + @Mock + private ConsumerManager consumerManager; + @Mock + private GrpcClientChannel channel; + + private ProxyAdminGrpcService service; + + private static final String TOPIC = "topicA"; + private static final String GROUP = "groupA"; + private static final String CLIENT_ID = "client-1"; + private static final String ADDRESS_A = "127.0.0.1:10911"; + private static final String ADDRESS_B = "127.0.0.2:10911"; + + @Before + public void setUp() { + when(serviceManager.getAdminService()).thenReturn(adminService); + when(serviceManager.getTopicRouteService()).thenReturn(topicRouteService); + service = new ProxyAdminGrpcService(serviceManager, messagingProcessor, grpcChannelManager, + grpcClientSettingsManager, forwarder); + } + + @After + public void tearDown() { + service.shutdown(); + } + + // ------------------------------------------------------------------ helpers + + private static class SimpleObserver implements StreamObserver { + T value; + Throwable error; + boolean completed; + + @Override + public void onNext(T value) { + this.value = value; + } + + @Override + public void onError(Throwable t) { + this.error = t; + } + + @Override + public void onCompleted() { + this.completed = true; + } + } + + private static CompletableFuture failedFuture(Throwable t) { + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(t); + return future; + } + + private void stubClusterInfo(String... addrs) { + Map table = new HashMap<>(); + for (int i = 0; i < addrs.length; i++) { + HashMap brokerAddrs = new HashMap<>(); + brokerAddrs.put(MixAll.MASTER_ID, addrs[i]); + table.put("broker-" + i, new BrokerData("DefaultCluster", "broker-" + i, brokerAddrs)); + } + ClusterInfo clusterInfo = new ClusterInfo(); + clusterInfo.setBrokerAddrTable(table); + when(adminService.getBrokerClusterInfo(anyLong())) + .thenReturn(CompletableFuture.completedFuture(clusterInfo)); + } + + private void stubRoute(boolean forWrite, String... brokerAddrs) throws Exception { + List queues = new ArrayList<>(); + for (int i = 0; i < brokerAddrs.length; i++) { + queues.add(new AddressableMessageQueue(new MessageQueue(TOPIC, "broker-" + i, i), brokerAddrs[i])); + } + MessageQueueSelector selector = mock(MessageQueueSelector.class); + when(selector.getQueues()).thenReturn(queues); + MessageQueueView view = mock(MessageQueueView.class); + if (forWrite) { + when(view.getWriteSelector()).thenReturn(selector); + } else { + when(view.getReadSelector()).thenReturn(selector); + } + when(topicRouteService.getAllMessageQueueView(any(), anyString())).thenReturn(view); + } + + private SubscriptionData subscriptionData(String topic, String expressionType, String expression) { + SubscriptionData data = new SubscriptionData(); + data.setTopic(topic); + data.setExpressionType(expressionType); + data.setSubString(expression); + return data; + } + + private ConsumerConnection consumerConnection(String clientId, String clientAddr, String... topics) { + ConsumerConnection connection = new ConsumerConnection(); + Connection conn = new Connection(); + conn.setClientId(clientId); + conn.setClientAddr(clientAddr); + conn.setLanguage(LanguageCode.JAVA); + conn.setVersion(355); + connection.getConnectionSet().add(conn); + for (String topic : topics) { + connection.getSubscriptionTable().put(topic, subscriptionData(topic, ExpressionType.TAG, "tagA")); + } + connection.setMessageModel(MessageModel.CLUSTERING); + connection.setConsumeType(ConsumeType.CONSUME_PASSIVELY); + return connection; + } + + private Settings settings(ClientType clientType, String group, String topic, String expression) { + return Settings.newBuilder() + .setClientType(clientType) + .setSubscription(Subscription.newBuilder() + .setGroup(Resource.newBuilder().setName(group).build()) + .addSubscriptions(SubscriptionEntry.newBuilder() + .setTopic(Resource.newBuilder().setName(topic).build()) + .setExpression(FilterExpression.newBuilder() + .setType(FilterType.TAG).setExpression(expression).build()) + .build()) + .build()) + .build(); + } + + private MessageExt messageExt(String msgId) { + MessageExt ext = new MessageExt(); + ext.setTopic(TOPIC); + ext.setMsgId(msgId); + ext.setBody("payload".getBytes(StandardCharsets.UTF_8)); + ext.setQueueId(0); + ext.setQueueOffset(100L); + ext.setBrokerName("broker-0"); + long now = System.currentTimeMillis(); + ext.setBornTimestamp(now); + ext.setStoreTimestamp(now); + ext.setBornHost(new InetSocketAddress("127.0.0.1", 10909)); + ext.setStoreHost(new InetSocketAddress("127.0.0.1", 10911)); + return ext; + } + + private ConsumeStats consumeStats(String topic, long brokerOffset, long consumerOffset, long pullOffset) { + ConsumeStats stats = new ConsumeStats(); + OffsetWrapper wrapper = new OffsetWrapper(); + wrapper.setBrokerOffset(brokerOffset); + wrapper.setConsumerOffset(consumerOffset); + wrapper.setPullOffset(pullOffset); + stats.getOffsetTable().put(new MessageQueue(topic, "broker-0", 0), wrapper); + return stats; + } + + private TopicConfig topicConfig(TopicMessageType type, int readQueueNums, int writeQueueNums) { + TopicConfig config = new TopicConfig(); + config.setTopicName(TOPIC); + config.setReadQueueNums(readQueueNums); + config.setWriteQueueNums(writeQueueNums); + config.setPerm(6); + if (type != null) { + config.setTopicMessageType(type); + } + return config; + } + + /** Registers the client as a local channel reachable through the consumer manager. */ + private void stubLocalClient() { + when(serviceManager.getConsumerManager()).thenReturn(consumerManager); + when(consumerManager.findChannel(GROUP, CLIENT_ID)) + .thenReturn(new ClientChannelInfo(channel, CLIENT_ID, LanguageCode.JAVA, 0)); + when(channel.isActive()).thenReturn(true); + } + + /** + * Simulates the relay round trip: when the service writes a {@link ProxyRelayRequest} into the + * client channel, the captured caller future is completed with the given result, exactly like + * {@code ClusterProxyRelayService} + the channel implementation would do in production. + */ + private void answerRelayWith(int expectedCode, ProxyRelayResult result) { + when(channel.writeAndFlush(any())).thenAnswer(invocation -> { + ProxyRelayRequest request = invocation.getArgument(0); + assertEquals(expectedCode, request.getCode()); + CompletableFuture> future = request.typedResponseFuture(); + future.complete(result); + return null; + }); + } + + private static Resource resource(String name) { + return Resource.newBuilder().setName(name).build(); + } + + // ------------------------------------------------------------------ 1. ChangeLogLevel + + @Test + public void changeLogLevelChangesRootLevelTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.changeLogLevel(ChangeLogLevelRequest.newBuilder() + .setLevel(ChangeLogLevelRequest.Level.DEBUG).build(), observer); + + assertNotNull(observer.value); + assertTrue(observer.value.getRemark(), observer.value.getRemark().contains("log level changed")); + assertTrue(observer.completed); + } + + // ------------------------------------------------------------------ 2. DescribeTopicStatus + + @Test + public void describeTopicStatusReadsMessageTypeFromAttributesTest() throws Exception { + stubRoute(false, ADDRESS_A); + when(adminService.getTopicConfig(anyString(), eq(TOPIC), anyLong())) + .thenReturn(CompletableFuture.completedFuture(topicConfig(TopicMessageType.FIFO, 4, 4))); + + SimpleObserver observer = new SimpleObserver<>(); + service.describeTopicStatus(DescribeTopicStatusRequest.newBuilder() + .setTopic(resource(TOPIC)).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(MessageType.FIFO, observer.value.getTopicMessageType()); + assertTrue(observer.value.getDescription().contains("readQueueNums=4")); + } + + @Test + public void describeTopicStatusMergesQueueNumsAcrossBrokersTest() throws Exception { + stubRoute(false, ADDRESS_A, ADDRESS_B); + when(adminService.getTopicConfig(anyString(), eq(TOPIC), anyLong())) + .thenReturn(CompletableFuture.completedFuture(topicConfig(TopicMessageType.FIFO, 4, 4))); + + SimpleObserver observer = new SimpleObserver<>(); + service.describeTopicStatus(DescribeTopicStatusRequest.newBuilder() + .setTopic(resource(TOPIC)).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(MessageType.FIFO, observer.value.getTopicMessageType()); + assertTrue(observer.value.getDescription().contains("readQueueNums=8")); + assertTrue(observer.value.getDescription().contains("writeQueueNums=8")); + } + + @Test + public void describeTopicStatusReportsMixedWhenBrokersDisagreeTest() throws Exception { + stubRoute(false, ADDRESS_A, ADDRESS_B); + when(adminService.getTopicConfig(eq(ADDRESS_A), eq(TOPIC), anyLong())) + .thenReturn(CompletableFuture.completedFuture(topicConfig(TopicMessageType.FIFO, 4, 4))); + when(adminService.getTopicConfig(eq(ADDRESS_B), eq(TOPIC), anyLong())) + .thenReturn(CompletableFuture.completedFuture(topicConfig(TopicMessageType.NORMAL, 4, 4))); + + SimpleObserver observer = new SimpleObserver<>(); + service.describeTopicStatus(DescribeTopicStatusRequest.newBuilder() + .setTopic(resource(TOPIC)).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + // MIXED degrades to UNSPECIFIED rather than pretending one broker is right + assertEquals(MessageType.MESSAGE_TYPE_UNSPECIFIED, observer.value.getTopicMessageType()); + } + + @Test + public void describeTopicStatusFailsWhenRouteMissingTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.describeTopicStatus(DescribeTopicStatusRequest.newBuilder() + .setTopic(resource("missingTopic")).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.INTERNAL_SERVER_ERROR, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("topic route not found")); + } + + // ------------------------------------------------------------------ 3/4. List & DescribeSubscription + + @Test + public void listSubscriptionRejectsEmptyFiltersTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.listSubscription(ListSubscriptionRequest.newBuilder().build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.BAD_REQUEST, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("at least one of topic or group")); + } + + @Test + public void describeSubscriptionRejectsEmptyFiltersTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.describeSubscription(DescribeSubscriptionRequest.newBuilder().build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.BAD_REQUEST, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("at least one of topic or group")); + } + + @Test + public void listSubscriptionReturnsBrokerSideSubscriptionsTest() { + stubClusterInfo(ADDRESS_A); + ConsumerConnection connection = consumerConnection(CLIENT_ID, "192.168.1.5:43210", TOPIC); + // retry topics are bookkeeping, not user-visible subscriptions + connection.getSubscriptionTable().put(MixAll.getRetryTopic(GROUP), + subscriptionData(MixAll.getRetryTopic(GROUP), ExpressionType.TAG, "*")); + connection.getSubscriptionTable().put(KeyBuilder.buildPopRetryTopicV2(TOPIC, GROUP), + subscriptionData(KeyBuilder.buildPopRetryTopicV2(TOPIC, GROUP), ExpressionType.TAG, "*")); + when(adminService.getConsumerConnectionList(eq(ADDRESS_A), eq(GROUP), anyLong())) + .thenReturn(CompletableFuture.completedFuture(connection)); + + SimpleObserver observer = new SimpleObserver<>(); + service.listSubscription(ListSubscriptionRequest.newBuilder() + .setGroup(resource(GROUP)).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(1, observer.value.getSubscriptionInfoCount()); + SubscriptionInfo info = observer.value.getSubscriptionInfo(0); + assertEquals(GROUP, info.getGroup().getName()); + assertEquals(TOPIC, info.getTopic().getName()); + assertEquals(FilterType.TAG, info.getExpression().getType()); + assertEquals("tagA", info.getExpression().getExpression()); + assertTrue(info.getOnline()); + assertEquals(apache.rocketmq.v2.MessageModel.CLUSTERING, info.getMessageModel()); + } + + @Test + public void listSubscriptionByTopicResolvesGroupsFromBrokersTest() { + stubClusterInfo(ADDRESS_A); + GroupList groupList = new GroupList(); + groupList.setGroupList(new HashSet<>(Collections.singletonList(GROUP))); + when(adminService.queryTopicConsumeByWho(eq(ADDRESS_A), eq(TOPIC), anyLong())) + .thenReturn(CompletableFuture.completedFuture(groupList)); + when(adminService.getConsumerConnectionList(eq(ADDRESS_A), eq(GROUP), anyLong())) + .thenReturn(CompletableFuture.completedFuture(consumerConnection(CLIENT_ID, "192.168.1.5:43210", TOPIC))); + + SimpleObserver observer = new SimpleObserver<>(); + service.listSubscription(ListSubscriptionRequest.newBuilder() + .setTopic(resource(TOPIC)).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(1, observer.value.getSubscriptionInfoCount()); + assertEquals(GROUP, observer.value.getSubscriptionInfo(0).getGroup().getName()); + verify(adminService).queryTopicConsumeByWho(eq(ADDRESS_A), eq(TOPIC), anyLong()); + } + + @Test + public void describeSubscriptionReportsOneEntryPerClientTest() { + stubClusterInfo(ADDRESS_A); + when(adminService.getConsumerConnectionList(eq(ADDRESS_A), eq(GROUP), anyLong())) + .thenReturn(CompletableFuture.completedFuture(consumerConnection(CLIENT_ID, "192.168.1.5:43210", TOPIC))); + + SimpleObserver observer = new SimpleObserver<>(); + service.describeSubscription(DescribeSubscriptionRequest.newBuilder() + .setGroup(resource(GROUP)).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(1, observer.value.getClientSubscriptionInfoCount()); + assertEquals(CLIENT_ID, observer.value.getClientSubscriptionInfo(0).getClientInfo().getClientId()); + SubscriptionInfo info = observer.value.getClientSubscriptionInfo(0).getSubscriptionInfo(); + assertEquals(GROUP, info.getGroup().getName()); + assertEquals(TOPIC, info.getTopic().getName()); + assertEquals("tagA", info.getExpression().getExpression()); + assertTrue(info.getOnline()); + } + + @Test + public void describeSubscriptionPrefersClientSettingsTest() { + stubClusterInfo(ADDRESS_A); + when(adminService.getConsumerConnectionList(eq(ADDRESS_A), eq(GROUP), anyLong())) + .thenReturn(CompletableFuture.completedFuture(consumerConnection(CLIENT_ID, "192.168.1.5:43210", TOPIC))); + when(grpcClientSettingsManager.getRawClientSettings(CLIENT_ID)) + .thenReturn(settings(ClientType.SIMPLE_CONSUMER, GROUP, TOPIC, "tagFromSettings")); + + SimpleObserver observer = new SimpleObserver<>(); + service.describeSubscription(DescribeSubscriptionRequest.newBuilder() + .setGroup(resource(GROUP)).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(1, observer.value.getClientSubscriptionInfoCount()); + assertEquals("tagFromSettings", + observer.value.getClientSubscriptionInfo(0).getSubscriptionInfo().getExpression().getExpression()); + } + + // ------------------------------------------------------------------ 5. DeleteSubscription + + @Test + public void deleteSubscriptionRejectsMissingTopicOrGroupTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.deleteSubscription(DeleteSubscriptionRequest.newBuilder() + .setGroup(resource(GROUP)).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.BAD_REQUEST, observer.value.getStatus().getCode()); + } + + @Test + public void deleteSubscriptionRemovesOnlyMatchingEntryTest() throws Exception { + stubRoute(true, ADDRESS_A); + SubscriptionGroupConfig config = new SubscriptionGroupConfig(); + config.setGroupName(GROUP); + Set dataSet = new HashSet<>(); + dataSet.add(new SimpleSubscriptionData(TOPIC, ExpressionType.TAG, "tagA", 0L)); + dataSet.add(new SimpleSubscriptionData(TOPIC, ExpressionType.TAG, "tagB", 0L)); + dataSet.add(new SimpleSubscriptionData("topicOther", ExpressionType.TAG, "*", 0L)); + config.setSubscriptionDataSet(dataSet); + when(adminService.getSubscriptionGroupConfig(eq(ADDRESS_A), eq(GROUP), anyLong())) + .thenReturn(CompletableFuture.completedFuture(config)); + when(adminService.updateSubscriptionGroupConfig(eq(ADDRESS_A), any(SubscriptionGroupConfig.class), anyLong())) + .thenReturn(CompletableFuture.completedFuture(null)); + + SimpleObserver observer = new SimpleObserver<>(); + service.deleteSubscription(DeleteSubscriptionRequest.newBuilder() + .setTopic(resource(TOPIC)) + .setGroup(resource(GROUP)) + .setExpression(FilterExpression.newBuilder().setType(FilterType.TAG).setExpression("tagA").build()) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + + ArgumentCaptor captor = ArgumentCaptor.forClass(SubscriptionGroupConfig.class); + verify(adminService).updateSubscriptionGroupConfig(eq(ADDRESS_A), captor.capture(), anyLong()); + Set remaining = captor.getValue().getSubscriptionDataSet(); + assertEquals(2, remaining.size()); + assertTrue(remaining.contains(new SimpleSubscriptionData(TOPIC, ExpressionType.TAG, "tagB", 0L))); + assertTrue(remaining.contains(new SimpleSubscriptionData("topicOther", ExpressionType.TAG, "*", 0L))); + assertFalse(remaining.contains(new SimpleSubscriptionData(TOPIC, ExpressionType.TAG, "tagA", 0L))); + // deleting one subscription must never delete the whole group + verify(adminService, never()).deleteSubscriptionGroup(anyString(), anyString(), anyBoolean(), anyLong()); + } + + @Test + public void deleteSubscriptionReturnsNotFoundWhenNothingMatchedTest() throws Exception { + stubRoute(true, ADDRESS_A); + SubscriptionGroupConfig config = new SubscriptionGroupConfig(); + config.setGroupName(GROUP); + config.setSubscriptionDataSet(new HashSet<>(Collections.singletonList( + new SimpleSubscriptionData("topicOther", ExpressionType.TAG, "*", 0L)))); + when(adminService.getSubscriptionGroupConfig(eq(ADDRESS_A), eq(GROUP), anyLong())) + .thenReturn(CompletableFuture.completedFuture(config)); + + SimpleObserver observer = new SimpleObserver<>(); + service.deleteSubscription(DeleteSubscriptionRequest.newBuilder() + .setTopic(resource(TOPIC)) + .setGroup(resource(GROUP)) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.NOT_FOUND, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("no subscription of group")); + verify(adminService, never()).updateSubscriptionGroupConfig(anyString(), + any(SubscriptionGroupConfig.class), anyLong()); + } + + @Test + public void deleteSubscriptionReturnsNotFoundWhenGroupUnknownTest() throws Exception { + stubRoute(true, ADDRESS_A); + when(adminService.getSubscriptionGroupConfig(eq(ADDRESS_A), eq(GROUP), anyLong())) + .thenReturn(CompletableFuture.completedFuture(null)); + + SimpleObserver observer = new SimpleObserver<>(); + service.deleteSubscription(DeleteSubscriptionRequest.newBuilder() + .setTopic(resource(TOPIC)) + .setGroup(resource(GROUP)) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.NOT_FOUND, observer.value.getStatus().getCode()); + } + + @Test + public void deleteSubscriptionPropagatesBrokerFailureTest() throws Exception { + stubRoute(true, ADDRESS_A); + when(adminService.getSubscriptionGroupConfig(eq(ADDRESS_A), eq(GROUP), anyLong())) + .thenReturn(failedFuture(new RuntimeException("broker down"))); + + SimpleObserver observer = new SimpleObserver<>(); + service.deleteSubscription(DeleteSubscriptionRequest.newBuilder() + .setTopic(resource(TOPIC)) + .setGroup(resource(GROUP)) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.INTERNAL_SERVER_ERROR, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("broker down")); + } + + // ------------------------------------------------------------------ 6. DescribeGroupAccumulation + + @Test + public void describeGroupAccumulationRejectsMissingGroupTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.describeGroupAccumulation(DescribeGroupAccumulationRequest.newBuilder().build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.BAD_REQUEST, observer.value.getStatus().getCode()); + } + + @Test + public void describeGroupAccumulationSplitsInflightReadyTest() { + stubClusterInfo(ADDRESS_A); + // the service asks the broker for the whole group (blank topic) and filters locally + when(adminService.getConsumeStats(eq(ADDRESS_A), eq(GROUP), eq(""), anyLong())) + .thenReturn(CompletableFuture.completedFuture(consumeStats(TOPIC, 100L, 60L, 75L))); + + SimpleObserver observer = new SimpleObserver<>(); + service.describeGroupAccumulation(DescribeGroupAccumulationRequest.newBuilder() + .setGroup(resource(GROUP)) + .addTopics(resource(TOPIC)) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(40L, observer.value.getAccumulation().getAccumulation()); + assertEquals(15L, observer.value.getAccumulation().getInflightMessages()); + assertEquals(25L, observer.value.getAccumulation().getReadyMessages()); + assertTrue(observer.value.getTopicAccumulationMap().containsKey(TOPIC)); + assertEquals(40L, observer.value.getTopicAccumulationMap().get(TOPIC).getAccumulation()); + } + + @Test + public void describeGroupAccumulationWholeGroupResolvesTopicsTest() { + stubClusterInfo(ADDRESS_A); + TopicList topicList = new TopicList(); + topicList.setTopicList(new HashSet<>(Collections.singletonList(TOPIC))); + when(adminService.queryTopicsByConsumer(eq(ADDRESS_A), eq(GROUP), anyLong())) + .thenReturn(CompletableFuture.completedFuture(topicList)); + when(adminService.getConsumeStats(eq(ADDRESS_A), eq(GROUP), eq(""), anyLong())) + .thenReturn(CompletableFuture.completedFuture(consumeStats(TOPIC, 100L, 60L, 60L))); + + SimpleObserver observer = new SimpleObserver<>(); + service.describeGroupAccumulation(DescribeGroupAccumulationRequest.newBuilder() + .setGroup(resource(GROUP)) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(40L, observer.value.getAccumulation().getAccumulation()); + verify(adminService).queryTopicsByConsumer(eq(ADDRESS_A), eq(GROUP), anyLong()); + } + + // ------------------------------------------------------------------ 7. ListConsumerConnection + + @Test + public void listConsumerConnectionRejectsMissingGroupTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.listConsumerConnection(ListConsumerConnectionRequest.newBuilder().build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.BAD_REQUEST, observer.value.getStatus().getCode()); + } + + @Test + public void listConsumerConnectionReturnsBrokerSideClientsTest() { + stubClusterInfo(ADDRESS_A); + when(adminService.getConsumerConnectionList(eq(ADDRESS_A), eq(GROUP), anyLong())) + .thenReturn(CompletableFuture.completedFuture( + consumerConnection(CLIENT_ID, "consumer-host@192.168.1.5:43210", TOPIC))); + + SimpleObserver observer = new SimpleObserver<>(); + service.listConsumerConnection(ListConsumerConnectionRequest.newBuilder() + .setGroup(resource(GROUP)).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(1, observer.value.getClientInfoCount()); + assertEquals(CLIENT_ID, observer.value.getClientInfo(0).getClientId()); + // egress_ip is the bare IP, hostname the client-reported name from "hostname@ip:port" + assertEquals("192.168.1.5", observer.value.getClientInfo(0).getEgressIp()); + assertEquals("consumer-host", observer.value.getClientInfo(0).getHostname()); + assertEquals("355", observer.value.getClientInfo(0).getVersion()); + assertEquals(apache.rocketmq.v2.MessageModel.CLUSTERING, observer.value.getClientInfo(0).getMessageModel()); + } + + @Test + public void listConsumerConnectionOfflineGroupIsOkTest() { + stubClusterInfo(ADDRESS_A); + when(adminService.getConsumerConnectionList(eq(ADDRESS_A), eq(GROUP), anyLong())) + .thenReturn(failedFuture(new RuntimeException("consumer group not online"))); + + SimpleObserver observer = new SimpleObserver<>(); + service.listConsumerConnection(ListConsumerConnectionRequest.newBuilder() + .setGroup(resource(GROUP)).build(), observer); + + assertNotNull(observer.value); + // an offline group is a normal answer for a listing RPC, not a server error + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(0, observer.value.getClientInfoCount()); + } + + @Test + public void listConsumerConnectionTopicFilterHidesNonSubscribersTest() { + stubClusterInfo(ADDRESS_A); + when(adminService.getConsumerConnectionList(eq(ADDRESS_A), eq(GROUP), anyLong())) + .thenReturn(CompletableFuture.completedFuture(consumerConnection(CLIENT_ID, "192.168.1.5:43210", TOPIC))); + + SimpleObserver observer = new SimpleObserver<>(); + service.listConsumerConnection(ListConsumerConnectionRequest.newBuilder() + .setGroup(resource(GROUP)) + .setTopic(resource("topicOther")) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(0, observer.value.getClientInfoCount()); + } + + // ------------------------------------------------------------------ 8. ResetGroupOffset + + @Test + public void resetGroupOffsetRejectsMissingTimestampTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.resetGroupOffset(ResetGroupOffsetRequest.newBuilder() + .setGroup(resource(GROUP)) + .setTopic(resource(TOPIC)) + .build(), observer); + + assertNotNull(observer.value); + // an unset Timestamp used to mean "replay everything since 1970" + assertEquals(Code.BAD_REQUEST, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("reset_timestamp")); + verify(adminService, never()).resetOffset(anyString(), anyString(), anyString(), anyLong(), anyBoolean(), + anyLong()); + } + + @Test + public void resetGroupOffsetRejectsZeroTimestampTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.resetGroupOffset(ResetGroupOffsetRequest.newBuilder() + .setGroup(resource(GROUP)) + .setTopic(resource(TOPIC)) + .setResetTimestamp(Timestamp.newBuilder().setSeconds(0L).build()) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.BAD_REQUEST, observer.value.getStatus().getCode()); + verify(adminService, never()).resetOffset(anyString(), anyString(), anyString(), anyLong(), anyBoolean(), + anyLong()); + } + + @Test + public void resetGroupOffsetReachesEveryBrokerTest() throws Exception { + stubRoute(true, ADDRESS_A, ADDRESS_B); + when(adminService.resetOffset(anyString(), eq(TOPIC), eq(GROUP), anyLong(), eq(true), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Collections.emptyMap())); + + SimpleObserver observer = new SimpleObserver<>(); + service.resetGroupOffset(ResetGroupOffsetRequest.newBuilder() + .setGroup(resource(GROUP)) + .setTopic(resource(TOPIC)) + .setResetTimestamp(Timestamp.newBuilder().setSeconds(1000L).setNanos(500_000_000).build()) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + // seconds + nanos must be converted to one millisecond timestamp + verify(adminService).resetOffset(eq(ADDRESS_A), eq(TOPIC), eq(GROUP), eq(1000500L), eq(true), anyLong()); + verify(adminService).resetOffset(eq(ADDRESS_B), eq(TOPIC), eq(GROUP), eq(1000500L), eq(true), anyLong()); + } + + @Test + public void resetGroupOffsetPropagatesBrokerFailureTest() throws Exception { + stubRoute(true, ADDRESS_A); + when(adminService.resetOffset(anyString(), eq(TOPIC), eq(GROUP), anyLong(), eq(true), anyLong())) + .thenReturn(failedFuture(new RuntimeException("reset rejected"))); + + SimpleObserver observer = new SimpleObserver<>(); + service.resetGroupOffset(ResetGroupOffsetRequest.newBuilder() + .setGroup(resource(GROUP)) + .setTopic(resource(TOPIC)) + .setResetTimestamp(Timestamp.newBuilder().setSeconds(1000L).build()) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.INTERNAL_SERVER_ERROR, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("reset rejected")); + } + + // ------------------------------------------------------------------ 9. QueryMessage + + @Test + public void queryMessageRejectsMissingTopicTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.queryMessage(ListMessageRequest.newBuilder().setMessageKey("key").build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.BAD_REQUEST, observer.value.getStatus().getCode()); + } + + @Test + public void queryMessageRejectsUnsupportedSearchKeyTest() throws Exception { + stubRoute(false, ADDRESS_A); + SimpleObserver noKeyObserver = new SimpleObserver<>(); + service.queryMessage(ListMessageRequest.newBuilder().setTopic(resource(TOPIC)).build(), noKeyObserver); + assertNotNull(noKeyObserver.value); + assertEquals(Code.BAD_REQUEST, noKeyObserver.value.getStatus().getCode()); + + SimpleObserver subscriptionObserver = new SimpleObserver<>(); + service.queryMessage(ListMessageRequest.newBuilder() + .setTopic(resource(TOPIC)) + .setSubscription("tagA") + .build(), subscriptionObserver); + assertNotNull(subscriptionObserver.value); + assertEquals(Code.BAD_REQUEST, subscriptionObserver.value.getStatus().getCode()); + assertTrue(subscriptionObserver.value.getStatus().getMessage().contains("not supported")); + } + + @Test + public void queryMessageByMessageIdUsesUniqueKeyTest() throws Exception { + stubRoute(false, ADDRESS_A); + when(adminService.queryMessage(anyString(), anyString(), anyString(), anyInt(), anyLong(), anyLong(), + anyBoolean(), anyBoolean(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Collections.singletonList(messageExt("UNIQ-1")))); + + SimpleObserver observer = new SimpleObserver<>(); + service.queryMessage(ListMessageRequest.newBuilder() + .setTopic(resource(TOPIC)) + .setMessageId("UNIQ-1") + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(1, observer.value.getMessagesCount()); + assertEquals("UNIQ-1", observer.value.getMessages(0).getSystemProperties().getMessageId()); + assertEquals("payload", observer.value.getMessages(0).getBody().toString(StandardCharsets.UTF_8)); + + // a v2 message_id is the client-generated unique key, so the lookup must use the + // unique-key index (uniqueKey=true), never the offset-decoding path + ArgumentCaptor uniqueKeyCaptor = ArgumentCaptor.forClass(Boolean.class); + verify(adminService).queryMessage(eq(ADDRESS_A), eq(TOPIC), eq("UNIQ-1"), anyInt(), anyLong(), anyLong(), + uniqueKeyCaptor.capture(), anyBoolean(), anyLong()); + assertTrue(uniqueKeyCaptor.getValue()); + } + + @Test + public void queryMessageByKeyUsesKeyIndexTest() throws Exception { + stubRoute(false, ADDRESS_A); + when(adminService.queryMessage(anyString(), anyString(), anyString(), anyInt(), anyLong(), anyLong(), + anyBoolean(), anyBoolean(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Collections.singletonList(messageExt("MSG-1")))); + + SimpleObserver observer = new SimpleObserver<>(); + service.queryMessage(ListMessageRequest.newBuilder() + .setTopic(resource(TOPIC)) + .setMessageKey("key-1") + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(1, observer.value.getMessagesCount()); + + ArgumentCaptor uniqueKeyCaptor = ArgumentCaptor.forClass(Boolean.class); + verify(adminService).queryMessage(eq(ADDRESS_A), eq(TOPIC), eq("key-1"), anyInt(), anyLong(), anyLong(), + uniqueKeyCaptor.capture(), anyBoolean(), anyLong()); + assertFalse(uniqueKeyCaptor.getValue()); + } + + @Test + public void queryMessageReturnsNotFoundWhenNoBrokerHasItTest() throws Exception { + stubRoute(false, ADDRESS_A, ADDRESS_B); + when(adminService.queryMessage(anyString(), anyString(), anyString(), anyInt(), anyLong(), anyLong(), + anyBoolean(), anyBoolean(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Collections.emptyList())); + + SimpleObserver observer = new SimpleObserver<>(); + service.queryMessage(ListMessageRequest.newBuilder() + .setTopic(resource(TOPIC)) + .setMessageId("UNIQ-MISSING") + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.MESSAGE_NOT_FOUND, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("no message found")); + assertEquals(0, observer.value.getMessagesCount()); + } + + @Test + public void queryMessageSurvivesPartialBrokerFailureTest() throws Exception { + stubRoute(false, ADDRESS_A, ADDRESS_B); + when(adminService.queryMessage(eq(ADDRESS_A), eq(TOPIC), eq("key-1"), anyInt(), anyLong(), anyLong(), + anyBoolean(), anyBoolean(), anyLong())) + .thenReturn(failedFuture(new RuntimeException("broker-a down"))); + when(adminService.queryMessage(eq(ADDRESS_B), eq(TOPIC), eq("key-1"), anyInt(), anyLong(), anyLong(), + anyBoolean(), anyBoolean(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Collections.singletonList(messageExt("MSG-2")))); + + SimpleObserver observer = new SimpleObserver<>(); + service.queryMessage(ListMessageRequest.newBuilder() + .setTopic(resource(TOPIC)) + .setMessageKey("key-1") + .build(), observer); + + assertNotNull(observer.value); + // a partially available cluster still returns the data it has + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(1, observer.value.getMessagesCount()); + } + + @Test + public void queryMessageRespectsMaxNumsTest() throws Exception { + stubRoute(false, ADDRESS_A); + when(adminService.queryMessage(anyString(), anyString(), anyString(), anyInt(), anyLong(), anyLong(), + anyBoolean(), anyBoolean(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Arrays.asList(messageExt("MSG-1"), messageExt("MSG-2")))); + + SimpleObserver observer = new SimpleObserver<>(); + service.queryMessage(ListMessageRequest.newBuilder() + .setTopic(resource(TOPIC)) + .setMessageKey("key-1") + .setMaxMessageNums(1) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(1, observer.value.getMessagesCount()); + verify(adminService).queryMessage(eq(ADDRESS_A), eq(TOPIC), eq("key-1"), eq(1), anyLong(), anyLong(), + anyBoolean(), anyBoolean(), anyLong()); + } + + // ------------------------------------------------------------------ 10/13. client relay RPCs + + @Test + public void printThreadStackTraceReturnsJstackTest() { + stubLocalClient(); + ConsumerRunningInfo runningInfo = new ConsumerRunningInfo(); + runningInfo.setJstack("STACK-TRACE"); + answerRelayWith(RequestCode.GET_CONSUMER_RUNNING_INFO, + new ProxyRelayResult<>(ResponseCode.SUCCESS, "ok", runningInfo)); + + SimpleObserver observer = new SimpleObserver<>(); + service.printThreadStackTrace(PrintThreadStackTraceRequest.newBuilder() + .setGroup(resource(GROUP)) + .setClientId(CLIENT_ID) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals("STACK-TRACE", observer.value.getThreadStackTrace()); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ProxyRelayRequest.class); + verify(channel).writeAndFlush(captor.capture()); + GetConsumerRunningInfoRequestHeader header = + (GetConsumerRunningInfoRequestHeader) captor.getValue().readCustomHeader(); + assertTrue(header.isJstackEnable()); + assertEquals(GROUP, header.getConsumerGroup()); + assertEquals(CLIENT_ID, header.getClientId()); + } + + @Test + public void printThreadStackTraceEmptyJstackIsNotFoundTest() { + stubLocalClient(); + answerRelayWith(RequestCode.GET_CONSUMER_RUNNING_INFO, + new ProxyRelayResult<>(ResponseCode.SUCCESS, "ok", new ConsumerRunningInfo())); + + SimpleObserver observer = new SimpleObserver<>(); + service.printThreadStackTrace(PrintThreadStackTraceRequest.newBuilder() + .setGroup(resource(GROUP)) + .setClientId(CLIENT_ID) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.NOT_FOUND, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("did not return a thread stack")); + } + + @Test + public void printThreadStackTraceMissingClientIsNotFoundTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.printThreadStackTrace(PrintThreadStackTraceRequest.newBuilder() + .setClientId("missing-client") + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.NOT_FOUND, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("not connected")); + } + + @Test + public void printThreadStackTraceForwardsToOwningProxyTest() { + when(forwarder.forwardIfRemote(eq(GROUP), eq(CLIENT_ID), any(), any())).thenReturn(true); + + SimpleObserver observer = new SimpleObserver<>(); + service.printThreadStackTrace(PrintThreadStackTraceRequest.newBuilder() + .setGroup(resource(GROUP)) + .setClientId(CLIENT_ID) + .build(), observer); + + // ownership of the observer moved to the peer proxy: the local service must stay silent + assertNull(observer.value); + assertNull(observer.error); + assertFalse(observer.completed); + } + + @Test + public void getConsumerRunningInfoRejectsMissingClientIdTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.getConsumerRunningInfo(GetConsumerRunningInfoRequest.newBuilder() + .setGroup(resource(GROUP)).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.BAD_REQUEST, observer.value.getStatus().getCode()); + } + + @Test + public void getConsumerRunningInfoReturnsFullInfoTest() { + stubLocalClient(); + ConsumerRunningInfo runningInfo = new ConsumerRunningInfo(); + Properties properties = runningInfo.getProperties(); + properties.setProperty(ConsumerRunningInfo.PROP_CLIENT_VERSION, "V5_0_0"); + runningInfo.getSubscriptionSet().add(subscriptionData(TOPIC, ExpressionType.TAG, "tagA")); + answerRelayWith(RequestCode.GET_CONSUMER_RUNNING_INFO, + new ProxyRelayResult<>(ResponseCode.SUCCESS, "ok", runningInfo)); + + SimpleObserver observer = new SimpleObserver<>(); + service.getConsumerRunningInfo(GetConsumerRunningInfoRequest.newBuilder() + .setGroup(resource(GROUP)) + .setClientId(CLIENT_ID) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals("V5_0_0", observer.value.getConsumerRunningInfo() + .getPropertiesMap().get(ConsumerRunningInfo.PROP_CLIENT_VERSION)); + assertTrue(observer.value.getConsumerRunningInfo().getSubscriptionsMap().containsKey(TOPIC)); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ProxyRelayRequest.class); + verify(channel).writeAndFlush(captor.capture()); + GetConsumerRunningInfoRequestHeader header = + (GetConsumerRunningInfoRequestHeader) captor.getValue().readCustomHeader(); + assertFalse(header.isJstackEnable()); + } + + @Test + public void getConsumerRunningInfoFallsBackToSubscriptionsForGrpcClientTest() { + stubLocalClient(); + // the GrpcClientChannel jstack guard completes the relay future with this result for a + // gRPC v2 client that cannot answer a running-info request without a thread dump + answerRelayWith(RequestCode.GET_CONSUMER_RUNNING_INFO, + new ProxyRelayResult<>(ResponseCode.REQUEST_CODE_NOT_SUPPORTED, + "gRPC v2 protocol cannot report consumer running info without jstack, " + + "retry with jstackEnable=true", null)); + when(grpcClientSettingsManager.getRawClientSettings(CLIENT_ID)) + .thenReturn(settings(ClientType.SIMPLE_CONSUMER, GROUP, TOPIC, "tagA")); + + SimpleObserver observer = new SimpleObserver<>(); + service.getConsumerRunningInfo(GetConsumerRunningInfoRequest.newBuilder() + .setGroup(resource(GROUP)) + .setClientId(CLIENT_ID) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("only subscriptions are available")); + assertEquals("tagA", observer.value.getConsumerRunningInfo() + .getSubscriptionsMap().get(TOPIC).getExpression()); + } + + @Test + public void getConsumerRunningInfoErrorsWithoutSettingsOrChannelTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.getConsumerRunningInfo(GetConsumerRunningInfoRequest.newBuilder() + .setClientId("missing-client") + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.INTERNAL_SERVER_ERROR, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("not connected")); + } + + // ------------------------------------------------------------------ 11. VerifyMessage + + @Test + public void verifyMessageRejectsMissingFieldsTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.verifyMessage(VerifyMessageRequest.newBuilder() + .setClientId(CLIENT_ID) + .setTopic(resource(TOPIC)) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.BAD_REQUEST, observer.value.getStatus().getCode()); + } + + @Test + public void verifyMessageSuccessTest() throws Exception { + stubRoute(false, ADDRESS_A); + stubLocalClient(); + when(adminService.queryMessage(anyString(), anyString(), anyString(), anyInt(), anyLong(), anyLong(), + anyBoolean(), anyBoolean(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Collections.singletonList(messageExt("UNIQ-9")))); + ConsumeMessageDirectlyResult directlyResult = new ConsumeMessageDirectlyResult(); + directlyResult.setConsumeResult(CMResult.CR_SUCCESS); + answerRelayWith(RequestCode.CONSUME_MESSAGE_DIRECTLY, + new ProxyRelayResult<>(ResponseCode.SUCCESS, "ok", directlyResult)); + + SimpleObserver observer = new SimpleObserver<>(); + service.verifyMessage(VerifyMessageRequest.newBuilder() + .setGroup(resource(GROUP)) + .setClientId(CLIENT_ID) + .setTopic(resource(TOPIC)) + .setMessageId("UNIQ-9") + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + + // the real message is fetched through the unique-key index and relayed to the client + verify(adminService).queryMessage(eq(ADDRESS_A), eq(TOPIC), eq("UNIQ-9"), eq(1), eq(0L), eq(Long.MAX_VALUE), + eq(true), eq(false), anyLong()); + ArgumentCaptor captor = ArgumentCaptor.forClass(ProxyRelayRequest.class); + verify(channel).writeAndFlush(captor.capture()); + ConsumeMessageDirectlyResultRequestHeader header = + (ConsumeMessageDirectlyResultRequestHeader) captor.getValue().readCustomHeader(); + assertEquals(GROUP, header.getConsumerGroup()); + assertEquals(CLIENT_ID, header.getClientId()); + assertEquals(TOPIC, header.getTopic()); + assertEquals("UNIQ-9", header.getMsgId()); + assertNotNull(captor.getValue().getBody()); + } + + @Test + public void verifyMessageReportsCorruptedOnFailureTest() throws Exception { + stubRoute(false, ADDRESS_A); + stubLocalClient(); + when(adminService.queryMessage(anyString(), anyString(), anyString(), anyInt(), anyLong(), anyLong(), + anyBoolean(), anyBoolean(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Collections.singletonList(messageExt("UNIQ-9")))); + ConsumeMessageDirectlyResult directlyResult = new ConsumeMessageDirectlyResult(); + directlyResult.setConsumeResult(CMResult.CR_THROW_EXCEPTION); + directlyResult.setRemark("consume crashed"); + answerRelayWith(RequestCode.CONSUME_MESSAGE_DIRECTLY, + new ProxyRelayResult<>(ResponseCode.SUCCESS, "ok", directlyResult)); + + SimpleObserver observer = new SimpleObserver<>(); + service.verifyMessage(VerifyMessageRequest.newBuilder() + .setGroup(resource(GROUP)) + .setClientId(CLIENT_ID) + .setTopic(resource(TOPIC)) + .setMessageId("UNIQ-9") + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.MESSAGE_CORRUPTED, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("failed to consume message UNIQ-9")); + assertTrue(observer.value.getStatus().getMessage().contains("consume crashed")); + } + + @Test + public void verifyMessageFailsWhenMessageNotFoundTest() throws Exception { + stubRoute(false, ADDRESS_A); + when(adminService.queryMessage(anyString(), anyString(), anyString(), anyInt(), anyLong(), anyLong(), + anyBoolean(), anyBoolean(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Collections.emptyList())); + + SimpleObserver observer = new SimpleObserver<>(); + service.verifyMessage(VerifyMessageRequest.newBuilder() + .setGroup(resource(GROUP)) + .setClientId(CLIENT_ID) + .setTopic(resource(TOPIC)) + .setMessageId("UNIQ-MISSING") + .build(), observer); + + assertNotNull(observer.value); + // a missing message is a normal answer, so it must not be reported as a proxy fault; + // same grading queryMessage uses + assertEquals(Code.MESSAGE_NOT_FOUND, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("not found")); + } + + // ------------------------------------------------------------------ 12. AdminSendMessage + + @Test + public void adminSendMessageReturnsIdTest() { + SendResult sendResult = new SendResult(); + sendResult.setMsgId("SENT-1"); + when(messagingProcessor.sendMessage(any(), any(), anyString(), anyInt(), anyList(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Collections.singletonList(sendResult))); + + SimpleObserver observer = new SimpleObserver<>(); + service.adminSendMessage(AdminSendMessageRequest.newBuilder() + .setTopic(resource(TOPIC)) + .setBody(ByteString.copyFromUtf8("hello")) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals("SENT-1", observer.value.getMessageId()); + } + + @Test + public void adminSendMessageCarriesTagsKeysAndPropertiesTest() { + SendResult sendResult = new SendResult(); + sendResult.setMsgId("SENT-2"); + when(messagingProcessor.sendMessage(any(), any(), anyString(), anyInt(), anyList(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Collections.singletonList(sendResult))); + + SimpleObserver observer = new SimpleObserver<>(); + service.adminSendMessage(AdminSendMessageRequest.newBuilder() + .setTopic(resource(TOPIC)) + .setBody(ByteString.copyFromUtf8("hello")) + .setTag("tagA") + .setKey("key1") + .putUserProperties("traceId", "abc") + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals("SENT-2", observer.value.getMessageId()); + + @SuppressWarnings("rawtypes") + ArgumentCaptor messagesCaptor = ArgumentCaptor.forClass(List.class); + verify(messagingProcessor).sendMessage(any(), any(), anyString(), anyInt(), messagesCaptor.capture(), + anyLong()); + Message message = (Message) messagesCaptor.getValue().get(0); + assertEquals(TOPIC, message.getTopic()); + assertEquals("tagA", message.getTags()); + assertEquals("key1", message.getKeys()); + assertEquals("abc", message.getUserProperty("traceId")); + } + + @Test + public void adminSendMessageRejectsPastDeliveryTimestampTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.adminSendMessage(AdminSendMessageRequest.newBuilder() + .setTopic(resource(TOPIC)) + .setBody(ByteString.copyFromUtf8("hello")) + .setSystemProperties(SystemProperties.newBuilder() + .setDeliveryTimestamp(Timestamp.newBuilder() + .setSeconds(System.currentTimeMillis() / 1000L - 100L).build()) + .build()) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.ILLEGAL_DELIVERY_TIME, observer.value.getStatus().getCode()); + } + + @Test + public void adminSendMessageFailsGracefullyWhenSendFailsTest() { + when(messagingProcessor.sendMessage(any(), any(), anyString(), anyInt(), anyList(), anyLong())) + .thenReturn(failedFuture(new RuntimeException("send failed"))); + + SimpleObserver observer = new SimpleObserver<>(); + service.adminSendMessage(AdminSendMessageRequest.newBuilder() + .setTopic(resource(TOPIC)) + .setBody(ByteString.copyFromUtf8("hello")) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.INTERNAL_SERVER_ERROR, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("send failed")); + } + + // ------------------------------------------------------------------ 14. GetTopicRoute + + @Test + public void getTopicRouteReturnsJsonTest() { + TopicRouteData routeData = new TopicRouteData(); + routeData.setOrderTopicConf("orderConf"); + when(adminService.getTopicRouteData(TOPIC)).thenReturn(CompletableFuture.completedFuture(routeData)); + + SimpleObserver observer = new SimpleObserver<>(); + service.getTopicRoute(GetTopicRouteRequest.newBuilder().setTopic(resource(TOPIC)).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(JSON.toJSONString(routeData), observer.value.getTopicRouteData()); + } + + @Test + public void getTopicRouteFailsWhenAdminServiceFailsTest() { + when(adminService.getTopicRouteData(TOPIC)) + .thenReturn(failedFuture(new RuntimeException("namesrv unreachable"))); + + SimpleObserver observer = new SimpleObserver<>(); + service.getTopicRoute(GetTopicRouteRequest.newBuilder().setTopic(resource(TOPIC)).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.INTERNAL_SERVER_ERROR, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("namesrv unreachable")); + } + + @Test + public void getTopicRouteRejectsMissingTopicTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.getTopicRoute(GetTopicRouteRequest.newBuilder().build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.BAD_REQUEST, observer.value.getStatus().getCode()); + } + + // ------------------------------------------------------------------ 15. QueryTimeSpan + + @Test + public void queryTimeSpanRejectsMissingGroupTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.queryTimeSpan(QueryTimeSpanRequest.newBuilder() + .addTopics(resource(TOPIC)).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.BAD_REQUEST, observer.value.getStatus().getCode()); + } + + @Test + public void queryTimeSpanRejectsMissingTopicsTest() { + SimpleObserver observer = new SimpleObserver<>(); + service.queryTimeSpan(QueryTimeSpanRequest.newBuilder() + .setGroup(resource(GROUP)).build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.BAD_REQUEST, observer.value.getStatus().getCode()); + } + + @Test + public void queryTimeSpanReturnsBrokerComputedSpansTest() throws Exception { + stubRoute(false, ADDRESS_A); + QueueTimeSpan span = new QueueTimeSpan(); + span.setMessageQueue(new MessageQueue(TOPIC, "broker-0", 0)); + span.setMinTimeStamp(100L); + span.setMaxTimeStamp(200L); + span.setConsumeTimeStamp(150L); + span.setDelayTime(42L); + when(adminService.queryConsumeTimeSpan(eq(ADDRESS_A), eq(TOPIC), eq(GROUP), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Collections.singletonList(span))); + + SimpleObserver observer = new SimpleObserver<>(); + service.queryTimeSpan(QueryTimeSpanRequest.newBuilder() + .setGroup(resource(GROUP)) + .addTopics(resource(TOPIC)) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(1, observer.value.getQueueTimeSpanListCount()); + QueryTimeSpanResponse.QueueTimeSpan v2Span = observer.value.getQueueTimeSpanList(0); + assertEquals("broker-0", v2Span.getMessageQueue().getBroker().getName()); + assertEquals(100L, v2Span.getMinTimestamp()); + assertEquals(200L, v2Span.getMaxTimestamp()); + assertEquals(150L, v2Span.getConsumeTimestamp()); + assertEquals(42L, v2Span.getDelayTimeMs()); + } + + @Test + public void queryTimeSpanPropagatesFailureWhenAllBrokersFailTest() throws Exception { + stubRoute(false, ADDRESS_A); + when(adminService.queryConsumeTimeSpan(eq(ADDRESS_A), eq(TOPIC), eq(GROUP), anyLong())) + .thenReturn(failedFuture(new RuntimeException("span query failed"))); + + SimpleObserver observer = new SimpleObserver<>(); + service.queryTimeSpan(QueryTimeSpanRequest.newBuilder() + .setGroup(resource(GROUP)) + .addTopics(resource(TOPIC)) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.INTERNAL_SERVER_ERROR, observer.value.getStatus().getCode()); + assertTrue(observer.value.getStatus().getMessage().contains("span query failed")); + } + + // ------------------------------------------------------------------ 16. GetProxyRuntimeStats + + @Test + public void getProxyRuntimeStatsCountsLitePushConsumerAsConsumerTest() { + GrpcClientChannel consumerChannel = mock(GrpcClientChannel.class); + when(consumerChannel.getClientId()).thenReturn(CLIENT_ID); + GrpcClientChannel producerChannel = mock(GrpcClientChannel.class); + when(producerChannel.getClientId()).thenReturn("producer-1"); + GrpcClientChannel litePushChannel = mock(GrpcClientChannel.class); + when(litePushChannel.getClientId()).thenReturn("lite-push-1"); + when(grpcChannelManager.getClientChannels()) + .thenReturn(Arrays.asList(consumerChannel, producerChannel, litePushChannel)); + when(grpcClientSettingsManager.getRawClientSettings(CLIENT_ID)) + .thenReturn(settings(ClientType.SIMPLE_CONSUMER, GROUP, TOPIC, "*")); + when(grpcClientSettingsManager.getRawClientSettings("producer-1")) + .thenReturn(settings(ClientType.PRODUCER, GROUP, TOPIC, "*")); + when(grpcClientSettingsManager.getRawClientSettings("lite-push-1")) + .thenReturn(settings(ClientType.LITE_PUSH_CONSUMER, GROUP, TOPIC, "*")); + + SimpleObserver observer = new SimpleObserver<>(); + service.getProxyRuntimeStats(GetProxyRuntimeStatsRequest.newBuilder().build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + // LITE_PUSH_CONSUMER is a consumer, not a producer + assertEquals(2L, observer.value.getConsumers()); + assertEquals(1L, observer.value.getProducers()); + assertEquals(3L, observer.value.getConnections()); + // the version comes from MQVersion, not from a hardcoded string + assertEquals(MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION), observer.value.getVersion()); + assertEquals(ConfigurationManager.getProxyConfig().getProxyName(), observer.value.getProxyName()); + } + + @Test + public void getProxyRuntimeStatsSkipsChannelsWithoutSettingsTest() { + GrpcClientChannel unknownChannel = mock(GrpcClientChannel.class); + when(unknownChannel.getClientId()).thenReturn("unknown"); + when(grpcChannelManager.getClientChannels()).thenReturn(Collections.singletonList(unknownChannel)); + + SimpleObserver observer = new SimpleObserver<>(); + service.getProxyRuntimeStats(GetProxyRuntimeStatsRequest.newBuilder().build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + assertEquals(1L, observer.value.getConnections()); + assertEquals(0L, observer.value.getProducers()); + assertEquals(0L, observer.value.getConsumers()); + } + + // ------------------------------------------------------------------ cross-RPC + + @Test + public void describeGroupAccumulationKeepsRetryBacklogInTotalTest() { + stubClusterInfo(ADDRESS_A); + ConsumeStats stats = consumeStats(TOPIC, 100L, 60L, 60L); + OffsetWrapper retryWrapper = new OffsetWrapper(); + retryWrapper.setBrokerOffset(30L); + retryWrapper.setConsumerOffset(10L); + retryWrapper.setPullOffset(10L); + stats.getOffsetTable().put(new MessageQueue(MixAll.getRetryTopic(GROUP), "broker-0", 0), retryWrapper); + when(adminService.getConsumeStats(eq(ADDRESS_A), eq(GROUP), eq(""), anyLong())) + .thenReturn(CompletableFuture.completedFuture(stats)); + + SimpleObserver observer = new SimpleObserver<>(); + service.describeGroupAccumulation(DescribeGroupAccumulationRequest.newBuilder() + .setGroup(resource(GROUP)) + .addTopics(resource(TOPIC)) + .build(), observer); + + assertNotNull(observer.value); + assertEquals(Code.OK, observer.value.getStatus().getCode()); + // 40 from the topic + 20 from the group's retry topic + assertEquals(60L, observer.value.getAccumulation().getAccumulation()); + assertEquals(40L, observer.value.getTopicAccumulationMap().get(TOPIC).getAccumulation()); + assertFalse(observer.value.getTopicAccumulationMap().containsKey(MixAll.getRetryTopic(GROUP))); + } +} diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannelTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannelTest.java index 1bdbdd9befe..4b84366f24a 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannelTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannelTest.java @@ -20,6 +20,10 @@ import apache.rocketmq.v2.Publishing; import apache.rocketmq.v2.Resource; import apache.rocketmq.v2.Settings; +import apache.rocketmq.v2.TelemetryCommand; +import io.grpc.stub.StreamObserver; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.RandomStringUtils; import org.apache.rocketmq.proxy.common.ProxyContext; import org.apache.rocketmq.proxy.config.InitConfigTest; @@ -27,17 +31,30 @@ import org.apache.rocketmq.proxy.processor.channel.ChannelProtocolType; import org.apache.rocketmq.proxy.processor.channel.RemoteChannel; import org.apache.rocketmq.proxy.remoting.channel.RemotingChannel; +import org.apache.rocketmq.proxy.service.relay.ClusterProxyRelayService; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayRequest; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayResult; import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; +import org.apache.rocketmq.proxy.service.transaction.TransactionService; +import org.apache.rocketmq.remoting.protocol.RequestCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; +import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.remoting.protocol.header.GetConsumerRunningInfoRequestHeader; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -79,4 +96,60 @@ public void testChannelExtendAttributeParse() { assertEquals(clientSettings, GrpcClientChannel.parseChannelExtendAttribute(this.grpcClientChannel)); assertNull(GrpcClientChannel.parseChannelExtendAttribute(mock(RemotingChannel.class))); } + + @Test + public void getConsumerRunningInfoWithoutJstackCompletesWithNotSupportedTest() throws Exception { + GrpcClientChannel channel = new GrpcClientChannel( + new ClusterProxyRelayService(mock(TransactionService.class)), grpcClientSettingsManager, + grpcChannelManager, + ProxyContext.create().setRemoteAddress("10.152.39.53:9768").setLocalAddress("11.193.0.1:1210"), + clientId); + + GetConsumerRunningInfoRequestHeader header = new GetConsumerRunningInfoRequestHeader(); + header.setConsumerGroup("group"); + header.setClientId(clientId); + header.setJstackEnable(false); + CompletableFuture> callerFuture = new CompletableFuture<>(); + + channel.writeAndFlush(ProxyRelayRequest.createRequestCommand( + RequestCode.GET_CONSUMER_RUNNING_INFO, header, callerFuture)); + + // the gRPC v2 telemetry protocol has no running-info reply without a thread dump; the guard + // must complete the caller immediately instead of letting it hang until the relay timeout + assertTrue(callerFuture.isDone()); + ProxyRelayResult result = callerFuture.get(1, TimeUnit.SECONDS); + assertEquals(ResponseCode.REQUEST_CODE_NOT_SUPPORTED, result.getCode()); + assertTrue(result.getRemark().contains("jstackEnable=true")); + assertNull(result.getResult()); + } + + @Test + @SuppressWarnings("unchecked") + public void getConsumerRunningInfoWithJstackWritesTelemetryCommandTest() { + GrpcClientChannel channel = new GrpcClientChannel( + new ClusterProxyRelayService(mock(TransactionService.class)), grpcClientSettingsManager, + grpcChannelManager, + ProxyContext.create().setRemoteAddress("10.152.39.53:9768").setLocalAddress("11.193.0.1:1210"), + clientId); + StreamObserver observer = mock(StreamObserver.class); + channel.setClientObserver(observer); + when(grpcChannelManager.addResponseFuture(any())).thenReturn("nonce-1"); + + GetConsumerRunningInfoRequestHeader header = new GetConsumerRunningInfoRequestHeader(); + header.setConsumerGroup("group"); + header.setClientId(clientId); + header.setJstackEnable(true); + CompletableFuture> callerFuture = new CompletableFuture<>(); + + channel.writeAndFlush(ProxyRelayRequest.createRequestCommand( + RequestCode.GET_CONSUMER_RUNNING_INFO, header, callerFuture)); + + ArgumentCaptor captor = ArgumentCaptor.forClass(TelemetryCommand.class); + verify(observer).onNext(captor.capture()); + TelemetryCommand command = captor.getValue(); + assertTrue(command.hasPrintThreadStackTraceCommand()); + assertEquals("nonce-1", command.getPrintThreadStackTraceCommand().getNonce()); + // the answer arrives through the telemetry stream, so the caller future stays pending here + assertFalse(callerFuture.isDone()); + } } \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminServiceTest.java index cdfc7f7fc23..b8136e948dd 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminServiceTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/admin/DefaultAdminServiceTest.java @@ -14,19 +14,42 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.rocketmq.proxy.service.admin; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Set; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.client.impl.admin.MqClientAdminImpl; +import org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt; +import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.remoting.protocol.ResponseCode; -import org.apache.rocketmq.remoting.protocol.route.BrokerData; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; +import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; +import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; +import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; +import org.apache.rocketmq.remoting.protocol.body.GroupList; +import org.apache.rocketmq.remoting.protocol.body.QueueTimeSpan; +import org.apache.rocketmq.remoting.protocol.body.TopicList; +import org.apache.rocketmq.remoting.protocol.header.DeleteSubscriptionGroupRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetConsumeStatsRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetConsumerConnectionListRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.GetTopicStatsInfoRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryConsumeTimeSpanRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryTopicConsumeByWhoRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.QueryTopicsByConsumerRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; -import org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt; -import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; +import org.apache.rocketmq.remoting.protocol.statictopic.TopicConfigAndQueueMapping; +import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -35,69 +58,320 @@ import org.mockito.junit.MockitoJUnitRunner; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doNothing; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class DefaultAdminServiceTest { + @Mock private MQClientAPIFactory mqClientAPIFactory; @Mock private MQClientAPIExt mqClientAPIExt; + @Mock + private MqClientAdminImpl mqClientAdmin; - private DefaultAdminService defaultAdminService; + private DefaultAdminService adminService; + + private static final String ADDR = "127.0.0.1:10911"; + private static final String TOPIC = "topicA"; + private static final String GROUP = "groupA"; + private static final long TIMEOUT = 3000L; + private static final long GET_TIMEOUT_SECONDS = 5L; @Before - public void before() { + public void setUp() { when(mqClientAPIFactory.getClient()).thenReturn(mqClientAPIExt); - defaultAdminService = new DefaultAdminService(mqClientAPIFactory); - } - - @Test - public void testCreateTopic() throws Exception { - when(mqClientAPIExt.getTopicRouteInfoFromNameServer(eq("createTopic"), anyLong())) - .thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, "")) - .thenReturn(createTopicRouteData(1)); - when(mqClientAPIExt.getTopicRouteInfoFromNameServer(eq("sampleTopic"), anyLong())) - .thenReturn(createTopicRouteData(2)); - - ArgumentCaptor addrArgumentCaptor = ArgumentCaptor.forClass(String.class); - ArgumentCaptor topicConfigArgumentCaptor = ArgumentCaptor.forClass(TopicConfig.class); - doNothing().when(mqClientAPIExt).createTopic(addrArgumentCaptor.capture(), anyString(), topicConfigArgumentCaptor.capture(), anyLong()); - - assertTrue(defaultAdminService.createTopicOnTopicBrokerIfNotExist( - "createTopic", - "sampleTopic", - 7, - 8, - true, - 1 - )); - - assertEquals(2, addrArgumentCaptor.getAllValues().size()); - Set createAddr = new HashSet<>(addrArgumentCaptor.getAllValues()); - assertTrue(createAddr.contains("127.0.0.1:10911")); - assertTrue(createAddr.contains("127.0.0.2:10911")); - assertEquals("createTopic", topicConfigArgumentCaptor.getValue().getTopicName()); - assertEquals(7, topicConfigArgumentCaptor.getValue().getWriteQueueNums()); - assertEquals(8, topicConfigArgumentCaptor.getValue().getReadQueueNums()); - } - - private TopicRouteData createTopicRouteData(int brokerNum) { - TopicRouteData topicRouteData = new TopicRouteData(); - for (int i = 0; i < brokerNum; i++) { - BrokerData brokerData = new BrokerData(); - HashMap addrMap = new HashMap<>(); - addrMap.put(0L, "127.0.0." + (i + 1) + ":10911"); - brokerData.setBrokerAddrs(addrMap); - brokerData.setBrokerName("broker-" + i); - brokerData.setCluster("cluster"); - topicRouteData.getBrokerDatas().add(brokerData); + when(mqClientAPIExt.getMqClientAdmin()).thenReturn(mqClientAdmin); + adminService = new DefaultAdminService(mqClientAPIFactory); + } + + @After + public void tearDown() { + adminService.shutdown(); + } + + private static T get(CompletableFuture future) throws Exception { + return future.get(GET_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + + @Test + public void getBrokerClusterInfoQueriesNameserverTest() throws Exception { + ClusterInfo clusterInfo = new ClusterInfo(); + when(mqClientAdmin.getBrokerClusterInfo(isNull(), eq(TIMEOUT))) + .thenReturn(CompletableFuture.completedFuture(clusterInfo)); + + assertSame(clusterInfo, get(adminService.getBrokerClusterInfo(TIMEOUT))); + } + + @Test + public void getTopicRouteDataDelegatesToNameserverTest() throws Exception { + TopicRouteData routeData = new TopicRouteData(); + routeData.setOrderTopicConf("orderConf"); + when(mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(TOPIC), anyLong())).thenReturn(routeData); + + TopicRouteData result = get(adminService.getTopicRouteData(TOPIC)); + + assertNotNull(result); + assertEquals("orderConf", result.getOrderTopicConf()); + } + + @Test + public void getTopicRouteDataWrapsFailuresTest() throws Exception { + when(mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(TOPIC), anyLong())) + .thenThrow(new MQClientException("boom", null)); + + try { + get(adminService.getTopicRouteData(TOPIC)); + fail("expected getTopicRouteData to fail"); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + while (cause != null && !(cause instanceof MQClientException)) { + cause = cause.getCause(); + } + assertNotNull(cause); + assertTrue(cause.getMessage().contains("boom")); } - return topicRouteData; } -} \ No newline at end of file + + @Test + public void getTopicConfigRunsBlockingCallOnExecutorTest() throws Exception { + TopicConfigAndQueueMapping topicConfig = new TopicConfigAndQueueMapping(); + topicConfig.setTopicName(TOPIC); + topicConfig.setReadQueueNums(8); + when(mqClientAPIExt.getTopicConfig(ADDR, TOPIC, TIMEOUT)).thenReturn(topicConfig); + + TopicConfig result = get(adminService.getTopicConfig(ADDR, TOPIC, TIMEOUT)); + + assertSame(topicConfig, result); + assertEquals(8, result.getReadQueueNums()); + } + + @Test + public void getTopicStatsBuildsHeaderTest() throws Exception { + TopicStatsTable statsTable = new TopicStatsTable(); + when(mqClientAdmin.getTopicStatsInfo(eq(ADDR), any(GetTopicStatsInfoRequestHeader.class), eq(TIMEOUT))) + .thenReturn(CompletableFuture.completedFuture(statsTable)); + + assertSame(statsTable, get(adminService.getTopicStats(ADDR, TOPIC, TIMEOUT))); + + ArgumentCaptor captor = + ArgumentCaptor.forClass(GetTopicStatsInfoRequestHeader.class); + verify(mqClientAdmin).getTopicStatsInfo(eq(ADDR), captor.capture(), eq(TIMEOUT)); + assertEquals(TOPIC, captor.getValue().getTopic()); + } + + @Test + public void getConsumeStatsBuildsHeaderTest() throws Exception { + ConsumeStats consumeStats = new ConsumeStats(); + when(mqClientAdmin.getConsumeStats(eq(ADDR), any(GetConsumeStatsRequestHeader.class), eq(TIMEOUT))) + .thenReturn(CompletableFuture.completedFuture(consumeStats)); + + assertSame(consumeStats, get(adminService.getConsumeStats(ADDR, GROUP, TOPIC, TIMEOUT))); + // a blank topic means "every topic of the group" and must reach the broker as "" + assertSame(consumeStats, get(adminService.getConsumeStats(ADDR, GROUP, null, TIMEOUT))); + + ArgumentCaptor captor = + ArgumentCaptor.forClass(GetConsumeStatsRequestHeader.class); + verify(mqClientAdmin, times(2)) + .getConsumeStats(eq(ADDR), captor.capture(), eq(TIMEOUT)); + List headers = captor.getAllValues(); + assertEquals(GROUP, headers.get(0).getConsumerGroup()); + assertEquals(TOPIC, headers.get(0).getTopic()); + assertEquals(GROUP, headers.get(1).getConsumerGroup()); + assertEquals("", headers.get(1).getTopic()); + } + + @Test + public void queryConsumeTimeSpanBuildsHeaderTest() throws Exception { + List spans = Collections.singletonList(new QueueTimeSpan()); + when(mqClientAdmin.queryConsumeTimeSpan(eq(ADDR), any(QueryConsumeTimeSpanRequestHeader.class), eq(TIMEOUT))) + .thenReturn(CompletableFuture.completedFuture(spans)); + + assertSame(spans, get(adminService.queryConsumeTimeSpan(ADDR, TOPIC, GROUP, TIMEOUT))); + + ArgumentCaptor captor = + ArgumentCaptor.forClass(QueryConsumeTimeSpanRequestHeader.class); + verify(mqClientAdmin).queryConsumeTimeSpan(eq(ADDR), captor.capture(), eq(TIMEOUT)); + assertEquals(TOPIC, captor.getValue().getTopic()); + assertEquals(GROUP, captor.getValue().getGroup()); + } + + @Test + public void resetOffsetBuildsHeaderTest() throws Exception { + Map offsets = Collections.singletonMap(new MessageQueue(TOPIC, "broker-a", 0), 42L); + when(mqClientAdmin.invokeBrokerToResetOffset(eq(ADDR), any(ResetOffsetRequestHeader.class), eq(TIMEOUT))) + .thenReturn(CompletableFuture.completedFuture(offsets)); + + Map result = + get(adminService.resetOffset(ADDR, TOPIC, GROUP, 1234L, true, TIMEOUT)); + + assertEquals(42L, result.get(new MessageQueue(TOPIC, "broker-a", 0)).longValue()); + ArgumentCaptor captor = ArgumentCaptor.forClass(ResetOffsetRequestHeader.class); + verify(mqClientAdmin).invokeBrokerToResetOffset(eq(ADDR), captor.capture(), eq(TIMEOUT)); + assertEquals(TOPIC, captor.getValue().getTopic()); + assertEquals(GROUP, captor.getValue().getGroup()); + assertEquals(1234L, captor.getValue().getTimestamp()); + assertTrue(captor.getValue().isForce()); + } + + @Test + public void getConsumerConnectionListBuildsHeaderTest() throws Exception { + ConsumerConnection connection = new ConsumerConnection(); + when(mqClientAdmin.getConsumerConnectionList(eq(ADDR), + any(GetConsumerConnectionListRequestHeader.class), eq(TIMEOUT))) + .thenReturn(CompletableFuture.completedFuture(connection)); + + assertSame(connection, get(adminService.getConsumerConnectionList(ADDR, GROUP, TIMEOUT))); + + ArgumentCaptor captor = + ArgumentCaptor.forClass(GetConsumerConnectionListRequestHeader.class); + verify(mqClientAdmin).getConsumerConnectionList(eq(ADDR), captor.capture(), eq(TIMEOUT)); + assertEquals(GROUP, captor.getValue().getConsumerGroup()); + } + + @Test + public void queryTopicConsumeByWhoBuildsHeaderTest() throws Exception { + GroupList groupList = new GroupList(); + when(mqClientAdmin.queryTopicConsumeByWho(eq(ADDR), any(QueryTopicConsumeByWhoRequestHeader.class), + eq(TIMEOUT))).thenReturn(CompletableFuture.completedFuture(groupList)); + + assertSame(groupList, get(adminService.queryTopicConsumeByWho(ADDR, TOPIC, TIMEOUT))); + + ArgumentCaptor captor = + ArgumentCaptor.forClass(QueryTopicConsumeByWhoRequestHeader.class); + verify(mqClientAdmin).queryTopicConsumeByWho(eq(ADDR), captor.capture(), eq(TIMEOUT)); + assertEquals(TOPIC, captor.getValue().getTopic()); + } + + @Test + public void queryTopicsByConsumerBuildsHeaderTest() throws Exception { + TopicList topicList = new TopicList(); + when(mqClientAdmin.queryTopicsByConsumer(eq(ADDR), any(QueryTopicsByConsumerRequestHeader.class), + eq(TIMEOUT))).thenReturn(CompletableFuture.completedFuture(topicList)); + + assertSame(topicList, get(adminService.queryTopicsByConsumer(ADDR, GROUP, TIMEOUT))); + + ArgumentCaptor captor = + ArgumentCaptor.forClass(QueryTopicsByConsumerRequestHeader.class); + verify(mqClientAdmin).queryTopicsByConsumer(eq(ADDR), captor.capture(), eq(TIMEOUT)); + assertEquals(GROUP, captor.getValue().getGroup()); + } + + @Test + public void getSubscriptionGroupConfigRunsBlockingCallOnExecutorTest() throws Exception { + SubscriptionGroupConfig config = new SubscriptionGroupConfig(); + config.setGroupName(GROUP); + when(mqClientAPIExt.getSubscriptionGroupConfig(ADDR, GROUP, TIMEOUT)).thenReturn(config); + + SubscriptionGroupConfig result = get(adminService.getSubscriptionGroupConfig(ADDR, GROUP, TIMEOUT)); + + assertSame(config, result); + } + + @Test + public void updateSubscriptionGroupConfigDelegatesTest() throws Exception { + SubscriptionGroupConfig config = new SubscriptionGroupConfig(); + config.setGroupName(GROUP); + when(mqClientAdmin.updateOrCreateSubscriptionGroup(ADDR, config, TIMEOUT)) + .thenReturn(CompletableFuture.completedFuture(null)); + + get(adminService.updateSubscriptionGroupConfig(ADDR, config, TIMEOUT)); + + verify(mqClientAdmin).updateOrCreateSubscriptionGroup(ADDR, config, TIMEOUT); + } + + @Test + public void deleteSubscriptionGroupBuildsHeaderTest() throws Exception { + when(mqClientAdmin.deleteSubscriptionGroup(eq(ADDR), any(DeleteSubscriptionGroupRequestHeader.class), + eq(TIMEOUT))).thenReturn(CompletableFuture.completedFuture(null)); + + get(adminService.deleteSubscriptionGroup(ADDR, GROUP, true, TIMEOUT)); + + ArgumentCaptor captor = + ArgumentCaptor.forClass(DeleteSubscriptionGroupRequestHeader.class); + verify(mqClientAdmin).deleteSubscriptionGroup(eq(ADDR), captor.capture(), eq(TIMEOUT)); + assertEquals(GROUP, captor.getValue().getGroupName()); + assertTrue(captor.getValue().isCleanOffset()); + } + + @Test + public void queryMessagePassesUniqueKeyAndDecompressFlagsTest() throws Exception { + List messages = Collections.singletonList(new MessageExt()); + when(mqClientAdmin.queryMessage(eq(ADDR), anyBoolean(), anyBoolean(), + any(QueryMessageRequestHeader.class), eq(TIMEOUT))) + .thenReturn(CompletableFuture.completedFuture(messages)); + + // uniqueKey=true selects the client-generated unique-key index; header fields carried through + assertSame(messages, + get(adminService.queryMessage(ADDR, TOPIC, "UNIQ-1", 10, 100L, 200L, true, true, TIMEOUT))); + ArgumentCaptor captor = ArgumentCaptor.forClass(QueryMessageRequestHeader.class); + verify(mqClientAdmin).queryMessage(eq(ADDR), eq(true), eq(true), captor.capture(), eq(TIMEOUT)); + QueryMessageRequestHeader header = captor.getValue(); + assertEquals(TOPIC, header.getTopic()); + assertEquals("UNIQ-1", header.getKey()); + assertEquals(10, header.getMaxNum().intValue()); + assertEquals(100L, header.getBeginTimestamp().longValue()); + assertEquals(200L, header.getEndTimestamp().longValue()); + + // uniqueKey=false matches against the message keys index instead + get(adminService.queryMessage(ADDR, TOPIC, "key-1", 10, 0L, System.currentTimeMillis(), false, true, TIMEOUT)); + verify(mqClientAdmin).queryMessage(eq(ADDR), eq(false), eq(true), + any(QueryMessageRequestHeader.class), eq(TIMEOUT)); + + // decompressBody=false: VerifyMessage relays the message, so it wants the body as stored + get(adminService.queryMessage(ADDR, TOPIC, "UNIQ-1", 1, 0L, Long.MAX_VALUE, true, false, TIMEOUT)); + verify(mqClientAdmin).queryMessage(eq(ADDR), eq(true), eq(false), + any(QueryMessageRequestHeader.class), eq(TIMEOUT)); + } + + @Test + public void viewMessageBuildsHeaderTest() throws Exception { + MessageExt messageExt = new MessageExt(); + messageExt.setMsgId("m1"); + when(mqClientAdmin.viewMessage(eq(ADDR), any(ViewMessageRequestHeader.class), eq(TIMEOUT))) + .thenReturn(CompletableFuture.completedFuture(messageExt)); + + MessageExt result = get(adminService.viewMessage(ADDR, TOPIC, 12345L, TIMEOUT)); + + assertEquals("m1", result.getMsgId()); + ArgumentCaptor captor = ArgumentCaptor.forClass(ViewMessageRequestHeader.class); + verify(mqClientAdmin).viewMessage(eq(ADDR), captor.capture(), eq(TIMEOUT)); + assertEquals(TOPIC, captor.getValue().getTopic()); + assertEquals(12345L, captor.getValue().getOffset().longValue()); + } + + @Test + public void searchOffsetByTimestampRunsBlockingCallOnExecutorTest() throws Exception { + MessageQueue messageQueue = new MessageQueue(TOPIC, "broker-a", 0); + when(mqClientAPIExt.searchOffset(ADDR, messageQueue, 1234L, TIMEOUT)).thenReturn(42L); + + Long result = get(adminService.searchOffsetByTimestamp(ADDR, messageQueue, 1234L, TIMEOUT)); + + assertEquals(42L, result.longValue()); + } + + @Test + public void topicExistReflectsRouteLookupTest() throws Exception { + // a resolvable route means the topic exists + when(mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(TOPIC), anyLong())).thenReturn(new TopicRouteData()); + assertTrue(adminService.topicExist(TOPIC)); + + // a route lookup failure is treated as "does not exist" + when(mqClientAPIExt.getTopicRouteInfoFromNameServer(anyString(), anyLong())) + .thenThrow(new MQClientException("No route info of this topic", null)); + assertFalse(adminService.topicExist(TOPIC)); + } +} diff --git a/rocketmq-apis b/rocketmq-apis new file mode 160000 index 00000000000..3e60073c6da --- /dev/null +++ b/rocketmq-apis @@ -0,0 +1 @@ +Subproject commit 3e60073c6dab3430feaec443824802e23ecda681 diff --git a/rocketmq-proto/BUILD.bazel b/rocketmq-proto/BUILD.bazel new file mode 100644 index 00000000000..3d6108fd385 --- /dev/null +++ b/rocketmq-proto/BUILD.bazel @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Protocol classes generated at build time from the rocketmq-apis git submodule. +# +# The admin contract (admin.proto) is not part of any rocketmq-proto +# artifact published to Maven Central, so the Java + gRPC classes are generated +# here from the submodule's .proto sources rather than resolved as a Maven +# artifact. This mirrors what the rocketmq-proto Maven module does for the Maven +# build. + +genrule( + name = "rocketmq_proto_srcjar", + srcs = [ + "@com_google_protobuf_wkt//:wkt_protos", + "@rocketmq_apis//:v2_protos", + ], + tools = [ + "@com_google_protobuf_protoc_linux_x86_64//file", + "@com_google_protobuf_protoc_osx_x86_64//file", + "@io_grpc_protoc_gen_grpc_java_linux_x86_64//file", + "@io_grpc_protoc_gen_grpc_java_osx_x86_64//file", + ], + outs = ["rocketmq_proto.srcjar"], + cmd = """ + set -e + case "$$(uname -s)" in + Linux) + PROTOC="$$(pwd)/$(location @com_google_protobuf_protoc_linux_x86_64//file)" + PLUGIN="$$(pwd)/$(location @io_grpc_protoc_gen_grpc_java_linux_x86_64//file)" + ;; + Darwin) + PROTOC="$$(pwd)/$(location @com_google_protobuf_protoc_osx_x86_64//file)" + PLUGIN="$$(pwd)/$(location @io_grpc_protoc_gen_grpc_java_osx_x86_64//file)" + ;; + *) + echo "unsupported platform for proto codegen: $$(uname -s)" >&2 + exit 1 + ;; + esac + PT="$$(pwd)/$(@D)/proto_tree" + JAVA_OUT="$$(pwd)/$(@D)/java_out" + OUT="$$(pwd)/$@" + rm -rf $$PT $$JAVA_OUT + mkdir -p $$PT/apache/rocketmq/v2 $$PT/google/protobuf $$JAVA_OUT + for f in $(SRCS); do + case "$$f" in + */apache/rocketmq/v2/*.proto) cp "$$f" $$PT/apache/rocketmq/v2/ ;; + *) cp "$$f" $$PT/google/protobuf/ ;; + esac + done + $$PROTOC \ + --plugin=protoc-gen-grpc-java=$$PLUGIN \ + -I$$PT \ + --java_out=$$JAVA_OUT \ + --grpc-java_out=$$JAVA_OUT \ + $$PT/apache/rocketmq/v2/*.proto + if command -v jar > /dev/null 2>&1; then + (cd $$JAVA_OUT && jar cf $$OUT .) + else + (cd $$JAVA_OUT && zip -qr $$OUT .) + fi + """, +) + +java_library( + name = "rocketmq-proto", + srcs = [":rocketmq_proto_srcjar"], + visibility = ["//visibility:public"], + deps = [ + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:javax_annotation_javax_annotation_api", + ], +) diff --git a/rocketmq-proto/pom.xml b/rocketmq-proto/pom.xml new file mode 100644 index 00000000000..cfb094236c1 --- /dev/null +++ b/rocketmq-proto/pom.xml @@ -0,0 +1,143 @@ + + + + + rocketmq-all + org.apache.rocketmq + ${revision} + + + 4.0.0 + jar + + rocketmq-proto + rocketmq-proto ${project.version} + Apache RocketMQ protocol APIs. Generated at build time from the rocketmq-apis + git submodule (apache/rocketmq/v2/*.proto), so the admin surface is consumed from + source rather than from a published rocketmq-proto artifact. + + + 8 + 8 + + true + true + true + + + + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + io.grpc + grpc-api + ${grpc.version} + + + io.grpc + grpc-core + ${grpc.version} + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + + javax.annotation + javax.annotation-api + 1.3.2 + provided + + + + com.google.api.grpc + proto-google-common-protos + 2.9.0 + provided + + + + com.google.protobuf + protobuf-java + + + + + + + + + kr.motd.maven + os-maven-plugin + 1.7.1 + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + + ${project.basedir}/../rocketmq-apis + + apache/rocketmq/v2/definition.proto + apache/rocketmq/v2/service.proto + apache/rocketmq/v2/admin.proto + + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} + + + + + compile + compile-custom + + + + + + + diff --git a/test/BUILD.bazel b/test/BUILD.bazel index eb3e0795962..37510e23a64 100644 --- a/test/BUILD.bazel +++ b/test/BUILD.bazel @@ -80,7 +80,7 @@ java_library( "@maven//:io_grpc_grpc_testing", "@maven//:io_netty_netty_all", "@maven//:org_apache_commons_commons_lang3", - "@maven//:org_apache_rocketmq_rocketmq_proto", + "//rocketmq-proto:rocketmq-proto", "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", ], )