diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a65f197..a9e42df 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -29,10 +29,10 @@ jobs: with: persist-credentials: false - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 with: - java-version: "11" + java-version: "17" distribution: temurin cache: maven diff --git a/.mvn/jvm.config b/.mvn/jvm.config new file mode 100644 index 0000000..32599ce --- /dev/null +++ b/.mvn/jvm.config @@ -0,0 +1,10 @@ +--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED +--add-opens jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED +--add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED diff --git a/README.md b/README.md index 154f976..9837190 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ The CoProcessor is built from source and included in the Stackable Apache HBase ## OPA authorizer > [!IMPORTANT] -> The authorizer work best with product images for Apache HBase 2.6.0 (and later) as the HBase code in these versions provides more comprehensive coverage for ACL hooks. +> Version 1.x of the authorizer is built for Apache HBase 3.0.0 and requires Java 17. +> For Apache HBase 2.x, use the 0.3.x releases. ### Configuration @@ -37,6 +38,8 @@ For every action a request similar to the one below is sent to OPA. The importan - the namespace - the table (optional: omitted when e.g. creating a namespace) - the action (one of `READ`, `WRITE`, `EXEC`, `CREATE`, `ADMIN`) +- the operation type of data access requests (e.g. `GET`, `SCAN`, `PUT`, `DELETE`; `NONE` for administrative actions) +- the column families and qualifiers being accessed (`families`, empty when the request is not restricted to specific families) ```json { @@ -86,13 +89,22 @@ The following actions are subject to ACL checks: - creation and deletion of tables - enabling and disabling of tables - truncation and modification of tables +- modification of store file trackers (table, column family) - reading data (`Get`, `Scan`) - writing data (`Put`, `Append`, `Delete`) - batch mutations +- moving, assigning and unassigning regions +- snapshot operations (create, list, clone, restore, delete) +- bulk loading of HFiles +- quotas and throttling +- balancer +- replication peer management (including sync replication state transitions) +- cluster management (stopping the Master and RegionServers, shutdown, decommissioning, clearing dead servers) +- procedures and locks +- split and merge switches The following actions are currently excluded but will be included in future releases: -- modification of store file trackers (table, column family) -- moving, assigning and unassigning tables -- snapshot operations (create, list, clone, restore, delete) -- bulk loading of HFiles +- RSGroup management +- listing namespaces +- reading cluster metrics diff --git a/pom.xml b/pom.xml index 3837963..4c67bfc 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ tech.stackable hbase-opa-authorizer - 0.3.0 + 1.0.0 jar Apache Hadoop HBase OPA Authorizer @@ -31,7 +31,7 @@ - 11 + 17 ${java.version} ${java.version} ${java.version} @@ -52,12 +52,26 @@ 3.5.5 2.44.5 - 2.6.4 - 2.5.0 + 3.0.0 + 4.1.14 3.2.0 - 1.12.3 + 1.20.2 + + 5.13.4 + + + + org.junit + junit-bom + ${junit.version} + pom + import + + + + com.github.ben-manes.caffeine @@ -65,9 +79,9 @@ ${caffeine.version} - com.google.protobuf - protobuf-java - ${hbase.protobuf.version} + org.apache.hbase.thirdparty + hbase-shaded-protobuf + ${hbase-thirdparty.version} provided @@ -123,13 +137,18 @@ com.fasterxml.jackson.core jackson-databind - 2.12.7.1 + + 2.21.1 provided - junit - junit - 4.13.2 + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine test @@ -271,6 +290,29 @@ org.apache.maven.plugins maven-surefire-plugin ${maven-surefire-plugin.version} + + + + -Dorg.apache.hbase.thirdparty.io.netty.tryReflectionSetAccessible=true + --add-modules jdk.unsupported + --add-opens java.base/java.io=ALL-UNNAMED + --add-opens java.base/java.nio=ALL-UNNAMED + --add-opens java.base/sun.nio.ch=ALL-UNNAMED + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/jdk.internal.ref=ALL-UNNAMED + --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.util.concurrent=ALL-UNNAMED + --add-exports java.base/jdk.internal.misc=ALL-UNNAMED + --add-exports java.security.jgss/sun.security.krb5=ALL-UNNAMED + --add-exports java.base/sun.net.dns=ALL-UNNAMED + --add-exports java.base/sun.net.util=ALL-UNNAMED + --add-opens java.base/jdk.internal.util.random=ALL-UNNAMED + --add-opens java.base/sun.security.x509=ALL-UNNAMED + --add-opens java.base/sun.security.util=ALL-UNNAMED + --add-opens java.base/java.net=ALL-UNNAMED + + com.googlecode.maven-download-plugin diff --git a/src/main/java/tech/stackable/hbase/OpenPolicyAgentAccessController.java b/src/main/java/tech/stackable/hbase/OpenPolicyAgentAccessController.java index a749327..bff3e65 100644 --- a/src/main/java/tech/stackable/hbase/OpenPolicyAgentAccessController.java +++ b/src/main/java/tech/stackable/hbase/OpenPolicyAgentAccessController.java @@ -2,10 +2,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.MapMaker; -import com.google.protobuf.Message; -import com.google.protobuf.RpcCallback; -import com.google.protobuf.RpcController; -import com.google.protobuf.Service; import java.io.IOException; import java.util.Collections; import java.util.Iterator; @@ -56,7 +52,6 @@ import org.apache.hadoop.hbase.filter.ByteArrayComparable; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.ipc.RpcServer; -import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos; import org.apache.hadoop.hbase.quotas.GlobalQuotaSettings; import org.apache.hadoop.hbase.regionserver.FlushLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.InternalScanner; @@ -68,6 +63,7 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; +import org.apache.hadoop.hbase.replication.SyncReplicationState; import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; @@ -75,10 +71,15 @@ import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.access.Permission.Action; import org.apache.hadoop.hbase.security.access.UserPermission; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AccessControlProtos; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.security.AccessControlException; +import org.apache.hbase.thirdparty.com.google.protobuf.Message; +import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback; +import org.apache.hbase.thirdparty.com.google.protobuf.RpcController; +import org.apache.hbase.thirdparty.com.google.protobuf.Service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import tech.stackable.hbase.opa.OpType; @@ -179,7 +180,9 @@ public void preDeleteNamespace( @Override public void preModifyNamespace( - ObserverContext ctx, NamespaceDescriptor ns) + ObserverContext ctx, + NamespaceDescriptor currentDesc, + NamespaceDescriptor newNesc) throws IOException { final User user = getActiveUser(ctx); LOG.debug("preModifyNamespace: user [{}]", user); @@ -262,7 +265,7 @@ public void preDisableTable( @Override public void preGetOp( - final ObserverContext ctx, + final ObserverContext ctx, final Get get, final List result) throws IOException { @@ -279,7 +282,9 @@ public void preGetOp( @Override public boolean preExists( - final ObserverContext ctx, final Get get, final boolean exists) + final ObserverContext ctx, + final Get get, + final boolean exists) throws IOException { final User user = getActiveUser(ctx); TableName tableName = ctx.getEnvironment().getRegionInfo().getTable(); @@ -295,7 +300,8 @@ public boolean preExists( @Override public void preScannerOpen( - final ObserverContext ctx, final Scan scan) throws IOException { + final ObserverContext ctx, final Scan scan) + throws IOException { final User user = getActiveUser(ctx); TableName tableName = ctx.getEnvironment().getRegionInfo().getTable(); // All users need read access to hbase:meta table. @@ -309,7 +315,7 @@ public void preScannerOpen( @Override public RegionScanner postScannerOpen( - final ObserverContext ctx, + final ObserverContext ctx, final Scan scan, final RegionScanner s) throws IOException { @@ -326,7 +332,7 @@ public RegionScanner postScannerOpen( @Override public boolean preScannerNext( - final ObserverContext ctx, + final ObserverContext ctx, final InternalScanner s, final List result, final int limit, @@ -342,7 +348,7 @@ public boolean preScannerNext( @Override public void preScannerClose( - final ObserverContext ctx, final InternalScanner s) + final ObserverContext ctx, final InternalScanner s) throws IOException { final User user = getActiveUser(ctx); TableName tableName = ctx.getEnvironment().getRegionInfo().getTable(); @@ -353,7 +359,7 @@ public void preScannerClose( @Override public void postScannerClose( - final ObserverContext ctx, final InternalScanner s) + final ObserverContext ctx, final InternalScanner s) throws IOException { final User user = getActiveUser(ctx); TableName tableName = ctx.getEnvironment().getRegionInfo().getTable(); @@ -376,7 +382,7 @@ private void requireScannerOwner(InternalScanner s) throws AccessDeniedException @Override public void prePut( - final ObserverContext ctx, + final ObserverContext ctx, final Put put, final WALEdit edit, final Durability durability) @@ -390,7 +396,7 @@ public void prePut( @Override public void preDelete( - final ObserverContext ctx, + final ObserverContext ctx, final Delete delete, final WALEdit edit, final Durability durability) @@ -404,7 +410,7 @@ public void preDelete( @Override public void postDelete( - final ObserverContext ctx, + final ObserverContext ctx, final Delete delete, final WALEdit edit, final Durability durability) { @@ -412,7 +418,8 @@ public void postDelete( } @Override - public Result preAppend(ObserverContext ctx, Append append) + public Result preAppend( + ObserverContext ctx, Append append) throws IOException { final User user = getActiveUser(ctx); TableName tableName = ctx.getEnvironment().getRegionInfo().getTable(); @@ -426,7 +433,7 @@ public Result preAppend(ObserverContext ctx, Appen @Override public void preBatchMutate( - ObserverContext ctx, + ObserverContext ctx, MiniBatchOperationInProgress miniBatchOp) throws IOException { final User user = getActiveUser(ctx); @@ -441,7 +448,8 @@ public void preBatchMutate( } @Override - public void preOpen(ObserverContext ctx) throws IOException { + public void preOpen(ObserverContext ctx) + throws IOException { final User user = getActiveUser(ctx); final Region region = ctx.getEnvironment().getRegion(); if (region == null) { @@ -454,7 +462,7 @@ public void preOpen(ObserverContext ctx) throws IO } @Override - public void postOpen(ObserverContext ctx) { + public void postOpen(ObserverContext ctx) { // not needed as the ACL table is not used } @@ -469,14 +477,14 @@ public void preTableFlush( @Override public void preFlush( - ObserverContext ctx, FlushLifeCycleTracker tracker) + ObserverContext ctx, FlushLifeCycleTracker tracker) throws IOException { // Internal storage engine flush — not a user-initiated operation, no authorization needed. } @Override public InternalScanner preCompact( - ObserverContext ctx, + ObserverContext ctx, Store store, InternalScanner scanner, ScanType scanType, @@ -564,7 +572,7 @@ public void postGetTableNames( @Override public boolean preCheckAndPut( - final ObserverContext ctx, + final ObserverContext ctx, final byte[] row, final byte[] family, final byte[] qualifier, @@ -583,7 +591,7 @@ public boolean preCheckAndPut( @Override public boolean preCheckAndPutAfterRowLock( - final ObserverContext ctx, + final ObserverContext ctx, final byte[] row, final byte[] family, final byte[] qualifier, @@ -603,7 +611,7 @@ public boolean preCheckAndPutAfterRowLock( @Override public boolean preCheckAndDelete( - final ObserverContext ctx, + final ObserverContext ctx, final byte[] row, final byte[] family, final byte[] qualifier, @@ -630,7 +638,7 @@ public boolean preCheckAndDelete( @Override public boolean preCheckAndDeleteAfterRowLock( - final ObserverContext ctx, + final ObserverContext ctx, final byte[] row, final byte[] family, final byte[] qualifier, @@ -660,7 +668,7 @@ public boolean preCheckAndDeleteAfterRowLock( @Override public boolean preCheckAndPut( - final ObserverContext ctx, + final ObserverContext ctx, final byte[] row, final Filter filter, final Put put, @@ -676,7 +684,7 @@ public boolean preCheckAndPut( @Override public boolean preCheckAndPutAfterRowLock( - final ObserverContext ctx, + final ObserverContext ctx, final byte[] row, final Filter filter, final Put put, @@ -693,7 +701,7 @@ public boolean preCheckAndPutAfterRowLock( @Override public boolean preCheckAndDelete( - final ObserverContext ctx, + final ObserverContext ctx, final byte[] row, final Filter filter, final Delete delete, @@ -717,7 +725,7 @@ public boolean preCheckAndDelete( @Override public boolean preCheckAndDeleteAfterRowLock( - final ObserverContext ctx, + final ObserverContext ctx, final byte[] row, final Filter filter, final Delete delete, @@ -744,7 +752,7 @@ public boolean preCheckAndDeleteAfterRowLock( @Override public CheckAndMutateResult preCheckAndMutate( - ObserverContext ctx, + ObserverContext ctx, CheckAndMutate checkAndMutate, CheckAndMutateResult result) throws IOException { @@ -760,7 +768,7 @@ public CheckAndMutateResult preCheckAndMutate( @Override public CheckAndMutateResult preCheckAndMutateAfterRowLock( - ObserverContext ctx, + ObserverContext ctx, CheckAndMutate checkAndMutate, CheckAndMutateResult result) throws IOException { @@ -815,7 +823,8 @@ public TableDescriptor preModifyTable( public void postModifyTable( ObserverContext ctx, TableName tableName, - final TableDescriptor htd) + final TableDescriptor currentDesc, + final TableDescriptor newDesc) throws IOException { final User user = getActiveUser(ctx); LOG.trace("postModifyTable: user [{}] on table [{}]", user, tableName); @@ -823,7 +832,7 @@ public void postModifyTable( @Override public Result preIncrement( - final ObserverContext ctx, final Increment increment) + final ObserverContext ctx, final Increment increment) throws IOException { final User user = getActiveUser(ctx); TableName tableName = ctx.getEnvironment().getRegionInfo().getTable(); @@ -840,7 +849,7 @@ public Result preIncrement( @Override public List> postIncrementBeforeWAL( - ObserverContext ctx, + ObserverContext ctx, Mutation mutation, List> cellPairs) { // we have no ACL table so return as per the similar case in the default controller @@ -849,7 +858,7 @@ public List> postIncrementBeforeWAL( @Override public List> postAppendBeforeWAL( - ObserverContext ctx, + ObserverContext ctx, Mutation mutation, List> cellPairs) { // we have no ACL table so return as per the similar case in the default controller @@ -1046,7 +1055,8 @@ public void preSplitRegion( @Override public void preBulkLoadHFile( - ObserverContext ctx, List> familyPaths) + ObserverContext ctx, + List> familyPaths) throws IOException { final User user = getActiveUser(ctx); final var tableName = ctx.getEnvironment().getRegion().getTableDescriptor().getTableName(); @@ -1328,7 +1338,7 @@ public void preStopRegionServer(ObserverContext ctx, + ObserverContext ctx, Service service, String methodName, Message request) @@ -1360,7 +1370,7 @@ public Message preEndpointInvocation( @Override public void postEndpointInvocation( - ObserverContext ctx, + ObserverContext ctx, Service service, String methodName, Message request, @@ -1541,6 +1551,19 @@ public void preListReplicationPeers( ctx, NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR, "listReplicationPeers", Action.ADMIN); } + @Override + public void preTransitReplicationPeerSyncReplicationState( + final ObserverContext ctx, + String peerId, + SyncReplicationState state) + throws IOException { + requirePermission( + ctx, + NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR, + "transitSyncReplicationPeerState", + Action.ADMIN); + } + @Override public void preExecuteProcedures(ObserverContext ctx) { // Not implemented: reference AC uses checkSystemOrSuperUser, a superuser mechanism diff --git a/src/test/java/tech/stackable/hbase/OpaFixtureCapture.java b/src/test/java/tech/stackable/hbase/OpaFixtureCapture.java new file mode 100644 index 0000000..233ed25 --- /dev/null +++ b/src/test/java/tech/stackable/hbase/OpaFixtureCapture.java @@ -0,0 +1,19 @@ +package tech.stackable.hbase; + +import com.github.tomakehurst.wiremock.extension.Parameters; +import com.github.tomakehurst.wiremock.extension.ServeEventListener; +import com.github.tomakehurst.wiremock.stubbing.ServeEvent; + +/** Passes every OPA request/response pair WireMock serves to {@link OpaFixtureWriter}. */ +public class OpaFixtureCapture implements ServeEventListener { + @Override + public String getName() { + return "opa-fixture-capture"; + } + + @Override + public void afterComplete(ServeEvent serveEvent, Parameters parameters) { + OpaFixtureWriter.capture( + serveEvent.getRequest().getBodyAsString(), serveEvent.getResponse().getBodyAsString()); + } +} diff --git a/src/test/java/tech/stackable/hbase/OpaFixtureWriter.java b/src/test/java/tech/stackable/hbase/OpaFixtureWriter.java index 795cae6..8abf2b1 100644 --- a/src/test/java/tech/stackable/hbase/OpaFixtureWriter.java +++ b/src/test/java/tech/stackable/hbase/OpaFixtureWriter.java @@ -108,4 +108,9 @@ private static String buildFixturesJson() { String denied = String.join(",", deniedFixtures); return "{\"fixtures\":{\"allowed\":[" + allowed + "],\"denied\":[" + denied + "]}}"; } + + /** Called by {@link OpaFixtureCapture} for each served request. Thread-safe. */ + public static synchronized void capture(String requestBody, String responseBody) { + captured.add(new String[] {requestBody, responseBody}); + } } diff --git a/src/test/java/tech/stackable/hbase/TestCoprocessorInterfaceCoverage.java b/src/test/java/tech/stackable/hbase/TestCoprocessorInterfaceCoverage.java index 7ee2f32..1b503a2 100644 --- a/src/test/java/tech/stackable/hbase/TestCoprocessorInterfaceCoverage.java +++ b/src/test/java/tech/stackable/hbase/TestCoprocessorInterfaceCoverage.java @@ -1,6 +1,6 @@ package tech.stackable.hbase; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.lang.reflect.Method; import java.lang.reflect.Modifier; @@ -14,7 +14,7 @@ import org.apache.hadoop.hbase.coprocessor.MasterObserver; import org.apache.hadoop.hbase.coprocessor.RegionObserver; import org.apache.hadoop.hbase.coprocessor.RegionServerObserver; -import org.junit.Test; +import org.junit.jupiter.api.Test; /** * Verifies that every method in the coprocessor observer interfaces is either explicitly overridden @@ -59,13 +59,6 @@ public class TestCoprocessorInterfaceCoverage { "preDelete(ObserverContext, Delete, WALEdit)", "preIncrement(ObserverContext, Increment, WALEdit)", "prePut(ObserverContext, Put, WALEdit)", - // Old single-descriptor overload; we override the 4-arg (old + new) variant. - "preModifyTable(ObserverContext, TableName, TableDescriptor)", - // Old 2-descriptor namespace overload; we override the 2-arg (new descriptor only) - // variant. - "preModifyNamespace(ObserverContext, NamespaceDescriptor, NamespaceDescriptor)", - // Old 3-arg unassign with boolean; we override the 2-arg variant. - "preUnassign(ObserverContext, RegionInfo, boolean)", // --- after-row-lock variants where we check at the pre-lock level --- // HBase calls the pre-lock hook before acquiring the row lock and the after-lock hook @@ -135,6 +128,8 @@ public class TestCoprocessorInterfaceCoverage { "preGetRSGroupInfoOfServer(ObserverContext, Address)", "preGetRSGroupInfoOfTable(ObserverContext, TableName)", "preListRSGroups(ObserverContext)", + "preListTablesInRSGroup(ObserverContext, String)", + "preGetConfiguredNamespacesAndTablesInRSGroup(ObserverContext, String)", "preMoveServers(ObserverContext, Set, String)", "preMoveServersAndTables(ObserverContext, Set, Set, String)", "preMoveTables(ObserverContext, Set, String)", @@ -179,10 +174,10 @@ public void testAllObserverMethodsAreExplicitlyOverridden() { .collect(Collectors.toList()); assertTrue( + unhandled.isEmpty(), "Observer interface pre-hooks found that are neither overridden nor in the exclusion list" + " — review each and either implement it or add it to EXCLUDED with a justification:\n" - + String.join("\n", unhandled), - unhandled.isEmpty()); + + String.join("\n", unhandled)); } private static String signature(Method m) { diff --git a/src/test/java/tech/stackable/hbase/TestDefaultAccessController.java b/src/test/java/tech/stackable/hbase/TestDefaultAccessController.java index 4b90852..304c72d 100644 --- a/src/test/java/tech/stackable/hbase/TestDefaultAccessController.java +++ b/src/test/java/tech/stackable/hbase/TestDefaultAccessController.java @@ -13,12 +13,11 @@ import static org.apache.hadoop.hbase.security.access.SecureTestUtil.revokeFromTableUsingAccessControlClient; import static org.apache.hadoop.hbase.security.access.SecureTestUtil.verifyAllowed; import static org.apache.hadoop.hbase.security.access.SecureTestUtil.verifyDenied; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; -import com.google.protobuf.BlockingRpcChannel; import java.util.Arrays; import java.util.Collection; import org.apache.hadoop.hbase.HConstants; @@ -27,7 +26,6 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos; import org.apache.hadoop.hbase.security.Superusers; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.AccessControlClient; @@ -35,16 +33,18 @@ import org.apache.hadoop.hbase.security.access.AccessController; import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.access.PermissionStorage; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AccessControlProtos; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Ignore; -import org.junit.Test; +import org.apache.hbase.thirdparty.com.google.protobuf.BlockingRpcChannel; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; /** * This class copies selected test cases from the HBase code so that it is easier to step-through - * the default co-processor code to see how it is used. It can be removed entirely once the + * the default coprocessor code to see how it is used. It can be removed entirely once the * implementation has stabilised. */ -@Ignore +@Disabled public class TestDefaultAccessController extends TestUtils { @Test diff --git a/src/test/java/tech/stackable/hbase/TestOpenPolicyAgentAccessController.java b/src/test/java/tech/stackable/hbase/TestOpenPolicyAgentAccessController.java index 6b62cdd..23c1ed5 100644 --- a/src/test/java/tech/stackable/hbase/TestOpenPolicyAgentAccessController.java +++ b/src/test/java/tech/stackable/hbase/TestOpenPolicyAgentAccessController.java @@ -4,15 +4,15 @@ import static com.github.tomakehurst.wiremock.client.WireMock.ok; import static com.github.tomakehurst.wiremock.client.WireMock.post; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; import static org.apache.hadoop.hbase.security.access.SecureTestUtil.createTable; import static org.apache.hadoop.hbase.security.access.SecureTestUtil.deleteTable; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.fail; import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.github.tomakehurst.wiremock.junit5.WireMockExtension; import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.BalanceRequest; @@ -28,35 +28,40 @@ import org.apache.hadoop.hbase.coprocessor.ObserverContextImpl; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.quotas.GlobalQuotaSettings; +import org.apache.hadoop.hbase.replication.SyncReplicationState; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.SecureTestUtil; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.security.AccessControlException; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; public class TestOpenPolicyAgentAccessController extends TestUtils { public static final String OPA_URL = "http://localhost:8089"; - @ClassRule public static WireMockRule wireMockRule = new WireMockRule(8089); + @RegisterExtension + static WireMockExtension wireMockExtension = + WireMockExtension.newInstance() + .options(wireMockConfig().port(8089).extensions(new OpaFixtureCapture())) + .configureStaticDsl(true) + .build(); - @BeforeClass + @BeforeAll public static void setUpClass() throws Exception { - wireMockRule.addMockServiceRequestListener(OpaFixtureWriter::capture); stubFor(post("/").willReturn(ok().withBody("{\"result\": \"true\"}"))); setup(OpenPolicyAgentAccessController.class, false, OPA_URL); } - @Before + @BeforeEach public void resetStubs() { WireMock.reset(); stubFor(post("/").willReturn(ok().withBody("{\"result\": \"true\"}"))); } - @AfterClass + @AfterAll public static void tearDownClass() throws Exception { tearDown(); } @@ -79,8 +84,8 @@ private OpenPolicyAgentAccessController getOpaController() { public void testCreateAndPut() throws Exception { LOG.info("testCreateAndPut - start"); - HTableDescriptor htd = getHTableDescriptor(); - createTable(TEST_UTIL, TEST_UTIL.getAdmin(), htd, new byte[][] {Bytes.toBytes("s")}); + TableDescriptor td = getTableDescriptor(); + createTable(TEST_UTIL, TEST_UTIL.getAdmin(), td, new byte[][] {Bytes.toBytes("s")}); List puts = new ArrayList<>(100); for (int i = 0; i < 100; i++) { @@ -88,7 +93,7 @@ public void testCreateAndPut() throws Exception { p.addColumn(TEST_FAMILY, Bytes.toBytes("myCol"), Bytes.toBytes("info " + i)); puts.add(p); } - Table table = TEST_UTIL.getConnection().getTable(htd.getTableName()); + Table table = TEST_UTIL.getConnection().getTable(td.getTableName()); table.put(puts); deleteTable(TEST_UTIL, TEST_TABLE); @@ -100,8 +105,8 @@ public void testDeniedCreate() throws Exception { LOG.info("testDeniedCreate - start"); try { stubFor(post("/").willReturn(ok().withBody("{\"result\": \"false\"}"))); - HTableDescriptor htd = getHTableDescriptor(); - createTable(TEST_UTIL, TEST_UTIL.getAdmin(), htd, new byte[][] {Bytes.toBytes("s")}); + TableDescriptor td = getTableDescriptor(); + createTable(TEST_UTIL, TEST_UTIL.getAdmin(), td, new byte[][] {Bytes.toBytes("s")}); fail("AccessControlException should have been thrown"); } catch (AccessControlException e) { logOk(e); @@ -114,7 +119,7 @@ public void testDeniedCreateByUser() throws Exception { User userDenied = User.createUserForTesting(conf, "cannotCreateTables", new String[0]); SecureTestUtil.AccessTestAction createTable = () -> { - getOpaController().preCreateTable(ctx(), getHTableDescriptor(), null); + getOpaController().preCreateTable(ctx(), getTableDescriptor(), null); return null; }; stubFor( @@ -173,7 +178,7 @@ public void testPreModifyNamespace() throws Exception { NamespaceDescriptor nsd = NamespaceDescriptor.create("default").build(); assertAllowedThenDenied( () -> { - getOpaController().preModifyNamespace(ctx(), nsd); + getOpaController().preModifyNamespace(ctx(), nsd, nsd); return null; }); } @@ -227,7 +232,7 @@ public void testPreTruncateTable() throws Exception { @Test public void testPreModifyTable() throws Exception { - TableDescriptor td = getHTableDescriptor(); + TableDescriptor td = getTableDescriptor(); assertAllowedThenDenied( () -> { getOpaController().preModifyTable(ctx(), TEST_TABLE, td, td); @@ -350,7 +355,7 @@ public void testPreModifyTableStoreFileTracker() throws Exception { @Test public void testPreSnapshot() throws Exception { SnapshotDescription snap = new SnapshotDescription("snap", TEST_TABLE); - TableDescriptor td = getHTableDescriptor(); + TableDescriptor td = getTableDescriptor(); assertAllowedThenDenied( () -> { getOpaController().preSnapshot(ctx(), snap, td); @@ -371,7 +376,7 @@ public void testPreListSnapshot() throws Exception { @Test public void testPreCloneSnapshot() throws Exception { SnapshotDescription snap = new SnapshotDescription("snap", TEST_TABLE); - TableDescriptor td = getHTableDescriptor(); + TableDescriptor td = getTableDescriptor(); assertAllowedThenDenied( () -> { getOpaController().preCloneSnapshot(ctx(), snap, td); @@ -382,7 +387,7 @@ public void testPreCloneSnapshot() throws Exception { @Test public void testPreRestoreSnapshot() throws Exception { SnapshotDescription snap = new SnapshotDescription("snap", TEST_TABLE); - TableDescriptor td = getHTableDescriptor(); + TableDescriptor td = getTableDescriptor(); assertAllowedThenDenied( () -> { getOpaController().preRestoreSnapshot(ctx(), snap, td); @@ -676,6 +681,17 @@ public void testPreListReplicationPeers() throws Exception { }); } + @Test + public void testPreTransitReplicationPeerSyncReplicationState() throws Exception { + assertAllowedThenDenied( + () -> { + getOpaController() + .preTransitReplicationPeerSyncReplicationState( + ctx(), "peer1", SyncReplicationState.ACTIVE); + return null; + }); + } + // --- throttle hooks --- @Test diff --git a/src/test/java/tech/stackable/hbase/TestOpenPolicyAgentAccessControllerRegion.java b/src/test/java/tech/stackable/hbase/TestOpenPolicyAgentAccessControllerRegion.java index 01f3690..13270c4 100644 --- a/src/test/java/tech/stackable/hbase/TestOpenPolicyAgentAccessControllerRegion.java +++ b/src/test/java/tech/stackable/hbase/TestOpenPolicyAgentAccessControllerRegion.java @@ -3,11 +3,12 @@ import static com.github.tomakehurst.wiremock.client.WireMock.ok; import static com.github.tomakehurst.wiremock.client.WireMock.post; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; import static org.apache.hadoop.hbase.security.access.SecureTestUtil.createTable; import static org.apache.hadoop.hbase.security.access.SecureTestUtil.deleteTable; import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.github.tomakehurst.wiremock.junit5.WireMockExtension; import java.util.Collections; import org.apache.hadoop.hbase.CompareOperator; import org.apache.hadoop.hbase.Coprocessor; @@ -32,11 +33,11 @@ import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost; import org.apache.hadoop.hbase.regionserver.ScanType; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; public class TestOpenPolicyAgentAccessControllerRegion extends TestUtils { public static final String OPA_URL = "http://localhost:8089"; @@ -45,16 +46,20 @@ public class TestOpenPolicyAgentAccessControllerRegion extends TestUtils { private static RegionCoprocessorEnvironment REGION_CP_ENV; private static RegionServerCoprocessorEnvironment RS_CP_ENV; - @ClassRule public static WireMockRule wireMockRule = new WireMockRule(8089); + @RegisterExtension + static WireMockExtension wireMockExtension = + WireMockExtension.newInstance() + .options(wireMockConfig().port(8089).extensions(new OpaFixtureCapture())) + .configureStaticDsl(true) + .build(); - @BeforeClass + @BeforeAll public static void setUpClass() throws Exception { - wireMockRule.addMockServiceRequestListener(OpaFixtureWriter::capture); stubFor(post("/").willReturn(ok().withBody("{\"result\": \"true\"}"))); setup(OpenPolicyAgentAccessController.class, false, OPA_URL); createTable( - TEST_UTIL, TEST_UTIL.getAdmin(), getHTableDescriptor(), new byte[][] {Bytes.toBytes("s")}); + TEST_UTIL, TEST_UTIL.getAdmin(), getTableDescriptor(), new byte[][] {Bytes.toBytes("s")}); HRegion region = TEST_UTIL.getHBaseCluster().getRegions(TEST_TABLE).get(0); RegionCoprocessorHost rcpHost = region.getCoprocessorHost(); @@ -73,13 +78,13 @@ public static void setUpClass() throws Exception { rsCpHost.createEnvironment(rsController, Coprocessor.PRIORITY_HIGHEST, 1, conf); } - @Before + @BeforeEach public void resetStubs() { WireMock.reset(); stubFor(post("/").willReturn(ok().withBody("{\"result\": \"true\"}"))); } - @AfterClass + @AfterAll public static void tearDownClass() throws Exception { deleteTable(TEST_UTIL, TEST_TABLE); tearDown(); diff --git a/src/test/java/tech/stackable/hbase/TestOpenPolicyAgentAccessControllerVariants.java b/src/test/java/tech/stackable/hbase/TestOpenPolicyAgentAccessControllerVariants.java index 0911e6c..0adc6f3 100644 --- a/src/test/java/tech/stackable/hbase/TestOpenPolicyAgentAccessControllerVariants.java +++ b/src/test/java/tech/stackable/hbase/TestOpenPolicyAgentAccessControllerVariants.java @@ -4,19 +4,20 @@ import static com.github.tomakehurst.wiremock.client.WireMock.ok; import static com.github.tomakehurst.wiremock.client.WireMock.post; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; -import static org.junit.Assert.assertEquals; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.junit.jupiter.api.Assertions.assertEquals; -import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.github.tomakehurst.wiremock.junit5.WireMockExtension; import java.util.Optional; -import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.coprocessor.ObserverContextImpl; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.SecureTestUtil; import org.apache.hadoop.security.AccessControlException; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** * Tests for non-default coprocessor configurations (dryRun, cache). Each test manages its own @@ -25,14 +26,12 @@ public class TestOpenPolicyAgentAccessControllerVariants extends TestUtils { public static final String OPA_URL = "http://localhost:8089"; - // @Rule (not @ClassRule) because each test starts and tears down its own mini-cluster - // (setup/tearDown are called inside the test body, not in @BeforeClass/@AfterClass). - @Rule public WireMockRule wireMockRule = new WireMockRule(8089); - - @Before - public void registerOpaListener() { - wireMockRule.addMockServiceRequestListener(OpaFixtureWriter::capture); - } + @RegisterExtension + WireMockExtension wireMockExtension = + WireMockExtension.newInstance() + .options(wireMockConfig().port(8089).extensions(new OpaFixtureCapture())) + .configureStaticDsl(true) + .build(); @Test public void testDryRun() throws Exception { @@ -43,9 +42,8 @@ public void testDryRun() throws Exception { SecureTestUtil.AccessTestAction createTable = () -> { - HTableDescriptor htd = getHTableDescriptor(); - getOpaController() - .preCreateTable(ObserverContextImpl.createAndPrepare(CP_ENV), htd, null); + TableDescriptor td = getTableDescriptor(); + getOpaController().preCreateTable(ObserverContextImpl.createAndPrepare(CP_ENV), td, null); return null; }; @@ -61,8 +59,6 @@ public void testDryRun() throws Exception { } catch (AccessControlException e) { throw new AssertionError("AccessControlException should not have been thrown", e); } - - tearDown(); } @Test @@ -74,9 +70,8 @@ public void testUseCache() throws Exception { SecureTestUtil.AccessTestAction createTable = () -> { - HTableDescriptor htd = getHTableDescriptor(); - getOpaController() - .preCreateTable(ObserverContextImpl.createAndPrepare(CP_ENV), htd, null); + TableDescriptor td = getTableDescriptor(); + getOpaController().preCreateTable(ObserverContextImpl.createAndPrepare(CP_ENV), td, null); return null; }; @@ -87,7 +82,12 @@ public void testUseCache() throws Exception { } assertEquals(Optional.of(1L), getOpaController().getAclCacheSize()); + } + // Each test starts its own mini-cluster in its body; shut it down here so that a failure in + // setup() or in the test itself does not leave the cluster running for the next test class. + @AfterEach + public void shutDownMiniCluster() throws Exception { tearDown(); } diff --git a/src/test/java/tech/stackable/hbase/TestUtils.java b/src/test/java/tech/stackable/hbase/TestUtils.java index 8011c1c..fa1f632 100644 --- a/src/test/java/tech/stackable/hbase/TestUtils.java +++ b/src/test/java/tech/stackable/hbase/TestUtils.java @@ -6,7 +6,9 @@ import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static org.apache.hadoop.hbase.AuthUtil.toGroupEntry; import static org.apache.hadoop.hbase.security.access.SecureTestUtil.*; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static tech.stackable.hbase.OpenPolicyAgentAccessController.OPA_POLICY_CACHE; import static tech.stackable.hbase.OpenPolicyAgentAccessController.OPA_POLICY_DRYRUN; @@ -16,8 +18,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; @@ -37,7 +42,7 @@ public class TestUtils { protected static final Logger LOG = LoggerFactory.getLogger(TestUtils.class); - protected static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + protected static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); protected static Configuration conf; protected static Connection systemUserConnection; @@ -174,12 +179,13 @@ protected static void setup( } protected static void setUpTables() throws Exception { - HTableDescriptor htd = new HTableDescriptor(TEST_TABLE); - HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY); - hcd.setMaxVersions(100); - htd.addFamily(hcd); - htd.setOwner(USER_OWNER); - createTable(TEST_UTIL, TEST_UTIL.getAdmin(), htd, new byte[][] {Bytes.toBytes("s")}); + TableDescriptor td = + TableDescriptorBuilder.newBuilder(TEST_TABLE) + .setColumnFamily( + ColumnFamilyDescriptorBuilder.newBuilder(TEST_FAMILY).setMaxVersions(100).build()) + .build(); + + createTable(TEST_UTIL, TEST_UTIL.getAdmin(), td, new byte[][] {Bytes.toBytes("s")}); HRegion region = TEST_UTIL.getHBaseCluster().getRegions(TEST_TABLE).get(0); RegionCoprocessorHost rcpHost = region.getCoprocessorHost(); @@ -580,20 +586,21 @@ protected void verifyGetUserPermissionResult( } protected void createTestTable(TableName tname, byte[] cf) throws Exception { - HTableDescriptor htd = new HTableDescriptor(tname); - HColumnDescriptor hcd = new HColumnDescriptor(cf); - hcd.setMaxVersions(100); - htd.addFamily(hcd); - htd.setOwner(USER_OWNER); - createTable(TEST_UTIL, TEST_UTIL.getAdmin(), htd, new byte[][] {Bytes.toBytes("s")}); + TableDescriptor td = + TableDescriptorBuilder.newBuilder(tname) + .setColumnFamily( + ColumnFamilyDescriptorBuilder.newBuilder(cf).setMaxVersions(100).build()) + .build(); + + createTable(TEST_UTIL, TEST_UTIL.getAdmin(), td, new byte[][] {Bytes.toBytes("s")}); } - protected static HTableDescriptor getHTableDescriptor() { - HTableDescriptor htd = new HTableDescriptor(TEST_TABLE); - HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY); - hcd.setMaxVersions(100); - htd.addFamily(hcd); - htd.setOwner(USER_OWNER); - return htd; + protected static TableDescriptor getTableDescriptor() { + TableDescriptor td = + TableDescriptorBuilder.newBuilder(TEST_TABLE) + .setColumnFamily( + ColumnFamilyDescriptorBuilder.newBuilder(TEST_FAMILY).setMaxVersions(100).build()) + .build(); + return td; } } diff --git a/test/Dockerfile b/test/Dockerfile deleted file mode 100644 index 7a31b29..0000000 --- a/test/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -# From root folder.... -# docker build -f ./test/Dockerfile -t docker.stackable.tech/sandbox/hbase:2.6.0-stackable0.0.0-dev . -FROM docker.stackable.tech/sandbox/hbase:2.6.0-stackable0.0.0-dev - -COPY --chown=stackable:stackable target/hbase-opa-authorizer-0.1.0.jar /stackable/hbase/lib/hbase-opa-authorizer-0.1.0.jar