2024-11-25 19:29:22,588 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116
2024-11-25 19:29:22,605 main DEBUG Took 0.014918 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-25 19:29:22,606 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-25 19:29:22,606 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-25 19:29:22,609 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-25 19:29:22,611 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 19:29:22,624 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-25 19:29:22,641 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 19:29:22,643 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 19:29:22,643 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 19:29:22,644 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 19:29:22,645 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 19:29:22,645 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 19:29:22,646 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 19:29:22,647 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 19:29:22,647 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 19:29:22,647 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 19:29:22,649 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 19:29:22,649 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 19:29:22,650 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 19:29:22,651 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 19:29:22,651 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 19:29:22,652 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 19:29:22,653 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 19:29:22,653 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 19:29:22,654 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 19:29:22,654 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 19:29:22,654 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 19:29:22,655 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 19:29:22,655 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 19:29:22,656 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-25 19:29:22,656 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 19:29:22,657 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-25 19:29:22,658 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-25 19:29:22,660 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-25 19:29:22,662 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-25 19:29:22,663 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-25 19:29:22,665 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-25 19:29:22,665 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-25 19:29:22,675 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-25 19:29:22,679 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-25 19:29:22,681 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-25 19:29:22,682 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-25 19:29:22,682 main DEBUG createAppenders(={Console})
2024-11-25 19:29:22,683 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 initialized
2024-11-25 19:29:22,684 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116
2024-11-25 19:29:22,684 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 OK.
2024-11-25 19:29:22,685 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-25 19:29:22,685 main DEBUG OutputStream closed
2024-11-25 19:29:22,686 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-25 19:29:22,686 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-25 19:29:22,687 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5d0a1059 OK
2024-11-25 19:29:22,775 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-25 19:29:22,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-25 19:29:22,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-25 19:29:22,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-25 19:29:22,780 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-25 19:29:22,780 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-25 19:29:22,780 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-25 19:29:22,781 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-25 19:29:22,781 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-25 19:29:22,781 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-25 19:29:22,782 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-25 19:29:22,782 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-25 19:29:22,782 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-25 19:29:22,783 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-25 19:29:22,783 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-25 19:29:22,783 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-25 19:29:22,784 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-25 19:29:22,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-25 19:29:22,787 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-25 19:29:22,787 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5812f68b) with optional ClassLoader: null
2024-11-25 19:29:22,787 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-25 19:29:22,788 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5812f68b] started OK.
2024-11-25T19:29:22,986 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3da10d8b-f9a1-f0c1-4e54-662e999d2f81
2024-11-25 19:29:22,989 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-25 19:29:22,989 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-25T19:29:23,001 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.security.TestSecureIPC timeout: 52 mins
2024-11-25T19:29:23,130 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=13, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=6830
2024-11-25T19:29:23,219 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-25T19:29:23,284 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:23,287 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:23,343 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562963340,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:23,980 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:24,070 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-25T19:29:24,114 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-25T19:29:24,126 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-25T19:29:24,151 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44691
2024-11-25T19:29:24,158 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:24,158 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:24,158 INFO  [Listener,port=44691 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44691: starting
2024-11-25T19:29:24,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f812377, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:24,286 DEBUG [Thread-6 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:24,333 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44691
2024-11-25T19:29:24,342 DEBUG [Thread-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,394 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562964392,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,403 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:24,406 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,410 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:24,410 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:24,423 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:24,425 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:24,427 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:24,427 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:24,428 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:24,429 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:24,429 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:24,430 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:24,431 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:24,431 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T19:29:24,431 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:24,432 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,432 DEBUG [Thread-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:24,432 DEBUG [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:24,432 INFO  [Reader=1,bindAddress=localhost,port=44691 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:24,450 INFO  [Reader=1,bindAddress=localhost,port=44691 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42056, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:24,451 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93
2024-11-25T19:29:24,456 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:240
2024-11-25T19:29:24,461 DEBUG [BRPC Connection (1039819159) to localhost:44691 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-25T19:29:24,487 DEBUG [BRPC Connection (1039819159) to localhost:44691 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:226
2024-11-25T19:29:24,491 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2295
2024-11-25T19:29:24,493 DEBUG [BRPC Connection (1039819159) to localhost:44691 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-25T19:29:24,496 DEBUG [BRPC Connection (1039819159) to localhost:44691 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2281
2024-11-25T19:29:24,510 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117
2024-11-25T19:29:24,511 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18145
2024-11-25T19:29:24,516 DEBUG [BRPC Connection (1039819159) to localhost:44691 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-25T19:29:24,532 DEBUG [BRPC Connection (1039819159) to localhost:44691 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22248
2024-11-25T19:29:24,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:24,538 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44691
2024-11-25T19:29:24,539 INFO  [Listener,port=44691 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44691: stopping
2024-11-25T19:29:24,539 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:24,539 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:24,545 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 13)
Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: pool-3-thread-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=7285 (was 6830) - AvailableMemoryMB LEAK? -
2024-11-25T19:29:24,551 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=7381
2024-11-25T19:29:24,563 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:24,563 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:24,570 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562964569,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:24,586 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:24,588 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=37439
2024-11-25T19:29:24,588 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:24,589 INFO  [Listener,port=37439 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=37439: starting
2024-11-25T19:29:24,589 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:24,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5920f344, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-25T19:29:24,592 DEBUG [Thread-11 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:24,592 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37439
2024-11-25T19:29:24,594 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,605 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562964603,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,609 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:24,610 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T19:29:24,611 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:37439
org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-25T19:29:24,617 WARN  [Thread-11 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732562964572
2024-11-25T19:29:24,709 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,713 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:24,714 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T19:29:24,715 DEBUG [Thread-11 {}] ipc.FailedServers(52): Added failed server with address localhost:37439 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:24,716 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 37439
2024-11-25T19:29:24,717 INFO  [Listener,port=37439 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=37439: stopping
2024-11-25T19:29:24,717 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:24,718 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:24,723 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=22 (was 19)
Potentially hanging thread: RpcServer idle connection scanner for port 37439
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/java.lang.ref.Reference.refersTo(Reference.java:366)
	java.base@17.0.11/java.lang.ThreadLocal$ThreadLocalMap.set(ThreadLocal.java:487)
	java.base@17.0.11/java.lang.ThreadLocal.set(ThreadLocal.java:222)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:127)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
	app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552)
	app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
	app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
	app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)

Potentially hanging thread: Listener,port=37439
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:255)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=7338 (was 7381)
2024-11-25T19:29:24,729 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=7337
2024-11-25T19:29:24,741 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:24,742 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:24,749 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562964748,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:24,755 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:24,757 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33041
2024-11-25T19:29:24,758 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:24,758 INFO  [Listener,port=33041 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33041: starting
2024-11-25T19:29:24,758 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:24,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@100261c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:24,760 DEBUG [Thread-14 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:24,761 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33041
2024-11-25T19:29:24,762 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,771 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562964770,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,774 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:24,774 DEBUG [Reader=1,bindAddress=localhost,port=33041 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,775 DEBUG [Reader=1,bindAddress=localhost,port=33041 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:24,776 DEBUG [Reader=1,bindAddress=localhost,port=33041 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:24,779 DEBUG [Reader=1,bindAddress=localhost,port=33041 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:24,779 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:24,780 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:24,780 DEBUG [Reader=1,bindAddress=localhost,port=33041 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:24,780 DEBUG [Reader=1,bindAddress=localhost,port=33041 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:24,781 DEBUG [Reader=1,bindAddress=localhost,port=33041 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:24,781 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:24,782 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T19:29:24,782 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:33041
javax.security.sasl.SaslException: No common protection layer between client and server
	at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?]
	at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?]
	at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-25T19:29:24,782 WARN  [Thread-14 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732562964752
2024-11-25T19:29:24,821 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,824 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:24,825 DEBUG [Reader=2,bindAddress=localhost,port=33041 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,826 DEBUG [Reader=2,bindAddress=localhost,port=33041 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:24,827 DEBUG [Reader=2,bindAddress=localhost,port=33041 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:24,830 DEBUG [Reader=2,bindAddress=localhost,port=33041 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:24,830 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:24,831 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:24,832 DEBUG [Reader=2,bindAddress=localhost,port=33041 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:24,832 DEBUG [Reader=2,bindAddress=localhost,port=33041 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:24,833 DEBUG [Reader=2,bindAddress=localhost,port=33041 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:24,833 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:24,833 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T19:29:24,834 DEBUG [Thread-14 {}] ipc.FailedServers(52): Added failed server with address localhost:33041 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:24,835 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33041
2024-11-25T19:29:24,835 INFO  [Listener,port=33041 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33041: stopping
2024-11-25T19:29:24,836 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:24,836 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:24,841 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19)
Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=7318 (was 7337)
2024-11-25T19:29:24,848 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=7317
2024-11-25T19:29:24,862 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:24,863 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:24,869 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562964868,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:24,874 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:24,875 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44333
2024-11-25T19:29:24,877 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:24,877 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:24,877 INFO  [Listener,port=44333 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44333: starting
2024-11-25T19:29:24,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a3394e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:24,880 DEBUG [Thread-17 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:24,881 DEBUG [Thread-17 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44333
2024-11-25T19:29:24,883 DEBUG [Thread-17 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,892 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562964891,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,894 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:24,895 DEBUG [Reader=1,bindAddress=localhost,port=44333 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,896 DEBUG [Reader=1,bindAddress=localhost,port=44333 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:24,896 DEBUG [Reader=1,bindAddress=localhost,port=44333 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:24,899 DEBUG [Reader=1,bindAddress=localhost,port=44333 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:24,899 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:24,900 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:24,900 DEBUG [Reader=1,bindAddress=localhost,port=44333 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:24,900 DEBUG [Reader=1,bindAddress=localhost,port=44333 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:24,900 DEBUG [Reader=1,bindAddress=localhost,port=44333 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:24,901 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:24,901 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:24,901 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T19:29:24,901 DEBUG [Thread-17 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:24,901 DEBUG [Reader=1,bindAddress=localhost,port=44333 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:24,902 DEBUG [Reader=1,bindAddress=localhost,port=44333 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:24,902 DEBUG [Reader=1,bindAddress=localhost,port=44333 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,902 DEBUG [Reader=1,bindAddress=localhost,port=44333 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:24,902 INFO  [Reader=1,bindAddress=localhost,port=44333 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:24,903 INFO  [Reader=1,bindAddress=localhost,port=44333 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39834, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:24,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:24,936 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44333
2024-11-25T19:29:24,936 INFO  [Listener,port=44333 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44333: stopping
2024-11-25T19:29:24,937 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:24,937 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:24,937 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:24,941 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38319
2024-11-25T19:29:24,942 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:24,944 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:24,944 INFO  [Listener,port=38319 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38319: starting
2024-11-25T19:29:24,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e9d319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:24,947 DEBUG [Thread-20 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:24,947 DEBUG [Thread-20 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38319
2024-11-25T19:29:24,948 DEBUG [Thread-20 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,951 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:24,952 DEBUG [Reader=1,bindAddress=localhost,port=38319 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,953 DEBUG [Reader=1,bindAddress=localhost,port=38319 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:24,953 DEBUG [Reader=1,bindAddress=localhost,port=38319 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:24,955 DEBUG [Reader=1,bindAddress=localhost,port=38319 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:24,955 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:24,956 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:24,957 DEBUG [Reader=1,bindAddress=localhost,port=38319 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:24,957 DEBUG [Reader=1,bindAddress=localhost,port=38319 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:24,958 DEBUG [Reader=1,bindAddress=localhost,port=38319 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:24,958 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:24,959 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:24,959 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T19:29:24,959 DEBUG [Thread-20 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:24,959 DEBUG [Reader=1,bindAddress=localhost,port=38319 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:24,959 DEBUG [Reader=1,bindAddress=localhost,port=38319 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:24,960 DEBUG [Reader=1,bindAddress=localhost,port=38319 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:24,960 DEBUG [Reader=1,bindAddress=localhost,port=38319 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:24,960 INFO  [Reader=1,bindAddress=localhost,port=38319 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:24,961 INFO  [Reader=1,bindAddress=localhost,port=38319 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59614, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:24,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:24,991 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38319
2024-11-25T19:29:24,991 INFO  [Listener,port=38319 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38319: stopping
2024-11-25T19:29:24,992 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:24,991 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:24,992 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:24,998 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36613
2024-11-25T19:29:25,001 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:25,001 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:25,001 INFO  [Listener,port=36613 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36613: starting
2024-11-25T19:29:25,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60f30885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:25,006 DEBUG [Thread-23 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:25,007 DEBUG [Thread-23 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36613
2024-11-25T19:29:25,009 DEBUG [Thread-23 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,013 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(103): Have sent token of size 498 from initSASLContext.
2024-11-25T19:29:25,013 DEBUG [Reader=1,bindAddress=localhost,port=36613 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,015 DEBUG [Reader=1,bindAddress=localhost,port=36613 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,015 DEBUG [Reader=1,bindAddress=localhost,port=36613 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=498 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,018 DEBUG [Reader=1,bindAddress=localhost,port=36613 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 107 from saslServer.
2024-11-25T19:29:25,025 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(119): Will read input token of size 107 for processing by initSASLContext
2024-11-25T19:29:25,026 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:25,026 DEBUG [Reader=1,bindAddress=localhost,port=36613 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,026 DEBUG [Reader=1,bindAddress=localhost,port=36613 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,027 DEBUG [Reader=1,bindAddress=localhost,port=36613 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:25,027 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:25,028 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:25,028 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T19:29:25,029 DEBUG [Reader=1,bindAddress=localhost,port=36613 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,029 DEBUG [Thread-23 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:25,029 DEBUG [Reader=1,bindAddress=localhost,port=36613 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,029 DEBUG [Reader=1,bindAddress=localhost,port=36613 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,029 DEBUG [Reader=1,bindAddress=localhost,port=36613 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:25,030 INFO  [Reader=1,bindAddress=localhost,port=36613 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:25,030 INFO  [Reader=1,bindAddress=localhost,port=36613 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47330, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:25,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:25,055 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36613
2024-11-25T19:29:25,056 INFO  [Listener,port=36613 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36613: stopping
2024-11-25T19:29:25,056 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:25,056 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:25,057 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:25,060 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38645
2024-11-25T19:29:25,061 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:25,061 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:25,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130e4517, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:25,062 INFO  [Listener,port=38645 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38645: starting
2024-11-25T19:29:25,063 DEBUG [Thread-26 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:25,065 DEBUG [Thread-26 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38645
2024-11-25T19:29:25,066 DEBUG [Thread-26 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,072 DEBUG [Reader=1,bindAddress=localhost,port=38645 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,073 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:25,074 DEBUG [Reader=1,bindAddress=localhost,port=38645 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,074 DEBUG [Reader=1,bindAddress=localhost,port=38645 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,076 DEBUG [Reader=1,bindAddress=localhost,port=38645 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:25,077 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:25,077 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:25,078 DEBUG [Reader=1,bindAddress=localhost,port=38645 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,078 DEBUG [Reader=1,bindAddress=localhost,port=38645 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,079 DEBUG [Reader=1,bindAddress=localhost,port=38645 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:25,079 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:25,079 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:25,079 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int
2024-11-25T19:29:25,080 DEBUG [Reader=1,bindAddress=localhost,port=38645 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,080 DEBUG [Thread-26 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:25,080 DEBUG [Reader=1,bindAddress=localhost,port=38645 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,080 DEBUG [Reader=1,bindAddress=localhost,port=38645 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,080 DEBUG [Reader=1,bindAddress=localhost,port=38645 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-25T19:29:25,080 INFO  [Reader=1,bindAddress=localhost,port=38645 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:25,081 INFO  [Reader=1,bindAddress=localhost,port=38645 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41582, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:25,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:25,103 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38645
2024-11-25T19:29:25,103 INFO  [Listener,port=38645 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38645: stopping
2024-11-25T19:29:25,103 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:25,103 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:25,104 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:25,105 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43763
2024-11-25T19:29:25,105 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:25,105 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:25,105 INFO  [Listener,port=43763 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43763: starting
2024-11-25T19:29:25,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ea53fb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:25,107 DEBUG [Thread-29 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:25,108 DEBUG [Thread-29 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43763
2024-11-25T19:29:25,109 DEBUG [Thread-29 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,111 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:25,111 DEBUG [Reader=1,bindAddress=localhost,port=43763 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,113 DEBUG [Reader=1,bindAddress=localhost,port=43763 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,113 DEBUG [Reader=1,bindAddress=localhost,port=43763 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,114 DEBUG [Reader=1,bindAddress=localhost,port=43763 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:25,114 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:25,115 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:25,115 DEBUG [Reader=1,bindAddress=localhost,port=43763 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,115 DEBUG [Reader=1,bindAddress=localhost,port=43763 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,116 DEBUG [Reader=1,bindAddress=localhost,port=43763 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:25,116 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:25,116 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:25,116 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T19:29:25,117 DEBUG [Reader=1,bindAddress=localhost,port=43763 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,117 DEBUG [Thread-29 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:25,117 DEBUG [Reader=1,bindAddress=localhost,port=43763 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,117 DEBUG [Reader=1,bindAddress=localhost,port=43763 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,117 DEBUG [Reader=1,bindAddress=localhost,port=43763 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:25,117 INFO  [Reader=1,bindAddress=localhost,port=43763 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:25,118 INFO  [Reader=1,bindAddress=localhost,port=43763 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33912, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:25,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:25,135 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43763
2024-11-25T19:29:25,135 INFO  [Listener,port=43763 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43763: stopping
2024-11-25T19:29:25,135 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:25,135 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:25,142 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=22 (was 19)
Potentially hanging thread: Reader=2,bindAddress=localhost,port=43763
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/java.nio.HeapByteBuffer.putInt(HeapByteBuffer.java:455)
	app//org.apache.maven.surefire.api.stream.AbstractStreamEncoder.encodeString(AbstractStreamEncoder.java:121)
	app//org.apache.maven.surefire.api.stream.AbstractStreamEncoder.encodeStringData(AbstractStreamEncoder.java:156)
	app//org.apache.maven.surefire.api.stream.AbstractStreamEncoder.encode(AbstractStreamEncoder.java:145)
	app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.encodeMessage(EventChannelEncoder.java:378)
	app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.setOutErr(EventChannelEncoder.java:172)
	app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.testOutput(EventChannelEncoder.java:168)
	app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:99)
	app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:43)
	app//org.apache.maven.surefire.common.junit4.JUnit4RunListener.writeTestOutput(JUnit4RunListener.java:202)
	app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:57)
	app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250)
	app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283)
	app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199)
	app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158)
	app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
	app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552)
	app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
	app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
	app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=7271 (was 7317)
2024-11-25T19:29:25,148 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=7271
2024-11-25T19:29:25,163 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:25,163 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:25,168 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562965168,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:25,173 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:25,176 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39747
2024-11-25T19:29:25,180 INFO  [Listener,port=39747 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39747: starting
2024-11-25T19:29:25,180 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:25,180 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:25,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c1dafed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:25,182 DEBUG [Thread-32 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T19:29:25,183 DEBUG [Thread-32 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39747
2024-11-25T19:29:25,187 DEBUG [BRPC Connection (1039819159) to localhost:39747 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-25T19:29:25,187 WARN  [BRPC Connection (1039819159) to localhost:39747 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-25T19:29:25,189 DEBUG [BRPC Connection (1039819159) to localhost:39747 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-25T19:29:25,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:25,190 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39747
2024-11-25T19:29:25,190 INFO  [Listener,port=39747 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39747: stopping
2024-11-25T19:29:25,190 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:25,191 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:25,199 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=7259 (was 7271)
2024-11-25T19:29:25,204 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=7259
2024-11-25T19:29:25,221 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:25,221 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:25,227 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562965227,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:25,232 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:25,236 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36661
2024-11-25T19:29:25,239 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-25T19:29:25,239 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-25T19:29:25,239 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-25T19:29:25,239 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-25T19:29:25,239 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-25T19:29:25,239 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-25T19:29:25,240 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-25T19:29:25,240 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-25T19:29:25,240 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:25,240 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:25,240 INFO  [Listener,port=36661 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36661: starting
2024-11-25T19:29:25,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30529b8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:25,242 DEBUG [Thread-35 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T19:29:25,242 DEBUG [Thread-35 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36661
2024-11-25T19:29:25,243 WARN  [Reader=1,bindAddress=localhost,port=36661 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-25T19:29:25,243 INFO  [Reader=1,bindAddress=localhost,port=36661 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T19:29:25,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:25,259 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36661
2024-11-25T19:29:25,259 INFO  [Listener,port=36661 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36661: stopping
2024-11-25T19:29:25,260 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:25,260 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:25,272 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=7238 (was 7259)
2024-11-25T19:29:25,284 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=7237
2024-11-25T19:29:25,300 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:25,301 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:25,310 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562965309,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:25,314 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:25,316 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35841
2024-11-25T19:29:25,317 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:25,321 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:25,321 INFO  [Listener,port=35841 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35841: starting
2024-11-25T19:29:25,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62953be4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:25,323 DEBUG [Thread-38 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:25,323 DEBUG [Thread-38 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35841
2024-11-25T19:29:25,325 DEBUG [Thread-38 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,333 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562965332,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,336 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:25,337 DEBUG [Reader=1,bindAddress=localhost,port=35841 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,338 DEBUG [Reader=1,bindAddress=localhost,port=35841 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,338 DEBUG [Reader=1,bindAddress=localhost,port=35841 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,340 DEBUG [Reader=1,bindAddress=localhost,port=35841 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:25,340 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:25,341 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:25,341 DEBUG [Reader=1,bindAddress=localhost,port=35841 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,341 DEBUG [Reader=1,bindAddress=localhost,port=35841 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,346 DEBUG [Reader=1,bindAddress=localhost,port=35841 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:25,348 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:25,350 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:25,351 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T19:29:25,351 DEBUG [Thread-38 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:25,351 DEBUG [Reader=1,bindAddress=localhost,port=35841 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,352 DEBUG [Reader=1,bindAddress=localhost,port=35841 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,352 DEBUG [Reader=1,bindAddress=localhost,port=35841 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,352 DEBUG [Reader=1,bindAddress=localhost,port=35841 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:25,352 INFO  [Reader=1,bindAddress=localhost,port=35841 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:25,353 INFO  [Reader=1,bindAddress=localhost,port=35841 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43042, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:25,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:25,376 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35841
2024-11-25T19:29:25,376 INFO  [Listener,port=35841 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35841: stopping
2024-11-25T19:29:25,377 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:25,377 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:25,386 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19)
Potentially hanging thread: RpcServer.responder
	app//org.apache.logging.log4j.core.layout.TextEncoderHelper.writeEncodedText(TextEncoderHelper.java:82)
	app//org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:65)
	app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68)
	app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32)
	app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:227)
	app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199)
	app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158)
	app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
	app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552)
	app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
	app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
	app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=7206 (was 7237)
2024-11-25T19:29:25,396 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=292, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=7200
2024-11-25T19:29:25,414 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:25,414 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:25,420 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562965420,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:25,425 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:25,427 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39709
2024-11-25T19:29:25,433 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:25,433 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:25,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fcd33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:25,434 INFO  [Listener,port=39709 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39709: starting
2024-11-25T19:29:25,435 DEBUG [Thread-41 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:25,435 DEBUG [Thread-41 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39709
2024-11-25T19:29:25,437 DEBUG [Thread-41 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,447 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562965446,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,450 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:25,450 DEBUG [Reader=1,bindAddress=localhost,port=39709 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,452 DEBUG [Reader=1,bindAddress=localhost,port=39709 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,452 DEBUG [Reader=1,bindAddress=localhost,port=39709 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,454 DEBUG [Reader=1,bindAddress=localhost,port=39709 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:25,454 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:25,454 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:25,455 DEBUG [Reader=1,bindAddress=localhost,port=39709 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,455 DEBUG [Reader=1,bindAddress=localhost,port=39709 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,455 DEBUG [Reader=1,bindAddress=localhost,port=39709 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:25,456 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:25,456 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:25,456 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T19:29:25,456 DEBUG [Reader=1,bindAddress=localhost,port=39709 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,456 DEBUG [Thread-41 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:25,456 DEBUG [Reader=1,bindAddress=localhost,port=39709 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,457 DEBUG [Reader=1,bindAddress=localhost,port=39709 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,457 DEBUG [Reader=1,bindAddress=localhost,port=39709 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:25,457 INFO  [Reader=1,bindAddress=localhost,port=39709 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:25,458 INFO  [Reader=1,bindAddress=localhost,port=39709 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51700, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:25,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:25,479 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39709
2024-11-25T19:29:25,479 INFO  [Listener,port=39709 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39709: stopping
2024-11-25T19:29:25,479 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:25,480 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:25,480 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:25,493 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40005
2024-11-25T19:29:25,495 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:25,497 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:25,497 INFO  [Listener,port=40005 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40005: starting
2024-11-25T19:29:25,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42853030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:25,499 DEBUG [Thread-44 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:25,499 DEBUG [Thread-44 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40005
2024-11-25T19:29:25,500 DEBUG [Thread-44 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,503 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:25,503 DEBUG [Reader=1,bindAddress=localhost,port=40005 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,505 DEBUG [Reader=1,bindAddress=localhost,port=40005 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,505 DEBUG [Reader=1,bindAddress=localhost,port=40005 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,506 DEBUG [Reader=1,bindAddress=localhost,port=40005 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 107 from saslServer.
2024-11-25T19:29:25,507 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(119): Will read input token of size 107 for processing by initSASLContext
2024-11-25T19:29:25,507 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:25,507 DEBUG [Reader=1,bindAddress=localhost,port=40005 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,508 DEBUG [Reader=1,bindAddress=localhost,port=40005 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,508 DEBUG [Reader=1,bindAddress=localhost,port=40005 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:25,508 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:25,508 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:25,508 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T19:29:25,509 DEBUG [Reader=1,bindAddress=localhost,port=40005 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:25,509 DEBUG [Thread-44 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:25,509 DEBUG [Reader=1,bindAddress=localhost,port=40005 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:25,509 DEBUG [Reader=1,bindAddress=localhost,port=40005 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:25,509 DEBUG [Reader=1,bindAddress=localhost,port=40005 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:25,509 INFO  [Reader=1,bindAddress=localhost,port=40005 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:25,510 INFO  [Reader=1,bindAddress=localhost,port=40005 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39894, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:27,512 ERROR [Thread-44 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function.
java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39894 remote=localhost/127.0.0.1:40005]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?]
	at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?]
	at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-25T19:29:27,512 DEBUG [Thread-44 {}] ipc.FailedServers(52): Added failed server with address localhost:40005 to list caused by java.io.IOException: Timeout while waiting connection header response
2024-11-25T19:29:27,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:27,513 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40005
2024-11-25T19:29:27,513 INFO  [Listener,port=40005 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40005: stopping
2024-11-25T19:29:27,513 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:27,513 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:27,519 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) - Thread LEAK? -, OpenFileDescriptor=290 (was 292), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 180) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6962 (was 7200)
2024-11-25T19:29:27,525 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6962
2024-11-25T19:29:27,540 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:27,540 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:27,545 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562967544,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:27,548 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:27,550 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42445
2024-11-25T19:29:27,551 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:27,551 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:27,551 INFO  [Listener,port=42445 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42445: starting
2024-11-25T19:29:27,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67daa0d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:27,553 DEBUG [Thread-47 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:27,554 DEBUG [Thread-47 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42445
2024-11-25T19:29:27,555 DEBUG [Thread-47 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,562 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562967561,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,564 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:27,564 DEBUG [Reader=1,bindAddress=localhost,port=42445 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,565 DEBUG [Reader=1,bindAddress=localhost,port=42445 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:27,565 DEBUG [Reader=1,bindAddress=localhost,port=42445 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:27,567 DEBUG [Reader=1,bindAddress=localhost,port=42445 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:27,567 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:27,567 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:27,568 DEBUG [Reader=1,bindAddress=localhost,port=42445 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:27,568 DEBUG [Reader=1,bindAddress=localhost,port=42445 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:27,568 DEBUG [Reader=1,bindAddress=localhost,port=42445 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:27,568 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:27,568 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:27,568 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T19:29:27,569 DEBUG [Reader=1,bindAddress=localhost,port=42445 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:27,569 DEBUG [Thread-47 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:27,569 DEBUG [Reader=1,bindAddress=localhost,port=42445 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:27,569 DEBUG [Reader=1,bindAddress=localhost,port=42445 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,569 DEBUG [Reader=1,bindAddress=localhost,port=42445 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:27,569 INFO  [Reader=1,bindAddress=localhost,port=42445 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:27,570 INFO  [Reader=1,bindAddress=localhost,port=42445 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51426, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:27,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:27,579 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42445
2024-11-25T19:29:27,579 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:27,579 INFO  [Listener,port=42445 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42445: stopping
2024-11-25T19:29:27,580 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:27,586 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=22 (was 19)
Potentially hanging thread: Listener,port=42445
	java.base@17.0.11/java.lang.ThreadLocal$ThreadLocalMap.set(ThreadLocal.java:486)
	java.base@17.0.11/java.lang.ThreadLocal.set(ThreadLocal.java:222)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:124)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
	app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552)
	app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
	app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
	app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:252)

Potentially hanging thread: RpcServer idle connection scanner for port 42445
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/jdk.internal.misc.VM.getNanoTimeAdjustment(Native Method)
	java.base@17.0.11/java.time.Clock.currentInstant(Clock.java:498)
	java.base@17.0.11/java.time.Clock$SystemClock.instant(Clock.java:614)
	app//org.apache.logging.log4j.core.util.SystemClock.init(SystemClock.java:44)
	app//org.apache.logging.log4j.core.time.MutableInstant.initFrom(MutableInstant.java:105)
	app//org.apache.logging.log4j.core.impl.Log4jLogEvent.<init>(Log4jLogEvent.java:458)
	app//org.apache.logging.log4j.core.impl.Log4jLogEvent.<init>(Log4jLogEvent.java:355)
	app//org.apache.logging.log4j.core.impl.DefaultLogEventFactory.createEvent(DefaultLogEventFactory.java:74)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:550)
	app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
	app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
	app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6943 (was 6962)
2024-11-25T19:29:27,596 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=12, AvailableMemoryMB=6942
2024-11-25T19:29:27,608 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:27,609 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:27,613 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562967613,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:27,616 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:27,617 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44515
2024-11-25T19:29:27,618 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:27,620 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:27,620 INFO  [Listener,port=44515 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44515: starting
2024-11-25T19:29:27,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7510aa00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:27,621 DEBUG [Thread-50 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:27,621 DEBUG [Thread-50 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44515
2024-11-25T19:29:27,622 DEBUG [Thread-50 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,628 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562967628,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,630 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:27,630 DEBUG [Reader=1,bindAddress=localhost,port=44515 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,631 DEBUG [Reader=1,bindAddress=localhost,port=44515 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:27,632 DEBUG [Reader=1,bindAddress=localhost,port=44515 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:27,633 DEBUG [Reader=1,bindAddress=localhost,port=44515 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:27,633 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:27,634 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:27,634 DEBUG [Reader=1,bindAddress=localhost,port=44515 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:27,634 DEBUG [Reader=1,bindAddress=localhost,port=44515 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:27,634 DEBUG [Reader=1,bindAddress=localhost,port=44515 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:27,635 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:27,635 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:27,635 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T19:29:27,635 DEBUG [Reader=1,bindAddress=localhost,port=44515 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:27,635 DEBUG [Thread-50 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:27,635 DEBUG [Reader=1,bindAddress=localhost,port=44515 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:27,635 DEBUG [Reader=1,bindAddress=localhost,port=44515 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,636 DEBUG [Reader=1,bindAddress=localhost,port=44515 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:27,636 INFO  [Reader=1,bindAddress=localhost,port=44515 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:27,636 INFO  [Reader=1,bindAddress=localhost,port=44515 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48578, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:27,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:27,644 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44515
2024-11-25T19:29:27,644 INFO  [Listener,port=44515 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44515: stopping
2024-11-25T19:29:27,644 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:27,645 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:27,649 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 12), AvailableMemoryMB=6924 (was 6942)
2024-11-25T19:29:27,653 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6924
2024-11-25T19:29:27,664 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:27,664 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:27,668 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562967668,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:27,672 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:27,674 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42255
2024-11-25T19:29:27,674 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:27,674 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:27,674 INFO  [Listener,port=42255 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42255: starting
2024-11-25T19:29:27,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55fd042, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:27,676 DEBUG [Thread-53 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:27,676 DEBUG [Thread-53 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42255
2024-11-25T19:29:27,677 DEBUG [Thread-53 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,682 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562967682,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,684 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:27,684 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth.
2024-11-25T19:29:27,684 DEBUG [Thread-53 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:27,684 INFO  [Reader=1,bindAddress=localhost,port=42255 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T19:29:27,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:27,691 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42255
2024-11-25T19:29:27,691 INFO  [Listener,port=42255 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42255: stopping
2024-11-25T19:29:27,691 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:27,692 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:27,696 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6909 (was 6924)
2024-11-25T19:29:27,701 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6909
2024-11-25T19:29:27,712 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:27,712 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:27,716 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562967716,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:27,719 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:27,730 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:27,733 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-25T19:29:27,764 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 81415 (auto-detected)
2024-11-25T19:29:27,766 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-25T19:29:27,785 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34017
2024-11-25T19:29:27,785 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:27,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c225c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:27,786 DEBUG [Thread-54 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:27,787 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34017
2024-11-25T19:29:27,788 DEBUG [Thread-54 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,795 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562967794,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,797 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:27,817 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:27,817 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,819 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:27,822 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:27,823 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:27,823 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:27,824 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:27,824 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:27,824 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:27,825 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T19:29:27,825 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:27,825 DEBUG [Thread-54 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:27,825 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,825 DEBUG [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:27,825 INFO  [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:27,831 INFO  [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36370, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:27,831 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93
2024-11-25T19:29:27,831 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:251
2024-11-25T19:29:27,832 DEBUG [BRPC Connection (1039819159) to localhost:34017 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-25T19:29:27,835 DEBUG [BRPC Connection (1039819159) to localhost:34017 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:237
2024-11-25T19:29:27,835 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2261
2024-11-25T19:29:27,836 DEBUG [BRPC Connection (1039819159) to localhost:34017 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-25T19:29:27,838 DEBUG [BRPC Connection (1039819159) to localhost:34017 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2247
2024-11-25T19:29:27,840 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117
2024-11-25T19:29:27,841 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18189
2024-11-25T19:29:27,842 DEBUG [BRPC Connection (1039819159) to localhost:34017 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-25T19:29:27,852 DEBUG [BRPC Connection (1039819159) to localhost:34017 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22292
2024-11-25T19:29:27,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:27,853 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34017
2024-11-25T19:29:27,865 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21 (was 19)
Potentially hanging thread: NettyRpcServer-2-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-2-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=299 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6872 (was 6909)
2024-11-25T19:29:27,872 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21, OpenFileDescriptor=299, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6871
2024-11-25T19:29:27,885 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:27,886 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:27,890 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562967890,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:27,894 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:27,894 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:27,895 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45207
2024-11-25T19:29:27,896 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:27,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73552824, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-25T19:29:27,898 DEBUG [Thread-55 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:27,898 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45207
2024-11-25T19:29:27,899 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,906 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562967905,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,907 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:27,907 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T19:29:27,908 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:45207
org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-25T19:29:27,908 WARN  [Thread-55 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732562967892
2024-11-25T19:29:27,988 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,991 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:27,991 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T19:29:27,991 DEBUG [Thread-55 {}] ipc.FailedServers(52): Added failed server with address localhost:45207 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:27,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:27,992 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45207
2024-11-25T19:29:27,998 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24 (was 21)
Potentially hanging thread: NettyRpcServer-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=308 (was 299) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6831 (was 6871)
2024-11-25T19:29:28,005 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6828
2024-11-25T19:29:28,019 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:28,019 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:28,024 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562968024,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:28,028 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:28,029 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:28,032 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46175
2024-11-25T19:29:28,032 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:28,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6275d190, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:28,034 DEBUG [Thread-56 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:28,035 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46175
2024-11-25T19:29:28,036 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,043 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562968043,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,046 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:28,047 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,048 DEBUG [NettyRpcServer-4-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,050 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:28,051 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:28,051 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:28,052 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,052 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:28,052 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:28,053 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T19:29:28,053 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:46175
javax.security.sasl.SaslException: No common protection layer between client and server
	at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?]
	at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?]
	at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-25T19:29:28,054 WARN  [Thread-56 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732562968026
2024-11-25T19:29:28,154 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,156 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:28,156 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,156 DEBUG [NettyRpcServer-4-3 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,159 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:28,160 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:28,160 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:28,160 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,161 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:28,161 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:28,162 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T19:29:28,162 DEBUG [Thread-56 {}] ipc.FailedServers(52): Added failed server with address localhost:46175 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:28,163 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46175
2024-11-25T19:29:28,172 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27 (was 24)
Potentially hanging thread: NettyRpcServer-4-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-4-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-4-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=317 (was 308) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6774 (was 6828)
2024-11-25T19:29:28,179 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27, OpenFileDescriptor=317, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6772
2024-11-25T19:29:28,198 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:28,199 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:28,205 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562968204,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:28,210 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:28,211 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:28,212 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43447
2024-11-25T19:29:28,213 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:28,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4327545d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:28,214 DEBUG [Thread-57 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:28,215 DEBUG [Thread-57 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43447
2024-11-25T19:29:28,217 DEBUG [Thread-57 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,227 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562968226,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,228 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:28,231 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,231 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,233 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:28,233 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:28,234 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:28,234 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,234 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:28,235 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:28,235 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:28,236 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T19:29:28,236 DEBUG [Thread-57 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:28,236 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,236 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,237 DEBUG [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:28,237 INFO  [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:28,237 INFO  [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48344, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:28,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:28,260 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43447
2024-11-25T19:29:28,262 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:28,262 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:28,265 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42837
2024-11-25T19:29:28,266 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:28,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bdb6e69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:28,268 DEBUG [Thread-58 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:28,269 DEBUG [Thread-58 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42837
2024-11-25T19:29:28,270 DEBUG [Thread-58 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,272 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:28,274 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,274 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,276 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:28,277 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:28,277 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:28,277 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,277 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:28,278 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:28,278 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:28,278 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T19:29:28,279 DEBUG [Thread-58 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:28,279 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,279 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,279 DEBUG [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:28,279 INFO  [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:28,280 INFO  [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54300, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:28,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:28,291 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42837
2024-11-25T19:29:28,292 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:28,292 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:28,294 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37685
2024-11-25T19:29:28,294 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:28,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634b09ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:28,295 DEBUG [Thread-59 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:28,296 DEBUG [Thread-59 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37685
2024-11-25T19:29:28,297 DEBUG [Thread-59 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,299 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:28,299 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,299 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,301 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:28,302 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:28,302 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:28,303 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,303 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:28,303 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:28,303 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:28,304 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T19:29:28,304 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,304 DEBUG [Thread-59 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:28,304 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,304 DEBUG [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:28,304 INFO  [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:28,305 INFO  [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35666, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:28,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:28,319 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37685
2024-11-25T19:29:28,320 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:28,321 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:28,322 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42357
2024-11-25T19:29:28,322 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:28,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eca0a7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:28,326 DEBUG [Thread-60 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:28,326 DEBUG [Thread-60 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42357
2024-11-25T19:29:28,330 DEBUG [Thread-60 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,331 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,332 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,331 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:28,334 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:28,335 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:28,335 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:28,335 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,336 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:28,336 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:28,336 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:28,336 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int
2024-11-25T19:29:28,337 DEBUG [Thread-60 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:28,337 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,337 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,337 DEBUG [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-25T19:29:28,337 INFO  [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:28,338 INFO  [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57380, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:28,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:28,349 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42357
2024-11-25T19:29:28,350 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:28,350 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:28,353 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43925
2024-11-25T19:29:28,354 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:28,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af2e8ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:28,355 DEBUG [Thread-61 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:28,355 DEBUG [Thread-61 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43925
2024-11-25T19:29:28,356 DEBUG [Thread-61 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,358 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:28,358 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,358 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,361 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:28,365 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:28,365 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:28,366 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,366 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:28,366 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:28,367 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:28,367 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T19:29:28,367 DEBUG [Thread-61 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:28,367 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,367 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,367 DEBUG [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:28,367 INFO  [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:28,369 INFO  [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41890, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:28,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:28,386 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43925
2024-11-25T19:29:28,393 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37 (was 27)
Potentially hanging thread: NettyRpcServer-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=362 (was 317) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6711 (was 6772)
2024-11-25T19:29:28,399 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37, OpenFileDescriptor=362, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6710
2024-11-25T19:29:28,416 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:28,417 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:28,422 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562968421,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:28,425 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:28,426 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:28,431 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37473
2024-11-25T19:29:28,432 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:28,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e4d30bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:28,433 DEBUG [Thread-62 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T19:29:28,433 DEBUG [Thread-62 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37473
2024-11-25T19:29:28,435 DEBUG [BRPC Connection (1039819159) to localhost:37473 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-25T19:29:28,436 WARN  [BRPC Connection (1039819159) to localhost:37473 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-25T19:29:28,436 DEBUG [BRPC Connection (1039819159) to localhost:37473 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-25T19:29:28,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:28,436 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37473
2024-11-25T19:29:28,442 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39 (was 37)
Potentially hanging thread: NettyRpcServer-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=371 (was 362) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6697 (was 6710)
2024-11-25T19:29:28,448 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39, OpenFileDescriptor=371, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6697
2024-11-25T19:29:28,460 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:28,461 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:28,465 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562968464,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:28,468 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:28,468 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:28,470 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37237
2024-11-25T19:29:28,470 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-25T19:29:28,470 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-25T19:29:28,470 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-25T19:29:28,470 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-25T19:29:28,470 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-25T19:29:28,470 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-25T19:29:28,470 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-25T19:29:28,470 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-25T19:29:28,470 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:28,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@369bb180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:28,472 DEBUG [Thread-63 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T19:29:28,472 DEBUG [Thread-63 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37237
2024-11-25T19:29:28,483 WARN  [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-25T19:29:28,483 INFO  [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T19:29:28,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:28,495 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37237
2024-11-25T19:29:28,503 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41 (was 39)
Potentially hanging thread: NettyRpcServer-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=380 (was 371) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6680 (was 6697)
2024-11-25T19:29:28,511 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6679
2024-11-25T19:29:28,526 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:28,527 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:28,530 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562968530,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:28,533 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:28,533 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:28,534 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39503
2024-11-25T19:29:28,534 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:28,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63ea3a9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:28,536 DEBUG [Thread-64 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:28,536 DEBUG [Thread-64 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:39503
2024-11-25T19:29:28,537 DEBUG [Thread-64 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,543 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562968543,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,545 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:28,545 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,545 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,547 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:28,548 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:28,548 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:28,549 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,549 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:28,549 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:28,550 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:28,550 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T19:29:28,550 DEBUG [Thread-64 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:28,550 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,550 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,550 DEBUG [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:28,550 INFO  [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:28,551 INFO  [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39654, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:28,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:28,564 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39503
2024-11-25T19:29:28,572 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43 (was 41)
Potentially hanging thread: NettyRpcServer-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=389 (was 380) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6657 (was 6679)
2024-11-25T19:29:28,586 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43, OpenFileDescriptor=389, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6655
2024-11-25T19:29:28,603 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:28,603 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:28,608 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562968607,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:28,611 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:28,611 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:28,613 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43723
2024-11-25T19:29:28,613 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:28,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6784b8d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:28,615 DEBUG [Thread-65 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:28,615 DEBUG [Thread-65 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43723
2024-11-25T19:29:28,617 DEBUG [Thread-65 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,623 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562968623,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,625 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:28,625 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,625 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,628 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:28,628 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:28,628 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:28,629 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,629 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:28,629 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:28,629 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:28,630 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T19:29:28,630 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,630 DEBUG [Thread-65 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:28,630 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,630 DEBUG [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:28,630 INFO  [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:28,631 INFO  [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34702, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:28,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:28,650 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43723
2024-11-25T19:29:28,651 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:28,652 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:28,657 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41057
2024-11-25T19:29:28,658 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:28,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b09d65e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:28,663 DEBUG [Thread-66 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:28,664 DEBUG [Thread-66 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:41057
2024-11-25T19:29:28,665 DEBUG [Thread-66 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,667 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:28,667 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,667 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,670 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:28,670 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:28,671 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:28,671 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,671 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:28,672 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:28,672 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:28,672 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-25T19:29:28,672 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:28,672 DEBUG [Thread-66 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:28,672 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:28,673 DEBUG [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:28,673 INFO  [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:28,674 INFO  [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45152, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:30,673 ERROR [Thread-66 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function.
java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45152 remote=localhost/127.0.0.1:41057]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?]
	at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?]
	at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-25T19:29:30,674 DEBUG [Thread-66 {}] ipc.FailedServers(52): Added failed server with address localhost:41057 to list caused by java.io.IOException: Timeout while waiting connection header response
2024-11-25T19:29:30,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:30,674 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41057
2024-11-25T19:29:30,680 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47 (was 43)
Potentially hanging thread: NettyRpcServer-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=407 (was 389) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6559 (was 6655)
2024-11-25T19:29:30,686 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47, OpenFileDescriptor=407, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6559
2024-11-25T19:29:30,699 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:30,699 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:30,703 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562970702,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:30,705 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:30,705 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:30,706 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40075
2024-11-25T19:29:30,706 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:30,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@248fc825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:30,707 DEBUG [Thread-67 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:30,707 DEBUG [Thread-67 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40075
2024-11-25T19:29:30,708 DEBUG [Thread-67 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,714 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562970713,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,715 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:30,715 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:30,716 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,717 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:30,717 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-25T19:29:30,718 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:30,718 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:30,718 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:30,719 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:30,719 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:30,719 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T19:29:30,719 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:30,719 DEBUG [Thread-67 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:30,719 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,719 DEBUG [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:30,720 INFO  [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:30,720 INFO  [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52678, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:30,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:30,728 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40075
2024-11-25T19:29:30,734 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49 (was 47)
Potentially hanging thread: NettyRpcServer-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=416 (was 407) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6541 (was 6559)
2024-11-25T19:29:30,740 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49, OpenFileDescriptor=416, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6540
2024-11-25T19:29:30,751 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:30,751 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:30,755 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562970755,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:30,758 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:30,758 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:30,759 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42291
2024-11-25T19:29:30,759 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:30,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11d7cbda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:30,761 DEBUG [Thread-68 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:30,761 DEBUG [Thread-68 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42291
2024-11-25T19:29:30,762 DEBUG [Thread-68 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,768 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562970767,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,769 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:30,770 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:30,770 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,772 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 107 from saslServer.
2024-11-25T19:29:30,772 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(119): Will read input token of size 107 for processing by initSASLContext
2024-11-25T19:29:30,772 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-25T19:29:30,773 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:30,773 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:30,773 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-25T19:29:30,773 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-25T19:29:30,773 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-25T19:29:30,773 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:30,773 DEBUG [Thread-68 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:30,773 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,774 DEBUG [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:30,774 INFO  [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:30,774 INFO  [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38042, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:30,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:30,780 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42291
2024-11-25T19:29:30,785 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51 (was 49)
Potentially hanging thread: NettyRpcServer-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=425 (was 416) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6523 (was 6540)
2024-11-25T19:29:30,791 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6523
2024-11-25T19:29:30,801 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:30,801 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:30,805 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562970805,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:30,807 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:30,808 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:30,809 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45105
2024-11-25T19:29:30,809 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:30,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f20523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:30,810 DEBUG [Thread-69 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:30,810 DEBUG [Thread-69 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45105
2024-11-25T19:29:30,811 DEBUG [Thread-69 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,816 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562970816,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,817 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-25T19:29:30,817 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth.
2024-11-25T19:29:30,817 DEBUG [Thread-69 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:30,818 INFO  [NettyRpcServer-17-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T19:29:30,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:30,824 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45105
2024-11-25T19:29:30,830 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=53 (was 51)
Potentially hanging thread: NettyRpcServer-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=434 (was 425) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6506 (was 6523)
2024-11-25T19:29:30,835 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=53, OpenFileDescriptor=434, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6506
2024-11-25T19:29:30,845 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:30,845 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:30,848 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562970848,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:30,850 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:30,851 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36521
2024-11-25T19:29:30,851 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:30,851 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:30,852 INFO  [Listener,port=36521 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36521: starting
2024-11-25T19:29:30,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395f7dcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:30,855 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-25T19:29:30,855 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-25T19:29:30,860 DEBUG [Thread-72 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:30,876 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,882 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562970881,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,884 DEBUG [Reader=1,bindAddress=localhost,port=36521 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,884 DEBUG [Reader=1,bindAddress=localhost,port=36521 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:30,884 DEBUG [Reader=1,bindAddress=localhost,port=36521 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:30,885 DEBUG [Reader=1,bindAddress=localhost,port=36521 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:30,886 DEBUG [Reader=1,bindAddress=localhost,port=36521 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:30,886 DEBUG [Reader=1,bindAddress=localhost,port=36521 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:30,886 DEBUG [Reader=1,bindAddress=localhost,port=36521 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:30,887 DEBUG [Reader=1,bindAddress=localhost,port=36521 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:30,887 DEBUG [Reader=1,bindAddress=localhost,port=36521 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:30,887 DEBUG [Reader=1,bindAddress=localhost,port=36521 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,887 DEBUG [Reader=1,bindAddress=localhost,port=36521 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:30,887 INFO  [Reader=1,bindAddress=localhost,port=36521 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:30,894 INFO  [Reader=1,bindAddress=localhost,port=36521 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55280, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:30,895 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:30,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:30,906 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36521
2024-11-25T19:29:30,906 INFO  [Listener,port=36521 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36521: stopping
2024-11-25T19:29:30,906 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:30,907 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:30,912 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=54 (was 53) - Thread LEAK? -, OpenFileDescriptor=466 (was 434) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6487 (was 6506)
2024-11-25T19:29:30,917 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=54, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6486
2024-11-25T19:29:30,929 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:30,929 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:30,933 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562970932,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:30,934 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:30,935 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=37731
2024-11-25T19:29:30,936 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:30,936 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:30,936 INFO  [Listener,port=37731 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=37731: starting
2024-11-25T19:29:30,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@320f4af1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-25T19:29:30,937 DEBUG [Thread-75 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:30,939 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,944 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562970944,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,946 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T19:29:30,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:30,947 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:37731 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
2024-11-25T19:29:30,947 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 37731
2024-11-25T19:29:30,947 INFO  [Listener,port=37731 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=37731: stopping
2024-11-25T19:29:30,947 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:30,947 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:30,953 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=56 (was 54)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6472 (was 6486)
2024-11-25T19:29:30,958 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=55, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6472
2024-11-25T19:29:30,969 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:30,969 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:30,973 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562970973,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:30,975 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:30,976 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39193
2024-11-25T19:29:30,977 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:30,977 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:30,977 INFO  [Listener,port=39193 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39193: starting
2024-11-25T19:29:30,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@779ebdae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:30,978 DEBUG [Thread-78 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:30,980 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,984 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562970984,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,986 DEBUG [Reader=1,bindAddress=localhost,port=39193 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:30,987 DEBUG [Reader=1,bindAddress=localhost,port=39193 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:30,987 DEBUG [Reader=1,bindAddress=localhost,port=39193 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:30,988 DEBUG [Reader=1,bindAddress=localhost,port=39193 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:30,989 DEBUG [Reader=1,bindAddress=localhost,port=39193 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:30,989 DEBUG [Reader=1,bindAddress=localhost,port=39193 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:30,989 DEBUG [Reader=1,bindAddress=localhost,port=39193 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:30,990 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T19:29:30,991 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:39193 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server
2024-11-25T19:29:30,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:30,991 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39193
2024-11-25T19:29:30,992 INFO  [Listener,port=39193 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39193: stopping
2024-11-25T19:29:30,992 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:30,992 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:30,997 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=58 (was 55)
Potentially hanging thread: Relogin-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:216)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6456 (was 6472)
2024-11-25T19:29:31,003 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=57, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6456
2024-11-25T19:29:31,013 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:31,013 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:31,016 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562971016,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:31,018 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:31,019 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33747
2024-11-25T19:29:31,020 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:31,020 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:31,020 INFO  [Listener,port=33747 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33747: starting
2024-11-25T19:29:31,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a500b0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:31,021 DEBUG [Thread-81 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:31,022 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,032 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562971032,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,034 DEBUG [Reader=1,bindAddress=localhost,port=33747 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,035 DEBUG [Reader=1,bindAddress=localhost,port=33747 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,035 DEBUG [Reader=1,bindAddress=localhost,port=33747 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,036 DEBUG [Reader=1,bindAddress=localhost,port=33747 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:31,036 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732562971017
2024-11-25T19:29:31,036 DEBUG [Reader=1,bindAddress=localhost,port=33747 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,036 DEBUG [Reader=1,bindAddress=localhost,port=33747 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,036 DEBUG [Reader=1,bindAddress=localhost,port=33747 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:31,037 DEBUG [Reader=1,bindAddress=localhost,port=33747 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,037 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:31,037 DEBUG [Reader=1,bindAddress=localhost,port=33747 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,037 DEBUG [Reader=1,bindAddress=localhost,port=33747 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,037 DEBUG [Reader=1,bindAddress=localhost,port=33747 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:31,037 INFO  [Reader=1,bindAddress=localhost,port=33747 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:31,038 INFO  [Reader=1,bindAddress=localhost,port=33747 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44440, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:31,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:31,043 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33747
2024-11-25T19:29:31,043 INFO  [Listener,port=33747 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33747: stopping
2024-11-25T19:29:31,044 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:31,044 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:31,044 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:31,045 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40539
2024-11-25T19:29:31,045 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:31,045 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:31,046 INFO  [Listener,port=40539 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40539: starting
2024-11-25T19:29:31,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3620493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:31,046 DEBUG [Thread-84 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:31,048 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,049 DEBUG [Reader=1,bindAddress=localhost,port=40539 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,050 DEBUG [Reader=1,bindAddress=localhost,port=40539 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,050 DEBUG [Reader=1,bindAddress=localhost,port=40539 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,051 DEBUG [Reader=1,bindAddress=localhost,port=40539 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:31,051 DEBUG [Reader=1,bindAddress=localhost,port=40539 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,051 DEBUG [Reader=1,bindAddress=localhost,port=40539 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,051 DEBUG [Reader=1,bindAddress=localhost,port=40539 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:31,052 DEBUG [Reader=1,bindAddress=localhost,port=40539 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,052 DEBUG [Reader=1,bindAddress=localhost,port=40539 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,052 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:31,052 DEBUG [Reader=1,bindAddress=localhost,port=40539 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,052 DEBUG [Reader=1,bindAddress=localhost,port=40539 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:31,052 INFO  [Reader=1,bindAddress=localhost,port=40539 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:31,053 INFO  [Reader=1,bindAddress=localhost,port=40539 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55620, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:31,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:31,059 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40539
2024-11-25T19:29:31,059 INFO  [Listener,port=40539 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40539: stopping
2024-11-25T19:29:31,060 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:31,060 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:31,060 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:31,061 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38947
2024-11-25T19:29:31,062 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:31,062 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:31,062 INFO  [Listener,port=38947 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38947: starting
2024-11-25T19:29:31,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d2657d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:31,063 DEBUG [Thread-87 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:31,064 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,066 DEBUG [Reader=1,bindAddress=localhost,port=38947 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,066 DEBUG [Reader=1,bindAddress=localhost,port=38947 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,066 DEBUG [Reader=1,bindAddress=localhost,port=38947 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,067 DEBUG [Reader=1,bindAddress=localhost,port=38947 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:31,067 DEBUG [Reader=1,bindAddress=localhost,port=38947 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,068 DEBUG [Reader=1,bindAddress=localhost,port=38947 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,068 DEBUG [Reader=1,bindAddress=localhost,port=38947 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:31,068 DEBUG [Reader=1,bindAddress=localhost,port=38947 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,068 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:31,068 DEBUG [Reader=1,bindAddress=localhost,port=38947 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,068 DEBUG [Reader=1,bindAddress=localhost,port=38947 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,068 DEBUG [Reader=1,bindAddress=localhost,port=38947 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:31,068 INFO  [Reader=1,bindAddress=localhost,port=38947 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:31,069 INFO  [Reader=1,bindAddress=localhost,port=38947 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46618, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:31,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:31,074 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38947
2024-11-25T19:29:31,074 INFO  [Listener,port=38947 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38947: stopping
2024-11-25T19:29:31,074 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:31,074 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:31,075 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:31,076 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45127
2024-11-25T19:29:31,076 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:31,076 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:31,076 INFO  [Listener,port=45127 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45127: starting
2024-11-25T19:29:31,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd580cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:31,077 DEBUG [Thread-90 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:31,078 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,080 DEBUG [Reader=1,bindAddress=localhost,port=45127 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,080 DEBUG [Reader=1,bindAddress=localhost,port=45127 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,080 DEBUG [Reader=1,bindAddress=localhost,port=45127 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,081 DEBUG [Reader=1,bindAddress=localhost,port=45127 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:31,082 DEBUG [Reader=1,bindAddress=localhost,port=45127 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,082 DEBUG [Reader=1,bindAddress=localhost,port=45127 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,082 DEBUG [Reader=1,bindAddress=localhost,port=45127 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:31,082 DEBUG [Reader=1,bindAddress=localhost,port=45127 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,082 DEBUG [Reader=1,bindAddress=localhost,port=45127 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,083 DEBUG [Reader=1,bindAddress=localhost,port=45127 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,083 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:31,083 DEBUG [Reader=1,bindAddress=localhost,port=45127 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-25T19:29:31,083 INFO  [Reader=1,bindAddress=localhost,port=45127 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:31,084 INFO  [Reader=1,bindAddress=localhost,port=45127 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:56042, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:31,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:31,090 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45127
2024-11-25T19:29:31,091 INFO  [Listener,port=45127 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45127: stopping
2024-11-25T19:29:31,091 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:31,091 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:31,091 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:31,092 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44969
2024-11-25T19:29:31,092 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:31,092 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:31,093 INFO  [Listener,port=44969 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44969: starting
2024-11-25T19:29:31,093 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fb4f73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:31,093 DEBUG [Thread-93 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:31,095 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,096 DEBUG [Reader=1,bindAddress=localhost,port=44969 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,097 DEBUG [Reader=1,bindAddress=localhost,port=44969 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,097 DEBUG [Reader=1,bindAddress=localhost,port=44969 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,098 DEBUG [Reader=1,bindAddress=localhost,port=44969 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:31,099 DEBUG [Reader=1,bindAddress=localhost,port=44969 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,099 DEBUG [Reader=1,bindAddress=localhost,port=44969 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,099 DEBUG [Reader=1,bindAddress=localhost,port=44969 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:31,099 DEBUG [Reader=1,bindAddress=localhost,port=44969 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,099 DEBUG [Reader=1,bindAddress=localhost,port=44969 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,099 DEBUG [Reader=1,bindAddress=localhost,port=44969 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,099 DEBUG [Reader=1,bindAddress=localhost,port=44969 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:31,099 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:31,100 INFO  [Reader=1,bindAddress=localhost,port=44969 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:31,101 INFO  [Reader=1,bindAddress=localhost,port=44969 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36216, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:31,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:31,110 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44969
2024-11-25T19:29:31,110 INFO  [Listener,port=44969 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44969: stopping
2024-11-25T19:29:31,110 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:31,110 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:31,116 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=62 (was 57)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-8
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.hasTasks(SingleThreadEventExecutor.java:330)
	app//org.apache.hbase.thirdparty.io.netty.channel.SingleThreadEventLoop.hasTasks(SingleThreadEventLoop.java:140)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:525)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6433 (was 6456)
2024-11-25T19:29:31,122 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=62, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6432
2024-11-25T19:29:31,134 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:31,134 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:31,136 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562971136,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:31,138 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:31,139 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41097
2024-11-25T19:29:31,139 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:31,139 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:31,140 INFO  [Listener,port=41097 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41097: starting
2024-11-25T19:29:31,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@559740a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:31,140 DEBUG [Thread-96 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T19:29:31,143 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-25T19:29:31,143 WARN  [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-25T19:29:31,143 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-25T19:29:31,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:31,144 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41097
2024-11-25T19:29:31,144 INFO  [Listener,port=41097 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41097: stopping
2024-11-25T19:29:31,144 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:31,144 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:31,149 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63 (was 62) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6431 (was 6432)
2024-11-25T19:29:31,155 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6431
2024-11-25T19:29:31,166 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:31,166 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:31,168 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562971168,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:31,171 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:31,172 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39783
2024-11-25T19:29:31,172 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-25T19:29:31,172 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-25T19:29:31,173 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-25T19:29:31,173 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-25T19:29:31,173 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-25T19:29:31,173 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-25T19:29:31,173 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-25T19:29:31,173 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-25T19:29:31,173 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:31,174 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:31,174 INFO  [Listener,port=39783 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39783: starting
2024-11-25T19:29:31,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb7e877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:31,178 DEBUG [Thread-99 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T19:29:31,180 WARN  [Reader=1,bindAddress=localhost,port=39783 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-25T19:29:31,180 INFO  [Reader=1,bindAddress=localhost,port=39783 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T19:29:31,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:31,185 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39783
2024-11-25T19:29:31,185 INFO  [Listener,port=39783 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39783: stopping
2024-11-25T19:29:31,186 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:31,186 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:31,192 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64 (was 63) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6429 (was 6431)
2024-11-25T19:29:31,199 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6428
2024-11-25T19:29:31,210 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:31,211 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:31,213 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562971213,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:31,215 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:31,217 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45643
2024-11-25T19:29:31,219 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:31,219 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:31,219 INFO  [Listener,port=45643 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45643: starting
2024-11-25T19:29:31,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@342f9172, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:31,221 DEBUG [Thread-102 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:31,229 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,233 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562971232,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,234 DEBUG [Reader=1,bindAddress=localhost,port=45643 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,235 DEBUG [Reader=1,bindAddress=localhost,port=45643 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,235 DEBUG [Reader=1,bindAddress=localhost,port=45643 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,237 DEBUG [Reader=1,bindAddress=localhost,port=45643 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:31,237 DEBUG [Reader=1,bindAddress=localhost,port=45643 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,237 DEBUG [Reader=1,bindAddress=localhost,port=45643 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,238 DEBUG [Reader=1,bindAddress=localhost,port=45643 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:31,238 DEBUG [Reader=1,bindAddress=localhost,port=45643 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,238 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:31,238 DEBUG [Reader=1,bindAddress=localhost,port=45643 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,238 DEBUG [Reader=1,bindAddress=localhost,port=45643 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,239 DEBUG [Reader=1,bindAddress=localhost,port=45643 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:31,239 INFO  [Reader=1,bindAddress=localhost,port=45643 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:31,239 INFO  [Reader=1,bindAddress=localhost,port=45643 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46474, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:31,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:31,247 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45643
2024-11-25T19:29:31,247 INFO  [Listener,port=45643 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45643: stopping
2024-11-25T19:29:31,247 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:31,249 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:31,254 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=67 (was 64)
Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/java.lang.StringLatin1.inflate(StringLatin1.java:780)
	java.base@17.0.11/java.lang.StringLatin1.getChars(StringLatin1.java:86)
	java.base@17.0.11/java.lang.AbstractStringBuilder.getChars(AbstractStringBuilder.java:512)
	java.base@17.0.11/java.lang.StringBuilder.getChars(StringBuilder.java:91)
	app//org.apache.logging.log4j.core.layout.TextEncoderHelper.encodeText(TextEncoderHelper.java:62)
	app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:68)
	app//org.apache.logging.log4j.core.layout.StringBuilderEncoder.encode(StringBuilderEncoder.java:32)
	app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:227)
	app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199)
	app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158)
	app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
	app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552)
	app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
	app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
	app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/java.lang.ThreadGroup.threadTerminated(ThreadGroup.java:977)
	java.base@17.0.11/java.lang.Thread.exit(Thread.java:853)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-11
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySet.remove(SelectedSelectionKeySet.java:50)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:269)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:112)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6425 (was 6428)
2024-11-25T19:29:31,259 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=6424
2024-11-25T19:29:31,269 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:31,269 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:31,272 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562971271,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:31,273 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:31,274 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43059
2024-11-25T19:29:31,275 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:31,275 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:31,275 INFO  [Listener,port=43059 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43059: starting
2024-11-25T19:29:31,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@425f0fdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:31,276 DEBUG [Thread-105 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:31,278 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,281 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562971281,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,283 DEBUG [Reader=1,bindAddress=localhost,port=43059 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,284 DEBUG [Reader=1,bindAddress=localhost,port=43059 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,284 DEBUG [Reader=1,bindAddress=localhost,port=43059 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,284 DEBUG [Reader=1,bindAddress=localhost,port=43059 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:31,285 DEBUG [Reader=1,bindAddress=localhost,port=43059 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,285 DEBUG [Reader=1,bindAddress=localhost,port=43059 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,285 DEBUG [Reader=1,bindAddress=localhost,port=43059 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:31,285 DEBUG [Reader=1,bindAddress=localhost,port=43059 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,286 DEBUG [Reader=1,bindAddress=localhost,port=43059 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,286 DEBUG [Reader=1,bindAddress=localhost,port=43059 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,286 DEBUG [Reader=1,bindAddress=localhost,port=43059 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:31,286 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:31,286 INFO  [Reader=1,bindAddress=localhost,port=43059 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:31,287 INFO  [Reader=1,bindAddress=localhost,port=43059 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34740, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:31,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:31,296 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43059
2024-11-25T19:29:31,297 INFO  [Listener,port=43059 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43059: stopping
2024-11-25T19:29:31,297 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:31,297 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:31,297 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:31,298 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34271
2024-11-25T19:29:31,301 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:31,301 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:31,301 INFO  [Listener,port=34271 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34271: starting
2024-11-25T19:29:31,301 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de339ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:31,302 DEBUG [Thread-108 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:31,305 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,306 DEBUG [Reader=1,bindAddress=localhost,port=34271 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,307 DEBUG [Reader=1,bindAddress=localhost,port=34271 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,307 DEBUG [Reader=1,bindAddress=localhost,port=34271 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,308 DEBUG [Reader=1,bindAddress=localhost,port=34271 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:31,309 DEBUG [Reader=1,bindAddress=localhost,port=34271 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,309 DEBUG [Reader=1,bindAddress=localhost,port=34271 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,309 DEBUG [Reader=1,bindAddress=localhost,port=34271 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:31,310 DEBUG [Reader=1,bindAddress=localhost,port=34271 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:31,310 DEBUG [Reader=1,bindAddress=localhost,port=34271 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:31,310 DEBUG [Reader=1,bindAddress=localhost,port=34271 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:31,310 DEBUG [Reader=1,bindAddress=localhost,port=34271 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:31,310 INFO  [Reader=1,bindAddress=localhost,port=34271 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:31,310 INFO  [Reader=1,bindAddress=localhost,port=34271 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58510, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:33,313 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T19:29:33,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,313 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:34271 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException
2024-11-25T19:29:33,313 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34271
2024-11-25T19:29:33,314 INFO  [Listener,port=34271 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34271: stopping
2024-11-25T19:29:33,314 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:33,314 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:33,322 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68 (was 65)
Potentially hanging thread: RpcServer.responder
	app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleWritableChannel.write(AbstractNoninterruptibleWritableChannel.java:57)
	app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleWritableChannel.writeBuffered(AbstractNoninterruptibleWritableChannel.java:49)
	app//org.apache.maven.surefire.api.stream.AbstractStreamEncoder.write(AbstractStreamEncoder.java:69)
	app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.write(EventChannelEncoder.java:288)
	app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.setOutErr(EventChannelEncoder.java:173)
	app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.testOutput(EventChannelEncoder.java:168)
	app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:99)
	app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:43)
	app//org.apache.maven.surefire.common.junit4.JUnit4RunListener.writeTestOutput(JUnit4RunListener.java:202)
	app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:57)
	app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250)
	app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283)
	app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199)
	app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158)
	app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
	app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552)
	app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
	app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
	app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=6328 (was 6424)
2024-11-25T19:29:33,329 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=67, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=6328
2024-11-25T19:29:33,342 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:33,343 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:33,345 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562973344,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:33,346 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,347 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732562973346
2024-11-25T19:29:33,347 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44291
2024-11-25T19:29:33,348 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:33,348 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,348 INFO  [Listener,port=44291 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44291: starting
2024-11-25T19:29:33,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205d8cef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,349 DEBUG [Thread-111 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:33,350 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,353 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562973352,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,354 DEBUG [Reader=1,bindAddress=localhost,port=44291 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,355 DEBUG [Reader=1,bindAddress=localhost,port=44291 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:33,355 DEBUG [Reader=1,bindAddress=localhost,port=44291 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,356 DEBUG [Reader=1,bindAddress=localhost,port=44291 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:33,357 DEBUG [Reader=1,bindAddress=localhost,port=44291 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:33,357 DEBUG [Reader=1,bindAddress=localhost,port=44291 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,357 DEBUG [Reader=1,bindAddress=localhost,port=44291 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:33,357 DEBUG [Reader=1,bindAddress=localhost,port=44291 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:33,357 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:33,357 DEBUG [Reader=1,bindAddress=localhost,port=44291 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,358 DEBUG [Reader=1,bindAddress=localhost,port=44291 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,358 DEBUG [Reader=1,bindAddress=localhost,port=44291 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:33,358 INFO  [Reader=1,bindAddress=localhost,port=44291 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:33,358 INFO  [Reader=1,bindAddress=localhost,port=44291 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36204, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:33,364 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,364 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44291
2024-11-25T19:29:33,364 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:33,364 INFO  [Listener,port=44291 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44291: stopping
2024-11-25T19:29:33,364 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:33,370 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69 (was 67)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-14
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.executor(AbstractChannelHandlerContext.java:132)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:193)
	app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelUnregistered(DefaultChannelPipeline.java:770)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:819)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/java.util.IdentityHashMap$IdentityHashMapIterator.hasNext(IdentityHashMap.java:724)
	java.base@17.0.11/jdk.internal.misc.TerminatingThreadLocal.threadTerminated(TerminatingThreadLocal.java:69)
	java.base@17.0.11/java.lang.Thread.exit(Thread.java:850)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=6326 (was 6328)
2024-11-25T19:29:33,375 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=6324
2024-11-25T19:29:33,384 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:33,384 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:33,386 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562973385,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:33,387 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,389 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46653
2024-11-25T19:29:33,389 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,389 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:33,389 INFO  [Listener,port=46653 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46653: starting
2024-11-25T19:29:33,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@742563ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,390 DEBUG [Thread-114 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:33,391 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,394 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562973394,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,395 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,396 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:33,396 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,397 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-25T19:29:33,398 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:33,398 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,399 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-25T19:29:33,399 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-25T19:29:33,399 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:33,399 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,399 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,399 DEBUG [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:33,399 INFO  [Reader=1,bindAddress=localhost,port=46653 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:33,400 INFO  [Reader=1,bindAddress=localhost,port=46653 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55430, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:33,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,407 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46653
2024-11-25T19:29:33,407 INFO  [Listener,port=46653 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46653: stopping
2024-11-25T19:29:33,407 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:33,407 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:33,414 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69 (was 68) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=6321 (was 6324)
2024-11-25T19:29:33,421 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=6320
2024-11-25T19:29:33,432 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:33,432 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:33,434 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562973434,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:33,436 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,437 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45731
2024-11-25T19:29:33,439 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-25T19:29:33,439 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,439 INFO  [Listener,port=45731 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45731: starting
2024-11-25T19:29:33,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68258e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,440 DEBUG [Thread-117 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:33,442 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,445 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562973444,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,446 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:33,447 INFO  [Reader=1,bindAddress=localhost,port=45731 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T19:29:33,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,455 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45731
2024-11-25T19:29:33,455 INFO  [Listener,port=45731 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45731: stopping
2024-11-25T19:29:33,455 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-25T19:29:33,455 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-25T19:29:33,463 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=73 (was 69)
Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.getAndAddInt(Unsafe.java:2444)
	java.base@17.0.11/java.util.concurrent.atomic.AtomicInteger.addAndGet(AtomicInteger.java:241)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.decrementWorkerCount(ThreadPoolExecutor.java:438)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-16
	java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.close0(Native Method)
	java.base@17.0.11/sun.nio.ch.SocketDispatcher.close(SocketDispatcher.java:70)
	java.base@17.0.11/sun.nio.ch.SocketChannelImpl.tryClose(SocketChannelImpl.java:980)
	java.base@17.0.11/sun.nio.ch.SocketChannelImpl.tryFinishClose(SocketChannelImpl.java:994)
	java.base@17.0.11/sun.nio.ch.SocketChannelImpl.kill(SocketChannelImpl.java:1088)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:277)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:112)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=45731
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=6313 (was 6320)
2024-11-25T19:29:33,470 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=70, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=12, AvailableMemoryMB=6312
2024-11-25T19:29:33,481 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:33,482 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:33,484 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562973483,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:33,486 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,486 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:33,487 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34569
2024-11-25T19:29:33,487 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a89e41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,489 DEBUG [Thread-118 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:33,490 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,493 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562973493,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,495 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,495 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,497 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:33,498 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,499 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:33,499 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,499 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,499 DEBUG [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:33,500 INFO  [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:33,501 INFO  [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55602, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:33,502 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:33,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,511 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34569
2024-11-25T19:29:33,519 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72 (was 70)
Potentially hanging thread: NettyRpcServer-19-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-19-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=475 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 12), AvailableMemoryMB=6306 (was 6312)
2024-11-25T19:29:33,528 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72, OpenFileDescriptor=475, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=6303
2024-11-25T19:29:33,540 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:33,540 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:33,543 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562973543,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:33,546 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,546 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:33,547 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43037
2024-11-25T19:29:33,547 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a2cbdfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-25T19:29:33,548 DEBUG [Thread-119 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:33,550 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,554 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562973553,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,556 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T19:29:33,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,556 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:43037 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
2024-11-25T19:29:33,556 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43037
2024-11-25T19:29:33,565 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74 (was 72)
Potentially hanging thread: NettyRpcServer-20-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-20-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=484 (was 475) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=6300 (was 6303)
2024-11-25T19:29:33,573 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74, OpenFileDescriptor=484, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=6299
2024-11-25T19:29:33,589 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:33,590 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:33,592 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562973592,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:33,594 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,595 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:33,597 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46365
2024-11-25T19:29:33,597 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3467a00c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,598 DEBUG [Thread-120 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:33,599 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,602 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562973602,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,604 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,604 DEBUG [NettyRpcServer-21-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,605 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:33,606 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,606 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:33,606 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T19:29:33,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,606 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:46365 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server
2024-11-25T19:29:33,606 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46365
2024-11-25T19:29:33,612 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76 (was 74)
Potentially hanging thread: NettyRpcServer-21-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-21-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=493 (was 484) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=6294 (was 6299)
2024-11-25T19:29:33,618 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76, OpenFileDescriptor=493, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=6294
2024-11-25T19:29:33,627 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:33,627 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:33,628 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732562973593
2024-11-25T19:29:33,629 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562973629,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:33,630 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,631 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:33,631 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42247
2024-11-25T19:29:33,631 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4663edc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,632 DEBUG [Thread-121 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:33,633 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,635 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562973635,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,636 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,636 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,638 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:33,638 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,638 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:33,639 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,639 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:33,639 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,639 DEBUG [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:33,639 INFO  [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:33,639 INFO  [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41596, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:33,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,645 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42247
2024-11-25T19:29:33,646 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,646 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:33,647 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39541
2024-11-25T19:29:33,647 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c1e8b88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,674 DEBUG [Thread-122 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:33,675 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,677 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,678 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,679 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:33,680 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,681 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:33,681 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:33,681 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,681 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,682 DEBUG [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:33,682 INFO  [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:33,682 INFO  [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50016, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:33,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,690 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39541
2024-11-25T19:29:33,690 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,691 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:33,692 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:39709
2024-11-25T19:29:33,692 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9d5881, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,692 DEBUG [Thread-123 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:33,693 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,695 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,695 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,696 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:33,696 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,696 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:33,697 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,697 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:33,697 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,697 DEBUG [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:33,697 INFO  [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:33,697 INFO  [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34890, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:33,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,704 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:39709
2024-11-25T19:29:33,704 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,705 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:33,705 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44631
2024-11-25T19:29:33,705 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c73e0fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,706 DEBUG [Thread-124 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:33,707 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,708 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,709 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,710 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:33,711 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,711 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:33,711 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,711 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:33,711 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,711 DEBUG [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-25T19:29:33,712 INFO  [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:33,712 INFO  [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51664, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:33,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,719 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44631
2024-11-25T19:29:33,720 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,720 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:33,721 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43935
2024-11-25T19:29:33,721 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e564d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,722 DEBUG [Thread-125 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:33,723 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,724 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,724 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,726 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:33,726 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,726 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:33,727 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,727 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,727 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:33,727 DEBUG [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:33,727 INFO  [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:33,728 INFO  [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60438, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:33,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,736 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43935
2024-11-25T19:29:33,743 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86 (was 76)
Potentially hanging thread: NettyRpcServer-25-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-23-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-25-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-24-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-24-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-22-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-26-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-26-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-23-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-22-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=538 (was 493) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=6284 (was 6294)
2024-11-25T19:29:33,748 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86, OpenFileDescriptor=538, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=6284
2024-11-25T19:29:33,758 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:33,758 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:33,760 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562973760,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:33,762 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,762 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:33,763 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40371
2024-11-25T19:29:33,763 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70bb2051, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,764 DEBUG [Thread-126 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T19:29:33,765 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-25T19:29:33,765 WARN  [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-25T19:29:33,766 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-25T19:29:33,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,766 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40371
2024-11-25T19:29:33,772 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88 (was 86)
Potentially hanging thread: NettyRpcServer-27-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-27-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=547 (was 538) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=6280 (was 6284)
2024-11-25T19:29:33,778 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88, OpenFileDescriptor=547, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=6280
2024-11-25T19:29:33,790 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:33,790 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:33,792 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562973792,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:33,793 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,794 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:33,794 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35345
2024-11-25T19:29:33,794 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-25T19:29:33,794 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-25T19:29:33,795 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-25T19:29:33,795 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-25T19:29:33,795 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-25T19:29:33,795 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-25T19:29:33,795 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-25T19:29:33,795 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-25T19:29:33,795 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492f2d14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,796 DEBUG [Thread-127 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-25T19:29:33,797 WARN  [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-25T19:29:33,797 INFO  [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T19:29:33,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,803 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35345
2024-11-25T19:29:33,809 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90 (was 88)
Potentially hanging thread: NettyRpcServer-28-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-28-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=556 (was 547) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=6278 (was 6280)
2024-11-25T19:29:33,815 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90, OpenFileDescriptor=556, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=6277
2024-11-25T19:29:33,827 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:33,827 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:33,829 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562973829,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:33,831 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,831 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:33,832 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37331
2024-11-25T19:29:33,832 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1589373e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,833 DEBUG [Thread-128 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:33,834 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,836 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562973836,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,838 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,838 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,839 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:33,840 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,840 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:33,840 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,840 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:33,840 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,840 DEBUG [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:33,840 INFO  [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:33,841 INFO  [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:32908, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:33,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,847 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37331
2024-11-25T19:29:33,854 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92 (was 90)
Potentially hanging thread: NettyRpcServer-29-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-29-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=565 (was 556) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=6274 (was 6277)
2024-11-25T19:29:33,860 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92, OpenFileDescriptor=565, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=6274
2024-11-25T19:29:33,872 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:33,872 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:33,874 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562973874,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:33,876 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,876 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:33,877 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42459
2024-11-25T19:29:33,877 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a5633cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,878 DEBUG [Thread-129 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:33,879 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,881 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562973881,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,882 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,882 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,884 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:33,884 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,884 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:33,885 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,885 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,885 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:33,885 DEBUG [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:33,885 INFO  [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:33,886 INFO  [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35516, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:33,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:33,894 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42459
2024-11-25T19:29:33,895 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:33,895 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:33,896 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:32781
2024-11-25T19:29:33,896 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:33,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5681b81a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:33,897 DEBUG [Thread-130 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:33,899 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,900 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,900 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,902 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:33,902 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,902 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:33,903 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:33,903 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:33,903 DEBUG [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-25T19:29:33,903 INFO  [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:33,904 INFO  [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44970, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:35,905 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-25T19:29:35,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:35,905 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:32781 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException
2024-11-25T19:29:35,906 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:32781
2024-11-25T19:29:35,913 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96 (was 92)
Potentially hanging thread: NettyRpcServer-30-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-30-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-31-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-31-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=583 (was 565) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=6235 (was 6274)
2024-11-25T19:29:35,919 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96, OpenFileDescriptor=583, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=6235
2024-11-25T19:29:35,929 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:35,930 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:35,932 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562975932,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:35,934 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:35,934 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:35,935 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44781
2024-11-25T19:29:35,936 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:35,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35437ead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:35,937 DEBUG [Thread-131 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:35,937 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:35,940 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562975940,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:35,942 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:35,942 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:35,944 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:35,944 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:35,944 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:35,945 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:35,945 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:35,945 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:35,945 DEBUG [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:35,945 INFO  [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:35,946 INFO  [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33908, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:35,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:35,952 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44781
2024-11-25T19:29:35,960 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98 (was 96)
Potentially hanging thread: NettyRpcServer-32-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-32-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=592 (was 583) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=6231 (was 6235)
2024-11-25T19:29:35,973 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98, OpenFileDescriptor=592, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=6229
2024-11-25T19:29:35,985 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:35,985 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:35,987 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562975987,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:35,989 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:35,989 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:35,990 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42845
2024-11-25T19:29:35,990 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:35,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b59cb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:35,991 DEBUG [Thread-132 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:35,992 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:35,995 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562975995,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:35,996 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:35,996 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:35,997 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-25T19:29:35,998 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:35,998 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-25T19:29:35,998 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-25T19:29:35,998 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:35,999 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:35,999 DEBUG [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-25T19:29:35,999 INFO  [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-25T19:29:35,999 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732562975988
2024-11-25T19:29:35,999 INFO  [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51734, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-25T19:29:36,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:36,005 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42845
2024-11-25T19:29:36,013 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100 (was 98)
Potentially hanging thread: NettyRpcServer-33-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-33-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=601 (was 592) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=6222 (was 6229)
2024-11-25T19:29:36,019 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100, OpenFileDescriptor=601, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=6220
2024-11-25T19:29:36,028 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-25T19:29:36,028 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-25T19:29:36,030 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732562976030,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-25T19:29:36,032 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-25T19:29:36,032 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-25T19:29:36,033 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34565
2024-11-25T19:29:36,033 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-25T19:29:36,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc4213e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-25T19:29:36,035 DEBUG [Thread-133 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-25T19:29:36,036 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:36,040 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732562976039,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-25T19:29:36,045 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-25T19:29:36,047 INFO  [NettyRpcServer-34-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-25T19:29:36,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-25T19:29:36,055 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34565
2024-11-25T19:29:36,067 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=102 (was 100)
Potentially hanging thread: NettyRpcServer-34-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-34-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=610 (was 601) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=6214 (was 6220)
2024-11-25T19:29:36,068 INFO  [Time-limited test {}] impl.DefaultInternalKdcServerImpl(102): Default Internal kdc server stopped.