2024-11-22 03:14:06,250 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a7f33
2024-11-22 03:14:06,312 main DEBUG Took 0.052879 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-22 03:14:06,313 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-22 03:14:06,313 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-22 03:14:06,314 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-22 03:14:06,317 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 03:14:06,334 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-22 03:14:06,360 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 03:14:06,363 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 03:14:06,366 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 03:14:06,366 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 03:14:06,367 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 03:14:06,368 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 03:14:06,369 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 03:14:06,369 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 03:14:06,370 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 03:14:06,370 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 03:14:06,372 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 03:14:06,373 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 03:14:06,374 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 03:14:06,374 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 03:14:06,375 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 03:14:06,375 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 03:14:06,376 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 03:14:06,377 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 03:14:06,377 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 03:14:06,378 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 03:14:06,378 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 03:14:06,379 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 03:14:06,380 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 03:14:06,383 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 03:14:06,385 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 03:14:06,385 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-22 03:14:06,388 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 03:14:06,390 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-22 03:14:06,394 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-22 03:14:06,401 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-22 03:14:06,403 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-22 03:14:06,405 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-22 03:14:06,419 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-22 03:14:06,423 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-22 03:14:06,425 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-22 03:14:06,426 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-22 03:14:06,427 main DEBUG createAppenders(={Console})
2024-11-22 03:14:06,428 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a7f33 initialized
2024-11-22 03:14:06,430 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a7f33
2024-11-22 03:14:06,431 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b7a7f33 OK.
2024-11-22 03:14:06,432 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-22 03:14:06,433 main DEBUG OutputStream closed
2024-11-22 03:14:06,435 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-22 03:14:06,435 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-22 03:14:06,436 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@485966cc OK
2024-11-22 03:14:06,575 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-22 03:14:06,579 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-22 03:14:06,582 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-22 03:14:06,584 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-22 03:14:06,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-22 03:14:06,589 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-22 03:14:06,590 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-22 03:14:06,591 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-22 03:14:06,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-22 03:14:06,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-22 03:14:06,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-22 03:14:06,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-22 03:14:06,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-22 03:14:06,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-22 03:14:06,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-22 03:14:06,600 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-22 03:14:06,601 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-22 03:14:06,602 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-22 03:14:06,609 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-22 03:14:06,609 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4426bff1) with optional ClassLoader: null
2024-11-22 03:14:06,610 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-22 03:14:06,611 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4426bff1] started OK.
2024-11-22T03:14:07,042 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbfc7c-1394-b19c-be5c-2abef6764095
2024-11-22 03:14:07,055 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-22 03:14:07,056 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-22T03:14:07,093 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.security.TestSecureIPC timeout: 52 mins
2024-11-22T03:14:07,534 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=13, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=734, ProcessCount=11, AvailableMemoryMB=6545
2024-11-22T03:14:07,649 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-22T03:14:07,729 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:07,734 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:07,814 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245247810,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:09,078 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:09,208 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-22T03:14:09,270 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-22T03:14:09,284 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-22T03:14:09,309 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35169
2024-11-22T03:14:09,316 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:09,317 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:09,317 INFO  [Listener,port=35169 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35169: starting
2024-11-22T03:14:09,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61d2771c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:09,453 DEBUG [Thread-6 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:09,537 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35169
2024-11-22T03:14:09,549 DEBUG [Thread-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:09,651 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245249649,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:09,676 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:09,684 DEBUG [Reader=1,bindAddress=localhost,port=35169 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:09,700 DEBUG [Reader=1,bindAddress=localhost,port=35169 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:09,701 DEBUG [Reader=1,bindAddress=localhost,port=35169 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:09,754 DEBUG [Reader=1,bindAddress=localhost,port=35169 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:09,757 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:09,759 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:09,762 DEBUG [Reader=1,bindAddress=localhost,port=35169 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:09,762 DEBUG [Reader=1,bindAddress=localhost,port=35169 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:09,765 DEBUG [Reader=1,bindAddress=localhost,port=35169 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:09,766 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:09,767 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:09,767 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-22T03:14:09,768 DEBUG [Reader=1,bindAddress=localhost,port=35169 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:09,768 DEBUG [Thread-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:09,771 DEBUG [Reader=1,bindAddress=localhost,port=35169 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:09,774 DEBUG [Reader=1,bindAddress=localhost,port=35169 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:09,775 DEBUG [Reader=1,bindAddress=localhost,port=35169 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:09,776 INFO  [Reader=1,bindAddress=localhost,port=35169 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:09,841 INFO  [Reader=1,bindAddress=localhost,port=35169 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50948, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:09,842 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93
2024-11-22T03:14:09,853 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:250
2024-11-22T03:14:09,857 DEBUG [BRPC Connection (486228291) to localhost:35169 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-22T03:14:09,910 DEBUG [BRPC Connection (486228291) to localhost:35169 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:236
2024-11-22T03:14:09,919 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2243
2024-11-22T03:14:09,923 DEBUG [BRPC Connection (486228291) to localhost:35169 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-22T03:14:09,929 DEBUG [BRPC Connection (486228291) to localhost:35169 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2229
2024-11-22T03:14:09,951 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117
2024-11-22T03:14:09,953 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18264
2024-11-22T03:14:09,964 DEBUG [BRPC Connection (486228291) to localhost:35169 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-22T03:14:09,985 DEBUG [BRPC Connection (486228291) to localhost:35169 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22367
2024-11-22T03:14:09,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:09,993 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35169
2024-11-22T03:14:09,993 INFO  [Listener,port=35169 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35169: stopping
2024-11-22T03:14:09,997 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:09,997 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:10,003 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=22 (was 13)
Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=1,bindAddress=localhost,port=35169
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:128)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer idle connection scanner for port 35169
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: pool-3-thread-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=292 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=787 (was 734) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6420 (was 6545)
2024-11-22T03:14:10,011 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=787, ProcessCount=11, AvailableMemoryMB=6419
2024-11-22T03:14:10,031 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:10,031 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:10,045 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245250044,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:10,068 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:10,080 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41061
2024-11-22T03:14:10,084 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:10,088 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:10,092 INFO  [Listener,port=41061 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41061: starting
2024-11-22T03:14:10,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23de2825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-22T03:14:10,101 DEBUG [Thread-11 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:10,102 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:41061
2024-11-22T03:14:10,108 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,126 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245250125,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,129 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:10,130 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-22T03:14:10,132 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:41061
org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-22T03:14:10,137 WARN  [Thread-11 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732245250048
2024-11-22T03:14:10,237 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,240 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:10,240 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-22T03:14:10,241 DEBUG [Thread-11 {}] ipc.FailedServers(52): Added failed server with address localhost:41061 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:10,243 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41061
2024-11-22T03:14:10,243 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:10,243 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:10,244 INFO  [Listener,port=41061 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41061: stopping
2024-11-22T03:14:10,250 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=22 (was 19)
Potentially hanging thread: Reader=1,bindAddress=localhost,port=41061
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Listener,port=41061
	app//org.apache.logging.log4j.spi.AbstractLogger.decrementRecursionDepth(AbstractLogger.java:2183)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2161)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:252)

Potentially hanging thread: RpcServer idle connection scanner for port 41061
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
 - Thread LEAK? -, OpenFileDescriptor=292 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=787 (was 787), ProcessCount=11 (was 11), AvailableMemoryMB=6409 (was 6419)
2024-11-22T03:14:10,257 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=787, ProcessCount=11, AvailableMemoryMB=6408
2024-11-22T03:14:10,274 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:10,274 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:10,282 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245250281,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:10,288 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:10,291 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42487
2024-11-22T03:14:10,292 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:10,294 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:10,294 INFO  [Listener,port=42487 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42487: starting
2024-11-22T03:14:10,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b54b404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:10,300 DEBUG [Thread-14 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:10,301 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42487
2024-11-22T03:14:10,314 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,329 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245250329,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,332 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:10,333 DEBUG [Reader=1,bindAddress=localhost,port=42487 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,334 DEBUG [Reader=1,bindAddress=localhost,port=42487 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,335 DEBUG [Reader=1,bindAddress=localhost,port=42487 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,337 DEBUG [Reader=1,bindAddress=localhost,port=42487 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:10,337 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:10,338 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:10,338 DEBUG [Reader=1,bindAddress=localhost,port=42487 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,339 DEBUG [Reader=1,bindAddress=localhost,port=42487 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,339 DEBUG [Reader=1,bindAddress=localhost,port=42487 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:10,339 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:10,340 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-22T03:14:10,340 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:42487
javax.security.sasl.SaslException: No common protection layer between client and server
	at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?]
	at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?]
	at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-22T03:14:10,341 WARN  [Thread-14 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732245250285
2024-11-22T03:14:10,440 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,443 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:10,443 DEBUG [Reader=2,bindAddress=localhost,port=42487 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,445 DEBUG [Reader=2,bindAddress=localhost,port=42487 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,445 DEBUG [Reader=2,bindAddress=localhost,port=42487 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,448 DEBUG [Reader=2,bindAddress=localhost,port=42487 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:10,450 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:10,450 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:10,451 DEBUG [Reader=2,bindAddress=localhost,port=42487 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,451 DEBUG [Reader=2,bindAddress=localhost,port=42487 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,452 DEBUG [Reader=2,bindAddress=localhost,port=42487 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:10,452 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:10,453 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-22T03:14:10,453 DEBUG [Thread-14 {}] ipc.FailedServers(52): Added failed server with address localhost:42487 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:10,454 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42487
2024-11-22T03:14:10,454 INFO  [Listener,port=42487 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42487: stopping
2024-11-22T03:14:10,454 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:10,455 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:10,470 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=21 (was 19)
Potentially hanging thread: Reader=2,bindAddress=localhost,port=42487
	java.base@17.0.11/sun.nio.ch.EPoll.getEvent(EPoll.java:87)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:187)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:135)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	app//org.apache.logging.log4j.core.pattern.DatePatternConverter.formatWithoutThreadLocals(DatePatternConverter.java:318)
	app//org.apache.logging.log4j.core.pattern.DatePatternConverter.format(DatePatternConverter.java:290)
	app//org.apache.logging.log4j.core.pattern.DatePatternConverter.format(DatePatternConverter.java:265)
	app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44)
	app//org.apache.logging.log4j.core.layout.PatternLayout$PatternFormatterPatternSerializer.toSerializable(PatternLayout.java:384)
	app//org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:240)
	app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:225)
	app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:59)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:215)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199)
	app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158)
	app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
	app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552)
	app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
	app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
	app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=787 (was 787), ProcessCount=11 (was 11), AvailableMemoryMB=6406 (was 6408)
2024-11-22T03:14:10,483 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=787, ProcessCount=11, AvailableMemoryMB=6406
2024-11-22T03:14:10,500 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:10,500 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:10,507 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245250507,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:10,513 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:10,520 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42959
2024-11-22T03:14:10,524 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:10,524 INFO  [Listener,port=42959 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42959: starting
2024-11-22T03:14:10,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1bd5bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:10,526 DEBUG [Thread-17 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:10,527 DEBUG [Thread-17 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42959
2024-11-22T03:14:10,524 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:10,528 DEBUG [Thread-17 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,537 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245250536,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,553 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:10,554 DEBUG [Reader=1,bindAddress=localhost,port=42959 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,557 DEBUG [Reader=1,bindAddress=localhost,port=42959 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,557 DEBUG [Reader=1,bindAddress=localhost,port=42959 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,561 DEBUG [Reader=1,bindAddress=localhost,port=42959 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:10,561 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:10,562 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:10,562 DEBUG [Reader=1,bindAddress=localhost,port=42959 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,563 DEBUG [Reader=1,bindAddress=localhost,port=42959 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,563 DEBUG [Reader=1,bindAddress=localhost,port=42959 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:10,563 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:10,564 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:10,564 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-22T03:14:10,564 DEBUG [Reader=1,bindAddress=localhost,port=42959 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,564 DEBUG [Thread-17 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:10,564 DEBUG [Reader=1,bindAddress=localhost,port=42959 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,565 DEBUG [Reader=1,bindAddress=localhost,port=42959 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,565 DEBUG [Reader=1,bindAddress=localhost,port=42959 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:10,565 INFO  [Reader=1,bindAddress=localhost,port=42959 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:10,565 INFO  [Reader=1,bindAddress=localhost,port=42959 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43562, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:10,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:10,609 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42959
2024-11-22T03:14:10,610 INFO  [Listener,port=42959 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42959: stopping
2024-11-22T03:14:10,610 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:10,610 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:10,611 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:10,614 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46845
2024-11-22T03:14:10,617 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:10,620 INFO  [Listener,port=46845 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46845: starting
2024-11-22T03:14:10,620 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:10,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ac6ab4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:10,624 DEBUG [Thread-20 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:10,628 DEBUG [Thread-20 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46845
2024-11-22T03:14:10,630 DEBUG [Thread-20 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,632 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:10,633 DEBUG [Reader=1,bindAddress=localhost,port=46845 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,634 DEBUG [Reader=1,bindAddress=localhost,port=46845 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,634 DEBUG [Reader=1,bindAddress=localhost,port=46845 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,637 DEBUG [Reader=1,bindAddress=localhost,port=46845 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:10,637 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:10,638 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:10,638 DEBUG [Reader=1,bindAddress=localhost,port=46845 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,639 DEBUG [Reader=1,bindAddress=localhost,port=46845 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,639 DEBUG [Reader=1,bindAddress=localhost,port=46845 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:10,639 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:10,640 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:10,640 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-22T03:14:10,640 DEBUG [Reader=1,bindAddress=localhost,port=46845 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,640 DEBUG [Thread-20 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:10,640 DEBUG [Reader=1,bindAddress=localhost,port=46845 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,641 DEBUG [Reader=1,bindAddress=localhost,port=46845 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,641 DEBUG [Reader=1,bindAddress=localhost,port=46845 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:10,641 INFO  [Reader=1,bindAddress=localhost,port=46845 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:10,642 INFO  [Reader=1,bindAddress=localhost,port=46845 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46192, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:10,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:10,675 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46845
2024-11-22T03:14:10,675 INFO  [Listener,port=46845 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46845: stopping
2024-11-22T03:14:10,677 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:10,677 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:10,677 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:10,681 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=37555
2024-11-22T03:14:10,682 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:10,682 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:10,682 INFO  [Listener,port=37555 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=37555: starting
2024-11-22T03:14:10,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a42437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:10,684 DEBUG [Thread-23 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:10,684 DEBUG [Thread-23 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37555
2024-11-22T03:14:10,686 DEBUG [Thread-23 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,688 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:10,688 DEBUG [Reader=1,bindAddress=localhost,port=37555 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,689 DEBUG [Reader=1,bindAddress=localhost,port=37555 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,689 DEBUG [Reader=1,bindAddress=localhost,port=37555 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,691 DEBUG [Reader=1,bindAddress=localhost,port=37555 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:10,692 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:10,692 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:10,693 DEBUG [Reader=1,bindAddress=localhost,port=37555 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,693 DEBUG [Reader=1,bindAddress=localhost,port=37555 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,693 DEBUG [Reader=1,bindAddress=localhost,port=37555 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:10,693 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:10,694 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:10,694 DEBUG [Reader=1,bindAddress=localhost,port=37555 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,694 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-22T03:14:10,694 DEBUG [Reader=1,bindAddress=localhost,port=37555 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,694 DEBUG [Thread-23 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:10,696 DEBUG [Reader=1,bindAddress=localhost,port=37555 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,696 DEBUG [Reader=1,bindAddress=localhost,port=37555 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:10,697 INFO  [Reader=1,bindAddress=localhost,port=37555 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:10,697 INFO  [Reader=1,bindAddress=localhost,port=37555 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37544, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:10,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:10,744 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 37555
2024-11-22T03:14:10,744 INFO  [Listener,port=37555 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=37555: stopping
2024-11-22T03:14:10,745 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:10,745 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:10,745 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:10,750 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36095
2024-11-22T03:14:10,751 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:10,751 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:10,751 INFO  [Listener,port=36095 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36095: starting
2024-11-22T03:14:10,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3acf87b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:10,754 DEBUG [Thread-26 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:10,754 DEBUG [Thread-26 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36095
2024-11-22T03:14:10,755 DEBUG [Thread-26 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,758 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:10,758 DEBUG [Reader=1,bindAddress=localhost,port=36095 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,760 DEBUG [Reader=1,bindAddress=localhost,port=36095 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,760 DEBUG [Reader=1,bindAddress=localhost,port=36095 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,761 DEBUG [Reader=1,bindAddress=localhost,port=36095 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:10,765 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:10,766 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:10,766 DEBUG [Reader=1,bindAddress=localhost,port=36095 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,766 DEBUG [Reader=1,bindAddress=localhost,port=36095 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,767 DEBUG [Reader=1,bindAddress=localhost,port=36095 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:10,768 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:10,768 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:10,768 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int
2024-11-22T03:14:10,769 DEBUG [Reader=1,bindAddress=localhost,port=36095 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,769 DEBUG [Thread-26 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:10,769 DEBUG [Reader=1,bindAddress=localhost,port=36095 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,769 DEBUG [Reader=1,bindAddress=localhost,port=36095 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,769 DEBUG [Reader=1,bindAddress=localhost,port=36095 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-22T03:14:10,769 INFO  [Reader=1,bindAddress=localhost,port=36095 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:10,770 INFO  [Reader=1,bindAddress=localhost,port=36095 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48386, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:10,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:10,788 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36095
2024-11-22T03:14:10,788 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:10,788 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:10,789 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:10,790 INFO  [Listener,port=36095 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36095: stopping
2024-11-22T03:14:10,796 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45233
2024-11-22T03:14:10,797 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:10,797 INFO  [Listener,port=45233 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45233: starting
2024-11-22T03:14:10,797 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:10,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f55e400, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:10,802 DEBUG [Thread-29 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:10,803 DEBUG [Thread-29 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45233
2024-11-22T03:14:10,804 DEBUG [Thread-29 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,809 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:10,809 DEBUG [Reader=1,bindAddress=localhost,port=45233 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,810 DEBUG [Reader=1,bindAddress=localhost,port=45233 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,810 DEBUG [Reader=1,bindAddress=localhost,port=45233 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,811 DEBUG [Reader=1,bindAddress=localhost,port=45233 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:10,812 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:10,816 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:10,816 DEBUG [Reader=1,bindAddress=localhost,port=45233 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,817 DEBUG [Reader=1,bindAddress=localhost,port=45233 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,817 DEBUG [Reader=1,bindAddress=localhost,port=45233 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:10,817 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:10,818 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:10,818 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-22T03:14:10,818 DEBUG [Reader=1,bindAddress=localhost,port=45233 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:10,818 DEBUG [Thread-29 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:10,818 DEBUG [Reader=1,bindAddress=localhost,port=45233 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:10,818 DEBUG [Reader=1,bindAddress=localhost,port=45233 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:10,819 DEBUG [Reader=1,bindAddress=localhost,port=45233 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:10,819 INFO  [Reader=1,bindAddress=localhost,port=45233 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:10,819 INFO  [Reader=1,bindAddress=localhost,port=45233 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55064, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:10,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:10,849 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45233
2024-11-22T03:14:10,849 INFO  [Listener,port=45233 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45233: stopping
2024-11-22T03:14:10,849 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:10,850 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:10,867 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=23 (was 19)
Potentially hanging thread: Reader=1,bindAddress=localhost,port=45233
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/jdk.internal.misc.Unsafe.allocateInstance(Native Method)
	java.base@17.0.11/java.lang.invoke.DirectMethodHandle.allocateInstance(DirectMethodHandle.java:520)
	java.base@17.0.11/java.lang.invoke.DirectMethodHandle$Holder.newInvokeSpecial(DirectMethodHandle$Holder)
	java.base@17.0.11/java.lang.invoke.Invokers$Holder.linkToTargetMethod(Invokers$Holder)
	app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:98)
	app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007faa8818c260.apply(Unknown Source)
	java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243)
	java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499)
	app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97)
	app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121)
	app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=45233
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=293 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=787 (was 787), ProcessCount=11 (was 11), AvailableMemoryMB=6389 (was 6406)
2024-11-22T03:14:10,873 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=787, ProcessCount=11, AvailableMemoryMB=6389
2024-11-22T03:14:10,889 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:10,890 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:10,898 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245250898,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:10,904 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:10,912 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43563
2024-11-22T03:14:10,920 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:10,924 INFO  [Listener,port=43563 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43563: starting
2024-11-22T03:14:10,924 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:10,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5325ba19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:10,927 DEBUG [Thread-32 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-22T03:14:10,927 DEBUG [Thread-32 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43563
2024-11-22T03:14:10,931 DEBUG [BRPC Connection (486228291) to localhost:43563 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-22T03:14:10,931 WARN  [BRPC Connection (486228291) to localhost:43563 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-22T03:14:10,937 DEBUG [BRPC Connection (486228291) to localhost:43563 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-22T03:14:10,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:10,939 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43563
2024-11-22T03:14:10,939 INFO  [Listener,port=43563 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43563: stopping
2024-11-22T03:14:10,939 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:10,939 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:10,954 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19)
Potentially hanging thread: Reader=0,bindAddress=localhost,port=43563
	java.base@17.0.11/jdk.internal.misc.Unsafe.getIntVolatile(Native Method)
	java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndExchangeByte(Unsafe.java:1532)
	java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndSetByte(Unsafe.java:1544)
	java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndSetBoolean(Unsafe.java:1784)
	java.base@17.0.11/java.lang.invoke.VarHandleBooleans$FieldInstanceReadWrite.compareAndSet(VarHandleBooleans.java:178)
	java.base@17.0.11/java.lang.invoke.VarHandleGuards.guard_LII_Z(VarHandleGuards.java:137)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:125)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:151)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=787 (was 787), ProcessCount=11 (was 11), AvailableMemoryMB=6379 (was 6389)
2024-11-22T03:14:10,966 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=787, ProcessCount=11, AvailableMemoryMB=6379
2024-11-22T03:14:10,983 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:10,989 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:10,999 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245250999,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:11,003 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:11,016 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44757
2024-11-22T03:14:11,020 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-22T03:14:11,020 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-22T03:14:11,021 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-22T03:14:11,021 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-22T03:14:11,021 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-22T03:14:11,021 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-22T03:14:11,021 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-22T03:14:11,021 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-22T03:14:11,023 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:11,023 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:11,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@859e3aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:11,024 INFO  [Listener,port=44757 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44757: starting
2024-11-22T03:14:11,026 DEBUG [Thread-35 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-22T03:14:11,026 DEBUG [Thread-35 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44757
2024-11-22T03:14:11,027 WARN  [Reader=1,bindAddress=localhost,port=44757 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-22T03:14:11,028 INFO  [Reader=1,bindAddress=localhost,port=44757 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-22T03:14:11,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:11,061 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44757
2024-11-22T03:14:11,061 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:11,061 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:11,064 INFO  [Listener,port=44757 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44757: stopping
2024-11-22T03:14:11,075 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=23 (was 19)
Potentially hanging thread: Listener,port=44757
	java.base@17.0.11/sun.nio.ch.EPoll.ctl(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implDereg(EPollSelectorImpl.java:230)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:267)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:134)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:213)

Potentially hanging thread: RpcServer idle connection scanner for port 44757
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=44757
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: BRPC Connection (486228291) to localhost:44757 from testuser
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.hbase.ipc.BlockingRpcConnection.waitForWork(BlockingRpcConnection.java:329)
	app//org.apache.hadoop.hbase.ipc.BlockingRpcConnection.run(BlockingRpcConnection.java:342)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=787 (was 787), ProcessCount=11 (was 11), AvailableMemoryMB=6368 (was 6379)
2024-11-22T03:14:11,082 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=787, ProcessCount=11, AvailableMemoryMB=6377
2024-11-22T03:14:11,099 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:11,100 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:11,106 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245251105,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:11,109 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:11,124 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33011
2024-11-22T03:14:11,127 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:11,128 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:11,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621aef8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:11,129 INFO  [Listener,port=33011 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33011: starting
2024-11-22T03:14:11,130 DEBUG [Thread-38 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:11,130 DEBUG [Thread-38 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33011
2024-11-22T03:14:11,131 DEBUG [Thread-38 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:11,144 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245251143,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:11,146 DEBUG [Reader=1,bindAddress=localhost,port=33011 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:11,148 DEBUG [Reader=1,bindAddress=localhost,port=33011 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:11,148 DEBUG [Reader=1,bindAddress=localhost,port=33011 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:11,149 DEBUG [Reader=1,bindAddress=localhost,port=33011 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:11,150 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:11,150 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:11,150 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:11,151 DEBUG [Reader=1,bindAddress=localhost,port=33011 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:11,151 DEBUG [Reader=1,bindAddress=localhost,port=33011 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:11,151 DEBUG [Reader=1,bindAddress=localhost,port=33011 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:11,151 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:11,152 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:11,152 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-22T03:14:11,152 DEBUG [Thread-38 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:11,152 DEBUG [Reader=1,bindAddress=localhost,port=33011 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:11,152 DEBUG [Reader=1,bindAddress=localhost,port=33011 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:11,153 DEBUG [Reader=1,bindAddress=localhost,port=33011 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:11,153 DEBUG [Reader=1,bindAddress=localhost,port=33011 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:11,153 INFO  [Reader=1,bindAddress=localhost,port=33011 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:11,153 INFO  [Reader=1,bindAddress=localhost,port=33011 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51180, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:11,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:11,215 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33011
2024-11-22T03:14:11,222 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:11,222 INFO  [Listener,port=33011 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33011: stopping
2024-11-22T03:14:11,222 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:11,236 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=25 (was 19)
Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)

Potentially hanging thread: RpcServer idle connection scanner for port 33011
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Reader=1,bindAddress=localhost,port=33011
	java.base@17.0.11/sun.nio.ch.SelectorImpl.processReadyEvents(SelectorImpl.java:299)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:195)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:135)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=33011
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Listener,port=33011
	java.base@17.0.11/sun.nio.ch.EPoll.ctl(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implDereg(EPollSelectorImpl.java:230)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:267)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:134)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:213)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=33011
	java.base@17.0.11/sun.nio.ch.IOUtil.drain(Native Method)
	java.base@17.0.11/sun.nio.ch.EventFD.reset(EventFD.java:52)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.clearInterrupt(EPollSelectorImpl.java:263)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:201)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:135)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=787 (was 787), ProcessCount=11 (was 11), AvailableMemoryMB=6359 (was 6377)
2024-11-22T03:14:11,246 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=292, MaxFileDescriptor=1048576, SystemLoadAverage=787, ProcessCount=11, AvailableMemoryMB=6359
2024-11-22T03:14:11,266 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:11,266 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:11,280 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245251277,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:11,283 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:11,288 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45917
2024-11-22T03:14:11,289 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:11,292 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:11,292 INFO  [Listener,port=45917 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45917: starting
2024-11-22T03:14:11,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@443438a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:11,301 DEBUG [Thread-41 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:11,301 DEBUG [Thread-41 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45917
2024-11-22T03:14:11,309 DEBUG [Thread-41 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:11,321 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245251321,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:11,325 DEBUG [Reader=1,bindAddress=localhost,port=45917 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:11,326 DEBUG [Reader=1,bindAddress=localhost,port=45917 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:11,327 DEBUG [Reader=1,bindAddress=localhost,port=45917 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:11,328 DEBUG [Reader=1,bindAddress=localhost,port=45917 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:11,328 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:11,329 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:11,329 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:11,329 DEBUG [Reader=1,bindAddress=localhost,port=45917 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:11,329 DEBUG [Reader=1,bindAddress=localhost,port=45917 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:11,330 DEBUG [Reader=1,bindAddress=localhost,port=45917 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:11,330 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:11,331 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:11,331 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-22T03:14:11,331 DEBUG [Reader=1,bindAddress=localhost,port=45917 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:11,331 DEBUG [Thread-41 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:11,331 DEBUG [Reader=1,bindAddress=localhost,port=45917 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:11,331 DEBUG [Reader=1,bindAddress=localhost,port=45917 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:11,331 DEBUG [Reader=1,bindAddress=localhost,port=45917 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:11,332 INFO  [Reader=1,bindAddress=localhost,port=45917 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:11,332 INFO  [Reader=1,bindAddress=localhost,port=45917 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54078, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:11,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:11,375 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45917
2024-11-22T03:14:11,375 INFO  [Listener,port=45917 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45917: stopping
2024-11-22T03:14:11,381 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:11,383 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:11,383 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:11,396 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41013
2024-11-22T03:14:11,399 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:11,399 INFO  [Listener,port=41013 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41013: starting
2024-11-22T03:14:11,399 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:11,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d1712b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:11,413 DEBUG [Thread-44 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:11,413 DEBUG [Thread-44 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:41013
2024-11-22T03:14:11,415 DEBUG [Thread-44 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:11,421 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:11,422 DEBUG [Reader=1,bindAddress=localhost,port=41013 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:11,425 DEBUG [Reader=1,bindAddress=localhost,port=41013 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:11,426 DEBUG [Reader=1,bindAddress=localhost,port=41013 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:11,427 DEBUG [Reader=1,bindAddress=localhost,port=41013 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:11,427 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:11,428 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:11,428 DEBUG [Reader=1,bindAddress=localhost,port=41013 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:11,428 DEBUG [Reader=1,bindAddress=localhost,port=41013 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:11,429 DEBUG [Reader=1,bindAddress=localhost,port=41013 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:11,429 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:11,429 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:11,430 DEBUG [Reader=1,bindAddress=localhost,port=41013 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:11,430 DEBUG [Reader=1,bindAddress=localhost,port=41013 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:11,430 DEBUG [Reader=1,bindAddress=localhost,port=41013 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:11,430 DEBUG [Reader=1,bindAddress=localhost,port=41013 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:11,430 INFO  [Reader=1,bindAddress=localhost,port=41013 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:11,431 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-22T03:14:11,431 DEBUG [Thread-44 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:11,433 INFO  [Reader=1,bindAddress=localhost,port=41013 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55492, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:13,434 ERROR [Thread-44 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function.
java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:55492 remote=localhost/127.0.0.1:41013]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?]
	at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?]
	at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-22T03:14:13,435 DEBUG [Thread-44 {}] ipc.FailedServers(52): Added failed server with address localhost:41013 to list caused by java.io.IOException: Timeout while waiting connection header response
2024-11-22T03:14:13,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:13,435 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41013
2024-11-22T03:14:13,436 INFO  [Listener,port=41013 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41013: stopping
2024-11-22T03:14:13,436 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:13,436 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:13,442 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) - Thread LEAK? -, OpenFileDescriptor=290 (was 292), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=787 (was 787), ProcessCount=11 (was 11), AvailableMemoryMB=6323 (was 6359)
2024-11-22T03:14:13,448 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=787, ProcessCount=11, AvailableMemoryMB=6322
2024-11-22T03:14:13,461 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:13,462 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:13,467 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245253467,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:13,471 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:13,480 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36111
2024-11-22T03:14:13,484 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:13,488 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:13,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d03bef9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:13,490 DEBUG [Thread-47 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:13,490 DEBUG [Thread-47 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36111
2024-11-22T03:14:13,491 DEBUG [Thread-47 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:13,491 INFO  [Listener,port=36111 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36111: starting
2024-11-22T03:14:13,497 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245253497,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:13,499 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:13,499 DEBUG [Reader=1,bindAddress=localhost,port=36111 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:13,500 DEBUG [Reader=1,bindAddress=localhost,port=36111 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:13,500 DEBUG [Reader=1,bindAddress=localhost,port=36111 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:13,502 DEBUG [Reader=1,bindAddress=localhost,port=36111 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:13,502 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:13,503 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:13,503 DEBUG [Reader=1,bindAddress=localhost,port=36111 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:13,503 DEBUG [Reader=1,bindAddress=localhost,port=36111 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:13,503 DEBUG [Reader=1,bindAddress=localhost,port=36111 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:13,503 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:13,504 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:13,504 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-22T03:14:13,504 DEBUG [Reader=1,bindAddress=localhost,port=36111 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:13,504 DEBUG [Reader=1,bindAddress=localhost,port=36111 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:13,504 DEBUG [Thread-47 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:13,504 DEBUG [Reader=1,bindAddress=localhost,port=36111 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:13,504 DEBUG [Reader=1,bindAddress=localhost,port=36111 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:13,504 INFO  [Reader=1,bindAddress=localhost,port=36111 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:13,505 INFO  [Reader=1,bindAddress=localhost,port=36111 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46664, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:13,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:13,513 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36111
2024-11-22T03:14:13,513 INFO  [Listener,port=36111 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36111: stopping
2024-11-22T03:14:13,517 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:13,517 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:13,522 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=21 (was 19)
Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/java.lang.StackTraceElement.initStackTraceElement(Native Method)
	java.base@17.0.11/java.lang.StackTraceElement.of(StackTraceElement.java:555)
	java.base@17.0.11/java.lang.StackFrameInfo.toStackTraceElement(StackFrameInfo.java:130)
	app//org.apache.logging.log4j.util.StackLocator$$Lambda$128/0x00007faa8818c958.apply(Unknown Source)
	java.base@17.0.11/java.util.Optional.map(Optional.java:260)
	app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:100)
	app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121)
	app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=787 (was 787), ProcessCount=11 (was 11), AvailableMemoryMB=6306 (was 6322)
2024-11-22T03:14:13,529 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=787, ProcessCount=11, AvailableMemoryMB=6305
2024-11-22T03:14:13,545 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:13,545 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:13,550 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245253550,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:13,553 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:13,555 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46137
2024-11-22T03:14:13,555 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:13,556 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:13,556 INFO  [Listener,port=46137 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46137: starting
2024-11-22T03:14:13,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1221c33f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:13,557 DEBUG [Thread-50 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:13,557 DEBUG [Thread-50 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:46137
2024-11-22T03:14:13,558 DEBUG [Thread-50 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:13,565 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245253565,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:13,567 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(103): Have sent token of size 498 from initSASLContext.
2024-11-22T03:14:13,567 DEBUG [Reader=1,bindAddress=localhost,port=46137 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:13,568 DEBUG [Reader=1,bindAddress=localhost,port=46137 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:13,568 DEBUG [Reader=1,bindAddress=localhost,port=46137 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=498 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:13,570 DEBUG [Reader=1,bindAddress=localhost,port=46137 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:13,570 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:13,570 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:13,570 DEBUG [Reader=1,bindAddress=localhost,port=46137 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:13,571 DEBUG [Reader=1,bindAddress=localhost,port=46137 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:13,571 DEBUG [Reader=1,bindAddress=localhost,port=46137 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:13,571 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:13,571 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:13,571 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-22T03:14:13,571 DEBUG [Reader=1,bindAddress=localhost,port=46137 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:13,571 DEBUG [Thread-50 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:13,572 DEBUG [Reader=1,bindAddress=localhost,port=46137 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:13,572 DEBUG [Reader=1,bindAddress=localhost,port=46137 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:13,572 DEBUG [Reader=1,bindAddress=localhost,port=46137 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:13,572 INFO  [Reader=1,bindAddress=localhost,port=46137 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:13,572 INFO  [Reader=1,bindAddress=localhost,port=46137 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46918, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:13,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:13,583 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46137
2024-11-22T03:14:13,583 INFO  [Listener,port=46137 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46137: stopping
2024-11-22T03:14:13,583 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:13,584 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:13,589 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=20 (was 19) - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=787 (was 787), ProcessCount=11 (was 11), AvailableMemoryMB=6295 (was 6305)
2024-11-22T03:14:13,594 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=787, ProcessCount=11, AvailableMemoryMB=6295
2024-11-22T03:14:13,607 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:13,607 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:13,612 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245253612,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:13,615 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:13,620 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41169
2024-11-22T03:14:13,621 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:13,621 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:13,621 INFO  [Listener,port=41169 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41169: starting
2024-11-22T03:14:13,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526badaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:13,623 DEBUG [Thread-53 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:13,623 DEBUG [Thread-53 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:41169
2024-11-22T03:14:13,624 DEBUG [Thread-53 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:13,630 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245253630,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:13,631 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:13,632 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth.
2024-11-22T03:14:13,632 DEBUG [Thread-53 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:13,632 INFO  [Reader=1,bindAddress=localhost,port=41169 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-22T03:14:13,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:13,641 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41169
2024-11-22T03:14:13,641 INFO  [Listener,port=41169 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41169: stopping
2024-11-22T03:14:13,642 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:13,642 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:13,647 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=22 (was 19)
Potentially hanging thread: Reader=2,bindAddress=localhost,port=41169
	java.base@17.0.11/sun.nio.ch.EPoll.getDescriptor(EPoll.java:94)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:188)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:135)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/java.util.IdentityHashMap$KeySet.iterator(IdentityHashMap.java:977)
	java.base@17.0.11/java.util.Collections$SetFromMap.iterator(Collections.java:5686)
	java.base@17.0.11/jdk.internal.misc.TerminatingThreadLocal.threadTerminated(TerminatingThreadLocal.java:69)
	java.base@17.0.11/java.lang.Thread.exit(Thread.java:850)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)
 - Thread LEAK? -, OpenFileDescriptor=294 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=787 (was 787), ProcessCount=11 (was 11), AvailableMemoryMB=6279 (was 6295)
2024-11-22T03:14:13,652 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=787, ProcessCount=11, AvailableMemoryMB=6279
2024-11-22T03:14:13,665 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:13,666 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:13,672 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245253671,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:13,676 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:13,693 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:13,698 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-22T03:14:13,791 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3635 (auto-detected)
2024-11-22T03:14:13,792 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected)
2024-11-22T03:14:13,828 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37813
2024-11-22T03:14:13,828 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:13,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711667c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:13,835 DEBUG [Thread-54 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:13,835 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37813
2024-11-22T03:14:13,837 DEBUG [Thread-54 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:13,848 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245253847,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:13,850 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:13,897 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:13,897 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:13,906 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:13,911 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:13,911 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:13,912 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:13,913 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:13,913 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:13,914 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:13,914 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-22T03:14:13,914 DEBUG [Thread-54 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:13,914 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:13,914 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:13,915 DEBUG [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:13,915 INFO  [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:13,923 INFO  [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59450, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:13,923 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93
2024-11-22T03:14:13,924 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:250
2024-11-22T03:14:13,927 DEBUG [BRPC Connection (486228291) to localhost:37813 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-22T03:14:13,932 DEBUG [BRPC Connection (486228291) to localhost:37813 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:236
2024-11-22T03:14:13,934 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2236
2024-11-22T03:14:13,936 DEBUG [BRPC Connection (486228291) to localhost:37813 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-22T03:14:13,937 DEBUG [BRPC Connection (486228291) to localhost:37813 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2222
2024-11-22T03:14:13,940 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117
2024-11-22T03:14:13,941 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18281
2024-11-22T03:14:13,943 DEBUG [BRPC Connection (486228291) to localhost:37813 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-22T03:14:13,965 DEBUG [BRPC Connection (486228291) to localhost:37813 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22384
2024-11-22T03:14:13,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:13,968 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37813
2024-11-22T03:14:13,982 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21 (was 19)
Potentially hanging thread: NettyRpcServer-2-2
	app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.destroyUp(DefaultChannelPipeline.java:789)
	app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.destroy(DefaultChannelPipeline.java:785)
	app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.access$700(DefaultChannelPipeline.java:45)
	app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelUnregistered(DefaultChannelPipeline.java:1339)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:215)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:195)
	app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelUnregistered(DefaultChannelPipeline.java:770)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:819)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:405)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-2-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=299 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 787) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6251 (was 6279)
2024-11-22T03:14:13,996 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21, OpenFileDescriptor=299, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6250
2024-11-22T03:14:14,014 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:14,015 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:14,022 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245254021,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:14,027 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:14,027 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:14,030 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44243
2024-11-22T03:14:14,030 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:14,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596dcc3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-22T03:14:14,033 DEBUG [Thread-55 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:14,034 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44243
2024-11-22T03:14:14,035 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,051 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245254050,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,053 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:14,056 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-22T03:14:14,056 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:44243
org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-22T03:14:14,057 WARN  [Thread-55 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732245254025
2024-11-22T03:14:14,098 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,108 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:14,108 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-22T03:14:14,109 DEBUG [Thread-55 {}] ipc.FailedServers(52): Added failed server with address localhost:44243 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:14,109 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44243
2024-11-22T03:14:14,123 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24 (was 21)
Potentially hanging thread: NettyRpcServer-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=308 (was 299) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6230 (was 6250)
2024-11-22T03:14:14,137 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6230
2024-11-22T03:14:14,155 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:14,155 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:14,160 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245254160,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:14,163 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:14,164 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:14,172 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45725
2024-11-22T03:14:14,173 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:14,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c45ba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:14,175 DEBUG [Thread-56 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:14,175 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45725
2024-11-22T03:14:14,176 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,188 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245254188,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,190 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:14,190 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,191 DEBUG [NettyRpcServer-4-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,193 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:14,193 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:14,194 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:14,194 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,194 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:14,195 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:14,195 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-22T03:14:14,196 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:45725
javax.security.sasl.SaslException: No common protection layer between client and server
	at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?]
	at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?]
	at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-22T03:14:14,196 WARN  [Thread-56 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732245254162
2024-11-22T03:14:14,200 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,202 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:14,205 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,206 DEBUG [NettyRpcServer-4-3 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,209 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:14,210 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:14,210 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:14,210 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,210 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:14,211 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:14,211 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-22T03:14:14,211 DEBUG [Thread-56 {}] ipc.FailedServers(52): Added failed server with address localhost:45725 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:14,212 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45725
2024-11-22T03:14:14,219 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27 (was 24)
Potentially hanging thread: NettyRpcServer-4-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-4-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-4-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=317 (was 308) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6207 (was 6230)
2024-11-22T03:14:14,226 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27, OpenFileDescriptor=317, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6206
2024-11-22T03:14:14,237 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:14,238 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:14,243 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245254242,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:14,246 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:14,247 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:14,247 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34203
2024-11-22T03:14:14,248 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:14,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea9bd94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:14,249 DEBUG [Thread-57 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:14,249 DEBUG [Thread-57 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34203
2024-11-22T03:14:14,250 DEBUG [Thread-57 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,255 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245254254,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,256 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:14,256 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,256 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,258 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:14,258 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:14,258 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:14,259 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,259 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:14,259 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:14,259 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:14,259 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-22T03:14:14,259 DEBUG [Thread-57 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:14,259 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,260 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,260 DEBUG [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:14,260 INFO  [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:14,260 INFO  [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42622, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:14,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:14,270 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34203
2024-11-22T03:14:14,271 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:14,271 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:14,272 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36341
2024-11-22T03:14:14,272 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:14,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbce53a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:14,273 DEBUG [Thread-58 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:14,273 DEBUG [Thread-58 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36341
2024-11-22T03:14:14,275 DEBUG [Thread-58 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,277 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:14,277 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,277 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,278 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:14,279 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:14,279 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:14,279 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,279 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:14,279 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:14,280 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:14,280 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-22T03:14:14,280 DEBUG [Thread-58 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:14,280 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,280 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,280 DEBUG [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:14,280 INFO  [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:14,281 INFO  [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57372, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:14,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:14,289 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36341
2024-11-22T03:14:14,290 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:14,290 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:14,291 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37487
2024-11-22T03:14:14,291 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:14,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54727e0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:14,292 DEBUG [Thread-59 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:14,293 DEBUG [Thread-59 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37487
2024-11-22T03:14:14,293 DEBUG [Thread-59 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,295 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:14,295 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,295 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,298 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:14,298 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:14,298 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:14,299 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,299 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:14,299 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:14,299 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:14,300 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-22T03:14:14,300 DEBUG [Thread-59 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:14,300 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,300 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,300 DEBUG [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:14,300 INFO  [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:14,301 INFO  [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40382, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:14,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:14,308 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37487
2024-11-22T03:14:14,309 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:14,310 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:14,311 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35253
2024-11-22T03:14:14,311 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:14,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48b8e3d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:14,312 DEBUG [Thread-60 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:14,312 DEBUG [Thread-60 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35253
2024-11-22T03:14:14,313 DEBUG [Thread-60 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,315 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:14,315 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,315 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,317 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:14,317 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:14,317 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:14,318 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,318 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:14,318 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:14,318 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:14,318 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int
2024-11-22T03:14:14,319 DEBUG [Thread-60 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:14,319 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,319 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,319 DEBUG [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-22T03:14:14,319 INFO  [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:14,319 INFO  [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49448, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:14,327 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:14,327 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35253
2024-11-22T03:14:14,328 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:14,328 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:14,330 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40219
2024-11-22T03:14:14,330 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:14,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eee6702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:14,331 DEBUG [Thread-61 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:14,331 DEBUG [Thread-61 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40219
2024-11-22T03:14:14,332 DEBUG [Thread-61 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,333 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:14,333 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,333 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,335 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:14,335 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:14,335 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:14,335 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,336 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:14,336 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:14,336 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:14,336 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-22T03:14:14,336 DEBUG [Thread-61 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:14,336 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,336 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,336 DEBUG [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:14,337 INFO  [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:14,338 INFO  [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48806, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:14,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:14,349 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40219
2024-11-22T03:14:14,354 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37 (was 27)
Potentially hanging thread: NettyRpcServer-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=362 (was 317) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6178 (was 6206)
2024-11-22T03:14:14,360 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37, OpenFileDescriptor=362, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6178
2024-11-22T03:14:14,372 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:14,373 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:14,377 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245254376,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:14,379 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:14,380 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:14,381 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:32961
2024-11-22T03:14:14,381 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:14,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b4af5b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:14,382 DEBUG [Thread-62 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-22T03:14:14,382 DEBUG [Thread-62 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:32961
2024-11-22T03:14:14,384 DEBUG [BRPC Connection (486228291) to localhost:32961 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-22T03:14:14,385 WARN  [BRPC Connection (486228291) to localhost:32961 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-22T03:14:14,385 DEBUG [BRPC Connection (486228291) to localhost:32961 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-22T03:14:14,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:14,385 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:32961
2024-11-22T03:14:14,391 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39 (was 37)
Potentially hanging thread: NettyRpcServer-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=371 (was 362) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6167 (was 6178)
2024-11-22T03:14:14,397 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39, OpenFileDescriptor=371, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6167
2024-11-22T03:14:14,409 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:14,409 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:14,413 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245254413,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:14,416 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:14,416 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:14,417 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36009
2024-11-22T03:14:14,417 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-22T03:14:14,417 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-22T03:14:14,417 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-22T03:14:14,417 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-22T03:14:14,417 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-22T03:14:14,418 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-22T03:14:14,418 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-22T03:14:14,418 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-22T03:14:14,418 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:14,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f25a636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:14,419 DEBUG [Thread-63 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-22T03:14:14,419 DEBUG [Thread-63 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:36009
2024-11-22T03:14:14,422 WARN  [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-22T03:14:14,422 INFO  [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-22T03:14:14,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:14,432 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36009
2024-11-22T03:14:14,440 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41 (was 39)
Potentially hanging thread: NettyRpcServer-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=380 (was 371) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6153 (was 6167)
2024-11-22T03:14:14,448 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6153
2024-11-22T03:14:14,460 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:14,460 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:14,465 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245254464,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:14,467 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:14,467 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:14,468 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40499
2024-11-22T03:14:14,469 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:14,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79899baa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:14,470 DEBUG [Thread-64 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:14,470 DEBUG [Thread-64 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40499
2024-11-22T03:14:14,471 DEBUG [Thread-64 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,476 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245254475,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,477 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:14,477 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,478 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,479 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:14,480 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:14,480 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:14,480 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,480 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:14,481 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:14,481 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:14,481 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-22T03:14:14,481 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,481 DEBUG [Thread-64 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:14,481 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,481 DEBUG [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:14,482 INFO  [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:14,482 INFO  [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42976, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:14,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:14,489 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40499
2024-11-22T03:14:14,496 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43 (was 41)
Potentially hanging thread: NettyRpcServer-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=389 (was 380) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6138 (was 6153)
2024-11-22T03:14:14,504 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43, OpenFileDescriptor=389, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6138
2024-11-22T03:14:14,515 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:14,516 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:14,520 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245254519,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:14,522 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:14,523 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:14,524 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44569
2024-11-22T03:14:14,524 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:14,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61895338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:14,525 DEBUG [Thread-65 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:14,525 DEBUG [Thread-65 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44569
2024-11-22T03:14:14,526 DEBUG [Thread-65 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,531 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245254530,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,532 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:14,532 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,532 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,534 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:14,534 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:14,534 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:14,534 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,535 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:14,535 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:14,535 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:14,535 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-22T03:14:14,535 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,535 DEBUG [Thread-65 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:14,536 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,536 DEBUG [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:14,536 INFO  [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:14,537 INFO  [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52552, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:14,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:14,547 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44569
2024-11-22T03:14:14,547 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:14,548 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:14,549 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40805
2024-11-22T03:14:14,549 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:14,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f344f1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:14,551 DEBUG [Thread-66 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:14,551 DEBUG [Thread-66 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40805
2024-11-22T03:14:14,552 DEBUG [Thread-66 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,554 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:14,554 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,554 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,556 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:14,557 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:14,557 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:14,558 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,558 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:14,558 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:14,559 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:14,559 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-22T03:14:14,559 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:14,559 DEBUG [Thread-66 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:14,559 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:14,559 DEBUG [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:14,559 INFO  [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:14,561 INFO  [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:39980, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:16,560 ERROR [Thread-66 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function.
java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39980 remote=localhost/127.0.0.1:40805]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?]
	at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?]
	at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-22T03:14:16,562 DEBUG [Thread-66 {}] ipc.FailedServers(52): Added failed server with address localhost:40805 to list caused by java.io.IOException: Timeout while waiting connection header response
2024-11-22T03:14:16,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:16,563 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40805
2024-11-22T03:14:16,574 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47 (was 43)
Potentially hanging thread: NettyRpcServer-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=407 (was 389) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6109 (was 6138)
2024-11-22T03:14:16,582 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47, OpenFileDescriptor=407, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6109
2024-11-22T03:14:16,596 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:16,597 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:16,600 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245256599,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:16,601 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:16,602 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:16,603 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44411
2024-11-22T03:14:16,603 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:16,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce178fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:16,604 DEBUG [Thread-67 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:16,604 DEBUG [Thread-67 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44411
2024-11-22T03:14:16,605 DEBUG [Thread-67 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,611 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245256611,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,613 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:16,613 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,613 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,614 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:16,615 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:16,615 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:16,615 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,615 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:16,616 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:16,616 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:16,616 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-22T03:14:16,616 DEBUG [Thread-67 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:16,616 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,616 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,616 DEBUG [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:16,616 INFO  [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:16,617 INFO  [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60482, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:16,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:16,626 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44411
2024-11-22T03:14:16,632 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49 (was 47)
Potentially hanging thread: NettyRpcServer-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=416 (was 407) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6091 (was 6109)
2024-11-22T03:14:16,638 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49, OpenFileDescriptor=416, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6090
2024-11-22T03:14:16,652 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:16,652 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:16,655 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245256655,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:16,657 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:16,657 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:16,658 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37837
2024-11-22T03:14:16,659 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:16,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fcb8016, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:16,659 DEBUG [Thread-68 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:16,660 DEBUG [Thread-68 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37837
2024-11-22T03:14:16,661 DEBUG [Thread-68 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,665 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245256665,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,666 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:16,666 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,666 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,668 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:16,669 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-22T03:14:16,669 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-22T03:14:16,669 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,669 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:16,670 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-22T03:14:16,670 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-22T03:14:16,670 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-22T03:14:16,670 DEBUG [Thread-68 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:16,670 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,670 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,670 DEBUG [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:16,670 INFO  [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:16,671 INFO  [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42894, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:16,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:16,677 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37837
2024-11-22T03:14:16,682 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51 (was 49)
Potentially hanging thread: NettyRpcServer-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=425 (was 416) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6074 (was 6090)
2024-11-22T03:14:16,688 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6074
2024-11-22T03:14:16,699 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:16,699 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:16,703 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245256703,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:16,705 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:16,705 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:16,707 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37239
2024-11-22T03:14:16,707 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:16,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3daade82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:16,708 DEBUG [Thread-69 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:16,708 DEBUG [Thread-69 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37239
2024-11-22T03:14:16,709 DEBUG [Thread-69 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,714 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245256714,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,715 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-22T03:14:16,715 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth.
2024-11-22T03:14:16,716 DEBUG [Thread-69 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:16,716 INFO  [NettyRpcServer-17-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-22T03:14:16,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:16,724 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37239
2024-11-22T03:14:16,732 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=53 (was 51)
Potentially hanging thread: NettyRpcServer-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=434 (was 425) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6058 (was 6074)
2024-11-22T03:14:16,739 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=53, OpenFileDescriptor=434, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6058
2024-11-22T03:14:16,754 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:16,754 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:16,757 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245256757,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:16,759 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:16,760 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41647
2024-11-22T03:14:16,760 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:16,761 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:16,761 INFO  [Listener,port=41647 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41647: starting
2024-11-22T03:14:16,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@532cd113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:16,763 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-22T03:14:16,764 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-22T03:14:16,768 DEBUG [Thread-72 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:16,782 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,787 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245256787,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,789 DEBUG [Reader=1,bindAddress=localhost,port=41647 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,790 DEBUG [Reader=1,bindAddress=localhost,port=41647 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,790 DEBUG [Reader=1,bindAddress=localhost,port=41647 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,791 DEBUG [Reader=1,bindAddress=localhost,port=41647 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:16,792 DEBUG [Reader=1,bindAddress=localhost,port=41647 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,792 DEBUG [Reader=1,bindAddress=localhost,port=41647 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,792 DEBUG [Reader=1,bindAddress=localhost,port=41647 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:16,793 DEBUG [Reader=1,bindAddress=localhost,port=41647 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,793 DEBUG [Reader=1,bindAddress=localhost,port=41647 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,793 DEBUG [Reader=1,bindAddress=localhost,port=41647 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,793 DEBUG [Reader=1,bindAddress=localhost,port=41647 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:16,793 INFO  [Reader=1,bindAddress=localhost,port=41647 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:16,799 INFO  [Reader=1,bindAddress=localhost,port=41647 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42406, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:16,801 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:16,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:16,813 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41647
2024-11-22T03:14:16,813 INFO  [Listener,port=41647 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41647: stopping
2024-11-22T03:14:16,814 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:16,815 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:16,820 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=55 (was 53)
Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/java.util.stream.WhileOps$1Op$1OpSink.accept(WhileOps.java:376)
	java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.tryAdvance(StackStreamFactory.java:595)
	java.base@17.0.11/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129)
	java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527)
	java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513)
	java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
	java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150)
	java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
	java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647)
	app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100)
	app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007faa8818c260.apply(Unknown Source)
	java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243)
	java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499)
	app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97)
	app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121)
	app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-1
	java.base@17.0.11/java.lang.ClassLoader.defineClass1(Native Method)
	java.base@17.0.11/java.lang.ClassLoader.defineClass(ClassLoader.java:1017)
	java.base@17.0.11/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150)
	java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862)
	java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:773)
	java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:768)
	java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776)
	java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:318)
	java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:781)
	java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681)
	java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639)
	java.base@17.0.11/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
	java.base@17.0.11/java.lang.ClassLoader.loadClass(ClassLoader.java:525)
	java.base@17.0.11/java.lang.ClassLoader.defineClass1(Native Method)
	java.base@17.0.11/java.lang.ClassLoader.defineClass(ClassLoader.java:1017)
	java.base@17.0.11/java.security.SecureClassLoader.defineClass(SecureClassLoader.java:150)
	java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.defineClass(BuiltinClassLoader.java:862)
	java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:773)
	java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader$4.run(BuiltinClassLoader.java:768)
	java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776)
	java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:318)
	java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:781)
	java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:681)
	java.base@17.0.11/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:639)
	java.base@17.0.11/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188)
	java.base@17.0.11/java.lang.ClassLoader.loadClass(ClassLoader.java:525)
	app//org.apache.hadoop.hbase.ipc.IPCUtil$1.initialValue(IPCUtil.java:294)
	app//org.apache.hadoop.hbase.ipc.IPCUtil$1.initialValue(IPCUtil.java:290)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.initialize(FastThreadLocal.java:177)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.get(FastThreadLocal.java:142)
	app//org.apache.hadoop.hbase.ipc.IPCUtil.execute(IPCUtil.java:304)
	app//org.apache.hadoop.hbase.ipc.NettyRpcConnection.shutdown(NettyRpcConnection.java:146)
	app//org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:161)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274)
	app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412)
	app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274)
	app//org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81)
	app//org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274)
	app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412)
	app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274)
	app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)
	app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 434) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6037 (was 6058)
2024-11-22T03:14:16,825 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=54, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6037
2024-11-22T03:14:16,839 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:16,840 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:16,843 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245256843,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:16,845 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:16,846 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45597
2024-11-22T03:14:16,847 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:16,847 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:16,847 INFO  [Listener,port=45597 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45597: starting
2024-11-22T03:14:16,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f2b87a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-22T03:14:16,848 DEBUG [Thread-75 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:16,849 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,854 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245256853,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,856 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-22T03:14:16,857 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:16,857 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:45597 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
2024-11-22T03:14:16,857 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45597
2024-11-22T03:14:16,857 INFO  [Listener,port=45597 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45597: stopping
2024-11-22T03:14:16,857 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:16,857 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:16,864 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=56 (was 54)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/jdk.internal.misc.Unsafe.freeMemory0(Native Method)
	java.base@17.0.11/jdk.internal.misc.Unsafe.freeMemory(Unsafe.java:924)
	java.base@17.0.11/sun.nio.ch.EPoll.freePollArray(EPoll.java:80)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:217)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6022 (was 6037)
2024-11-22T03:14:16,869 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=55, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6021
2024-11-22T03:14:16,879 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:16,879 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:16,884 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245256883,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:16,886 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:16,887 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=42003
2024-11-22T03:14:16,888 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:16,888 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:16,888 INFO  [Listener,port=42003 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=42003: starting
2024-11-22T03:14:16,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@642f4c3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:16,890 DEBUG [Thread-78 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:16,892 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,896 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245256895,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,898 DEBUG [Reader=1,bindAddress=localhost,port=42003 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,898 DEBUG [Reader=1,bindAddress=localhost,port=42003 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,899 DEBUG [Reader=1,bindAddress=localhost,port=42003 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,899 DEBUG [Reader=1,bindAddress=localhost,port=42003 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:16,900 DEBUG [Reader=1,bindAddress=localhost,port=42003 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,900 DEBUG [Reader=1,bindAddress=localhost,port=42003 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,900 DEBUG [Reader=1,bindAddress=localhost,port=42003 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:16,901 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-22T03:14:16,902 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:42003 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server
2024-11-22T03:14:16,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:16,902 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 42003
2024-11-22T03:14:16,902 INFO  [Listener,port=42003 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=42003: stopping
2024-11-22T03:14:16,902 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:16,902 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:16,908 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=58 (was 55)
Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.closeIntFD(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:216)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67)

Potentially hanging thread: Relogin-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6007 (was 6021)
2024-11-22T03:14:16,913 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=57, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6006
2024-11-22T03:14:16,923 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732245256885
2024-11-22T03:14:16,924 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:16,924 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:16,927 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245256927,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:16,929 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:16,930 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35567
2024-11-22T03:14:16,931 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:16,931 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:16,931 INFO  [Listener,port=35567 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35567: starting
2024-11-22T03:14:16,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447c92db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:16,932 DEBUG [Thread-81 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:16,933 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,942 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245256941,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,943 DEBUG [Reader=1,bindAddress=localhost,port=35567 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,944 DEBUG [Reader=1,bindAddress=localhost,port=35567 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,944 DEBUG [Reader=1,bindAddress=localhost,port=35567 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,944 DEBUG [Reader=1,bindAddress=localhost,port=35567 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:16,945 DEBUG [Reader=1,bindAddress=localhost,port=35567 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,945 DEBUG [Reader=1,bindAddress=localhost,port=35567 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,945 DEBUG [Reader=1,bindAddress=localhost,port=35567 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:16,946 DEBUG [Reader=1,bindAddress=localhost,port=35567 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,946 DEBUG [Reader=1,bindAddress=localhost,port=35567 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,946 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:16,946 DEBUG [Reader=1,bindAddress=localhost,port=35567 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,946 DEBUG [Reader=1,bindAddress=localhost,port=35567 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:16,946 INFO  [Reader=1,bindAddress=localhost,port=35567 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:16,947 INFO  [Reader=1,bindAddress=localhost,port=35567 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47682, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:16,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:16,952 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35567
2024-11-22T03:14:16,952 INFO  [Listener,port=35567 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35567: stopping
2024-11-22T03:14:16,953 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:16,953 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:16,953 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:16,954 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35249
2024-11-22T03:14:16,954 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:16,954 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:16,954 INFO  [Listener,port=35249 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35249: starting
2024-11-22T03:14:16,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d8c4a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:16,955 DEBUG [Thread-84 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:16,957 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,959 DEBUG [Reader=1,bindAddress=localhost,port=35249 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,959 DEBUG [Reader=1,bindAddress=localhost,port=35249 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,959 DEBUG [Reader=1,bindAddress=localhost,port=35249 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,960 DEBUG [Reader=1,bindAddress=localhost,port=35249 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:16,960 DEBUG [Reader=1,bindAddress=localhost,port=35249 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,961 DEBUG [Reader=1,bindAddress=localhost,port=35249 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,961 DEBUG [Reader=1,bindAddress=localhost,port=35249 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:16,961 DEBUG [Reader=1,bindAddress=localhost,port=35249 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,961 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:16,961 DEBUG [Reader=1,bindAddress=localhost,port=35249 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,961 DEBUG [Reader=1,bindAddress=localhost,port=35249 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,961 DEBUG [Reader=1,bindAddress=localhost,port=35249 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:16,961 INFO  [Reader=1,bindAddress=localhost,port=35249 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:16,962 INFO  [Reader=1,bindAddress=localhost,port=35249 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:46334, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:16,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:16,968 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35249
2024-11-22T03:14:16,968 INFO  [Listener,port=35249 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35249: stopping
2024-11-22T03:14:16,969 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:16,969 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:16,969 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:16,970 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41125
2024-11-22T03:14:16,971 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:16,971 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:16,971 INFO  [Listener,port=41125 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41125: starting
2024-11-22T03:14:16,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28e442c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:16,972 DEBUG [Thread-87 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:16,975 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,976 DEBUG [Reader=1,bindAddress=localhost,port=41125 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,977 DEBUG [Reader=1,bindAddress=localhost,port=41125 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,977 DEBUG [Reader=1,bindAddress=localhost,port=41125 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,978 DEBUG [Reader=1,bindAddress=localhost,port=41125 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:16,979 DEBUG [Reader=1,bindAddress=localhost,port=41125 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,979 DEBUG [Reader=1,bindAddress=localhost,port=41125 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,979 DEBUG [Reader=1,bindAddress=localhost,port=41125 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:16,980 DEBUG [Reader=1,bindAddress=localhost,port=41125 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,980 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:16,980 DEBUG [Reader=1,bindAddress=localhost,port=41125 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,980 DEBUG [Reader=1,bindAddress=localhost,port=41125 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,980 DEBUG [Reader=1,bindAddress=localhost,port=41125 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:16,980 INFO  [Reader=1,bindAddress=localhost,port=41125 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:16,981 INFO  [Reader=1,bindAddress=localhost,port=41125 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51370, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:16,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:16,989 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41125
2024-11-22T03:14:16,989 INFO  [Listener,port=41125 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41125: stopping
2024-11-22T03:14:16,989 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:16,989 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:16,989 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:16,990 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39815
2024-11-22T03:14:16,991 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:16,991 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:16,991 INFO  [Listener,port=39815 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39815: starting
2024-11-22T03:14:16,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56dd9428, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:16,992 DEBUG [Thread-90 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:16,993 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,995 DEBUG [Reader=1,bindAddress=localhost,port=39815 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,995 DEBUG [Reader=1,bindAddress=localhost,port=39815 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,995 DEBUG [Reader=1,bindAddress=localhost,port=39815 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,996 DEBUG [Reader=1,bindAddress=localhost,port=39815 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:16,997 DEBUG [Reader=1,bindAddress=localhost,port=39815 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,997 DEBUG [Reader=1,bindAddress=localhost,port=39815 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,997 DEBUG [Reader=1,bindAddress=localhost,port=39815 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:16,997 DEBUG [Reader=1,bindAddress=localhost,port=39815 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:16,997 DEBUG [Reader=1,bindAddress=localhost,port=39815 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:16,997 DEBUG [Reader=1,bindAddress=localhost,port=39815 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:16,998 DEBUG [Reader=1,bindAddress=localhost,port=39815 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-22T03:14:16,998 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:16,998 INFO  [Reader=1,bindAddress=localhost,port=39815 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:16,999 INFO  [Reader=1,bindAddress=localhost,port=39815 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51644, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:17,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:17,007 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39815
2024-11-22T03:14:17,007 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:17,007 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:17,007 INFO  [Listener,port=39815 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39815: stopping
2024-11-22T03:14:17,008 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:17,009 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34431
2024-11-22T03:14:17,009 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:17,009 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:17,009 INFO  [Listener,port=34431 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34431: starting
2024-11-22T03:14:17,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@559740a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:17,010 DEBUG [Thread-93 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:17,012 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:17,013 DEBUG [Reader=1,bindAddress=localhost,port=34431 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:17,014 DEBUG [Reader=1,bindAddress=localhost,port=34431 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:17,014 DEBUG [Reader=1,bindAddress=localhost,port=34431 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=498 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:17,015 DEBUG [Reader=1,bindAddress=localhost,port=34431 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 107 from saslServer.
2024-11-22T03:14:17,016 DEBUG [Reader=1,bindAddress=localhost,port=34431 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:17,016 DEBUG [Reader=1,bindAddress=localhost,port=34431 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:17,016 DEBUG [Reader=1,bindAddress=localhost,port=34431 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:17,016 DEBUG [Reader=1,bindAddress=localhost,port=34431 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:17,016 DEBUG [Reader=1,bindAddress=localhost,port=34431 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:17,016 DEBUG [Reader=1,bindAddress=localhost,port=34431 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:17,017 DEBUG [Reader=1,bindAddress=localhost,port=34431 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:17,017 INFO  [Reader=1,bindAddress=localhost,port=34431 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:17,017 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:17,018 INFO  [Reader=1,bindAddress=localhost,port=34431 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38762, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:17,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:17,026 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34431
2024-11-22T03:14:17,026 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:17,026 INFO  [Listener,port=34431 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34431: stopping
2024-11-22T03:14:17,026 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:17,033 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65 (was 57)
Potentially hanging thread: RpcServer idle connection scanner for port 34431
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EventFD.set0(Native Method)
	java.base@17.0.11/sun.nio.ch.EventFD.set(EventFD.java:48)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:251)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:183)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Listener,port=34431
	java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.close0(Native Method)
	java.base@17.0.11/sun.nio.ch.SocketDispatcher.close(SocketDispatcher.java:70)
	java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.tryClose(ServerSocketChannelImpl.java:554)
	java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.tryFinishClose(ServerSocketChannelImpl.java:568)
	java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.kill(ServerSocketChannelImpl.java:631)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:194)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:256)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-8
	java.base@17.0.11/sun.nio.ch.EPoll.ctl(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implDereg(EPollSelectorImpl.java:230)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:267)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:112)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=5979 (was 6006)
2024-11-22T03:14:17,040 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=62, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=5979
2024-11-22T03:14:17,052 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:17,053 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:17,055 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245257054,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:17,057 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:17,058 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34935
2024-11-22T03:14:17,059 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:17,059 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:17,059 INFO  [Listener,port=34935 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34935: starting
2024-11-22T03:14:17,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc60e92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:17,060 DEBUG [Thread-96 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-22T03:14:17,062 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-22T03:14:17,062 WARN  [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-22T03:14:17,062 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-22T03:14:17,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:17,063 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34935
2024-11-22T03:14:17,063 INFO  [Listener,port=34935 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34935: stopping
2024-11-22T03:14:17,063 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:17,063 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:17,069 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63 (was 62) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=5978 (was 5979)
2024-11-22T03:14:17,074 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=5978
2024-11-22T03:14:17,084 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:17,084 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:17,086 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245257086,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:17,088 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:17,089 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44037
2024-11-22T03:14:17,090 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-22T03:14:17,090 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-22T03:14:17,090 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-22T03:14:17,090 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-22T03:14:17,090 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-22T03:14:17,090 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-22T03:14:17,090 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-22T03:14:17,090 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-22T03:14:17,090 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:17,090 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:17,090 INFO  [Listener,port=44037 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44037: starting
2024-11-22T03:14:17,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3db7437d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:17,091 DEBUG [Thread-99 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-22T03:14:17,093 WARN  [Reader=1,bindAddress=localhost,port=44037 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-22T03:14:17,093 INFO  [Reader=1,bindAddress=localhost,port=44037 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-22T03:14:17,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:17,098 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44037
2024-11-22T03:14:17,098 INFO  [Listener,port=44037 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44037: stopping
2024-11-22T03:14:17,098 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:17,098 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:17,105 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65 (was 63)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer idle connection scanner for port 44037
	java.base@17.0.11/java.lang.Thread.exit(Thread.java:849)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=5977 (was 5978)
2024-11-22T03:14:17,111 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=5977
2024-11-22T03:14:17,120 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:17,121 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:17,123 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245257122,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:17,125 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:17,128 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41833
2024-11-22T03:14:17,129 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:17,129 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:17,129 INFO  [Listener,port=41833 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41833: starting
2024-11-22T03:14:17,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38e65c21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:17,131 DEBUG [Thread-102 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:17,133 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:17,136 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245257136,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:17,138 DEBUG [Reader=1,bindAddress=localhost,port=41833 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:17,138 DEBUG [Reader=1,bindAddress=localhost,port=41833 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:17,138 DEBUG [Reader=1,bindAddress=localhost,port=41833 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:17,139 DEBUG [Reader=1,bindAddress=localhost,port=41833 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:17,139 DEBUG [Reader=1,bindAddress=localhost,port=41833 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:17,140 DEBUG [Reader=1,bindAddress=localhost,port=41833 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:17,140 DEBUG [Reader=1,bindAddress=localhost,port=41833 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:17,140 DEBUG [Reader=1,bindAddress=localhost,port=41833 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:17,140 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:17,140 DEBUG [Reader=1,bindAddress=localhost,port=41833 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:17,140 DEBUG [Reader=1,bindAddress=localhost,port=41833 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:17,140 DEBUG [Reader=1,bindAddress=localhost,port=41833 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:17,141 INFO  [Reader=1,bindAddress=localhost,port=41833 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:17,141 INFO  [Reader=1,bindAddress=localhost,port=41833 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59358, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:17,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:17,146 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41833
2024-11-22T03:14:17,146 INFO  [Listener,port=41833 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41833: stopping
2024-11-22T03:14:17,147 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:17,147 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:17,153 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=66 (was 64)
Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndExchangeByte(Unsafe.java:1527)
	java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndSetByte(Unsafe.java:1544)
	java.base@17.0.11/jdk.internal.misc.Unsafe.compareAndSetBoolean(Unsafe.java:1784)
	java.base@17.0.11/java.lang.invoke.VarHandleBooleans$FieldInstanceReadWrite.compareAndSet(VarHandleBooleans.java:178)
	java.base@17.0.11/java.lang.invoke.VarHandleGuards.guard_LII_Z(VarHandleGuards.java:137)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:125)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=5974 (was 5977)
2024-11-22T03:14:17,159 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=5974
2024-11-22T03:14:17,174 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:17,174 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:17,176 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245257176,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:17,178 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:17,179 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39631
2024-11-22T03:14:17,180 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:17,180 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:17,180 INFO  [Listener,port=39631 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39631: starting
2024-11-22T03:14:17,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205d8cef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:17,184 DEBUG [Thread-105 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:17,187 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:17,191 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245257190,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:17,192 DEBUG [Reader=1,bindAddress=localhost,port=39631 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:17,193 DEBUG [Reader=1,bindAddress=localhost,port=39631 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:17,193 DEBUG [Reader=1,bindAddress=localhost,port=39631 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:17,193 DEBUG [Reader=1,bindAddress=localhost,port=39631 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:17,194 DEBUG [Reader=1,bindAddress=localhost,port=39631 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:17,194 DEBUG [Reader=1,bindAddress=localhost,port=39631 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:17,194 DEBUG [Reader=1,bindAddress=localhost,port=39631 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:17,195 DEBUG [Reader=1,bindAddress=localhost,port=39631 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:17,195 DEBUG [Reader=1,bindAddress=localhost,port=39631 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:17,195 DEBUG [Reader=1,bindAddress=localhost,port=39631 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:17,195 DEBUG [Reader=1,bindAddress=localhost,port=39631 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:17,195 INFO  [Reader=1,bindAddress=localhost,port=39631 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:17,195 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:17,196 INFO  [Reader=1,bindAddress=localhost,port=39631 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57582, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:17,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:17,206 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39631
2024-11-22T03:14:17,206 INFO  [Listener,port=39631 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39631: stopping
2024-11-22T03:14:17,206 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:17,206 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:17,206 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:17,207 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40543
2024-11-22T03:14:17,208 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:17,208 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:17,208 INFO  [Listener,port=40543 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40543: starting
2024-11-22T03:14:17,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@398767e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:17,211 DEBUG [Thread-108 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:17,213 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:17,215 DEBUG [Reader=1,bindAddress=localhost,port=40543 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:17,216 DEBUG [Reader=1,bindAddress=localhost,port=40543 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:17,216 DEBUG [Reader=1,bindAddress=localhost,port=40543 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:17,216 DEBUG [Reader=1,bindAddress=localhost,port=40543 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:17,217 DEBUG [Reader=1,bindAddress=localhost,port=40543 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:17,217 DEBUG [Reader=1,bindAddress=localhost,port=40543 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:17,217 DEBUG [Reader=1,bindAddress=localhost,port=40543 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:17,218 DEBUG [Reader=1,bindAddress=localhost,port=40543 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:17,218 DEBUG [Reader=1,bindAddress=localhost,port=40543 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:17,218 DEBUG [Reader=1,bindAddress=localhost,port=40543 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:17,218 DEBUG [Reader=1,bindAddress=localhost,port=40543 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:17,218 INFO  [Reader=1,bindAddress=localhost,port=40543 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:17,219 INFO  [Reader=1,bindAddress=localhost,port=40543 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43904, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:19,221 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-22T03:14:19,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,222 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40543
2024-11-22T03:14:19,222 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:40543 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException
2024-11-22T03:14:19,222 INFO  [Listener,port=40543 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40543: stopping
2024-11-22T03:14:19,222 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:19,222 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:19,228 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68 (was 65)
Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/java.lang.Thread.clearInterruptEvent(Native Method)
	java.base@17.0.11/java.lang.Thread.interrupted(Thread.java:1035)
	app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.write(EventChannelEncoder.java:286)
	app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.setOutErr(EventChannelEncoder.java:173)
	app//org.apache.maven.surefire.booter.spi.EventChannelEncoder.testOutput(EventChannelEncoder.java:168)
	app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:99)
	app//org.apache.maven.surefire.api.booter.ForkingRunListener.writeTestOutput(ForkingRunListener.java:43)
	app//org.apache.maven.surefire.common.junit4.JUnit4RunListener.writeTestOutput(JUnit4RunListener.java:202)
	app//org.apache.maven.surefire.api.report.ConsoleOutputCapture$ForwardingPrintStream.write(ConsoleOutputCapture.java:57)
	app//org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:250)
	app//org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:283)
	app//org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:294)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:217)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:208)
	app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:199)
	app//org.apache.hadoop.hbase.logging.HBaseTestAppender.append(HBaseTestAppender.java:158)
	app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
	app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
	app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:675)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:633)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:666)
	app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:635)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:616)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:552)
	app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
	app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
	app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=5930 (was 5974)
2024-11-22T03:14:19,234 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=67, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=5930
2024-11-22T03:14:19,235 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732245257177
2024-11-22T03:14:19,246 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:19,246 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:19,248 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245259248,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:19,250 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,251 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41001
2024-11-22T03:14:19,252 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,252 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:19,252 INFO  [Listener,port=41001 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41001: starting
2024-11-22T03:14:19,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c236562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,253 DEBUG [Thread-111 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:19,255 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,258 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245259257,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,259 DEBUG [Reader=1,bindAddress=localhost,port=41001 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,260 DEBUG [Reader=1,bindAddress=localhost,port=41001 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:19,260 DEBUG [Reader=1,bindAddress=localhost,port=41001 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,261 DEBUG [Reader=1,bindAddress=localhost,port=41001 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:19,261 DEBUG [Reader=1,bindAddress=localhost,port=41001 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:19,261 DEBUG [Reader=1,bindAddress=localhost,port=41001 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,261 DEBUG [Reader=1,bindAddress=localhost,port=41001 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:19,262 DEBUG [Reader=1,bindAddress=localhost,port=41001 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:19,262 DEBUG [Reader=1,bindAddress=localhost,port=41001 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,262 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:19,262 DEBUG [Reader=1,bindAddress=localhost,port=41001 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,262 DEBUG [Reader=1,bindAddress=localhost,port=41001 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:19,262 INFO  [Reader=1,bindAddress=localhost,port=41001 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:19,263 INFO  [Reader=1,bindAddress=localhost,port=41001 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58870, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:19,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,269 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41001
2024-11-22T03:14:19,269 INFO  [Listener,port=41001 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41001: stopping
2024-11-22T03:14:19,269 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:19,269 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:19,274 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68 (was 67) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=5930 (was 5930)
2024-11-22T03:14:19,279 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=5930
2024-11-22T03:14:19,288 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:19,288 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:19,290 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245259290,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:19,292 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,293 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33781
2024-11-22T03:14:19,294 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:19,294 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,294 INFO  [Listener,port=33781 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33781: starting
2024-11-22T03:14:19,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@432c2cb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,295 DEBUG [Thread-114 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:19,296 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,299 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245259298,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,300 DEBUG [Reader=1,bindAddress=localhost,port=33781 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,300 DEBUG [Reader=1,bindAddress=localhost,port=33781 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:19,300 DEBUG [Reader=1,bindAddress=localhost,port=33781 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,301 DEBUG [Reader=1,bindAddress=localhost,port=33781 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-22T03:14:19,301 DEBUG [Reader=1,bindAddress=localhost,port=33781 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:19,301 DEBUG [Reader=1,bindAddress=localhost,port=33781 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,301 DEBUG [Reader=1,bindAddress=localhost,port=33781 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-22T03:14:19,302 DEBUG [Reader=1,bindAddress=localhost,port=33781 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-22T03:14:19,302 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:19,302 DEBUG [Reader=1,bindAddress=localhost,port=33781 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,302 DEBUG [Reader=1,bindAddress=localhost,port=33781 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,302 DEBUG [Reader=1,bindAddress=localhost,port=33781 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:19,302 INFO  [Reader=1,bindAddress=localhost,port=33781 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:19,303 INFO  [Reader=1,bindAddress=localhost,port=33781 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37092, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:19,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,308 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33781
2024-11-22T03:14:19,308 INFO  [Listener,port=33781 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33781: stopping
2024-11-22T03:14:19,308 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:19,308 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:19,314 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=70 (was 68)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-15
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.safeSetSuccess(AbstractChannel.java:991)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:821)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.tryTerminate(ThreadPoolExecutor.java:710)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.processWorkerExit(ThreadPoolExecutor.java:1002)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1150)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=5929 (was 5930)
2024-11-22T03:14:19,320 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=5929
2024-11-22T03:14:19,329 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:19,329 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:19,332 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245259332,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:19,334 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,335 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39367
2024-11-22T03:14:19,335 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-22T03:14:19,335 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,335 INFO  [Listener,port=39367 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39367: starting
2024-11-22T03:14:19,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b6d1c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,336 DEBUG [Thread-117 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:19,340 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,342 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245259342,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,344 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:19,345 INFO  [Reader=1,bindAddress=localhost,port=39367 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-22T03:14:19,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,351 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39367
2024-11-22T03:14:19,351 INFO  [Listener,port=39367 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39367: stopping
2024-11-22T03:14:19,351 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-22T03:14:19,351 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-22T03:14:19,356 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=70 (was 69) - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=5903 (was 5929)
2024-11-22T03:14:19,362 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=70, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=5903
2024-11-22T03:14:19,371 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:19,371 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:19,373 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245259372,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:19,374 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,374 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:19,375 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43227
2024-11-22T03:14:19,375 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5150fd98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,376 DEBUG [Thread-118 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:19,377 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,379 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245259379,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,381 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,381 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,382 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:19,383 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,383 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:19,384 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,384 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,384 DEBUG [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:19,384 INFO  [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:19,385 INFO  [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:45954, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:19,386 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:19,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,394 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43227
2024-11-22T03:14:19,400 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72 (was 70)
Potentially hanging thread: NettyRpcServer-19-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-19-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=475 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=5903 (was 5903)
2024-11-22T03:14:19,406 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72, OpenFileDescriptor=475, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=5902
2024-11-22T03:14:19,415 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:19,416 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:19,418 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245259418,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:19,420 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,420 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:19,421 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44775
2024-11-22T03:14:19,421 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@163dc5be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-22T03:14:19,422 DEBUG [Thread-119 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:19,423 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,425 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245259425,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,427 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-22T03:14:19,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,427 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:44775 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
2024-11-22T03:14:19,427 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44775
2024-11-22T03:14:19,433 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74 (was 72)
Potentially hanging thread: NettyRpcServer-20-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-20-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=484 (was 475) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=5901 (was 5902)
2024-11-22T03:14:19,439 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74, OpenFileDescriptor=484, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=5900
2024-11-22T03:14:19,451 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:19,451 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:19,453 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245259453,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:19,454 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,455 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:19,455 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45317
2024-11-22T03:14:19,455 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c1e8b88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,456 DEBUG [Thread-120 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:19,457 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,460 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245259460,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,461 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,461 DEBUG [NettyRpcServer-21-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,463 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:19,463 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,464 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:19,464 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-22T03:14:19,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,464 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:45317 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server
2024-11-22T03:14:19,464 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45317
2024-11-22T03:14:19,470 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76 (was 74)
Potentially hanging thread: NettyRpcServer-21-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-21-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=493 (was 484) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=5898 (was 5900)
2024-11-22T03:14:19,476 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76, OpenFileDescriptor=493, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=5898
2024-11-22T03:14:19,484 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732245259453
2024-11-22T03:14:19,486 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:19,486 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:19,487 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245259487,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:19,489 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,489 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:19,490 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36763
2024-11-22T03:14:19,490 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@384237c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,491 DEBUG [Thread-121 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:19,492 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,496 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245259495,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,497 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,497 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,499 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:19,499 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,499 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:19,500 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,500 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:19,500 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,500 DEBUG [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:19,501 INFO  [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:19,501 INFO  [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:58814, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:19,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,507 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36763
2024-11-22T03:14:19,508 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,508 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:19,509 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40561
2024-11-22T03:14:19,509 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce12b9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,553 DEBUG [Thread-122 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:19,554 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,557 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,557 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,559 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:19,560 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,560 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:19,561 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,561 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:19,561 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,561 DEBUG [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:19,561 INFO  [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:19,561 INFO  [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43960, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:19,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,569 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40561
2024-11-22T03:14:19,570 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,570 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:19,571 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44291
2024-11-22T03:14:19,571 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@264475e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,572 DEBUG [Thread-123 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:19,573 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,577 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,577 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,578 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:19,579 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,579 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:19,580 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:19,580 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,580 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,580 DEBUG [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:19,580 INFO  [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:19,581 INFO  [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:32840, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:19,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,593 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44291
2024-11-22T03:14:19,594 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,595 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:19,596 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:36655
2024-11-22T03:14:19,596 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e3d814, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,597 DEBUG [Thread-124 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:19,598 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,600 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,601 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,602 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:19,602 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,602 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:19,603 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,603 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,603 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:19,603 DEBUG [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-22T03:14:19,603 INFO  [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:19,604 INFO  [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54524, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:19,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,615 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:36655
2024-11-22T03:14:19,615 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,616 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:19,617 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41295
2024-11-22T03:14:19,617 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b7881fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,620 DEBUG [Thread-125 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:19,621 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,623 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,623 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,624 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:19,625 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,625 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:19,626 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,626 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,626 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:19,626 DEBUG [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:19,626 INFO  [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:19,627 INFO  [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37398, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:19,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,636 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41295
2024-11-22T03:14:19,645 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86 (was 76)
Potentially hanging thread: NettyRpcServer-24-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-25-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-22-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-23-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-26-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-26-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-23-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-22-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-24-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-25-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=538 (was 493) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=5885 (was 5898)
2024-11-22T03:14:19,655 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86, OpenFileDescriptor=540, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=5884
2024-11-22T03:14:19,669 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:19,669 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:19,671 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245259671,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:19,673 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,674 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:19,675 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42723
2024-11-22T03:14:19,675 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38bbe685, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,676 DEBUG [Thread-126 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-22T03:14:19,678 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-22T03:14:19,678 WARN  [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-22T03:14:19,678 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-22T03:14:19,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,679 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42723
2024-11-22T03:14:19,685 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88 (was 86) - Thread LEAK? -, OpenFileDescriptor=547 (was 540) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=5882 (was 5884)
2024-11-22T03:14:19,691 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88, OpenFileDescriptor=547, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=5882
2024-11-22T03:14:19,701 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:19,701 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:19,703 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245259703,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:19,704 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,705 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:19,705 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33531
2024-11-22T03:14:19,706 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-22T03:14:19,706 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-22T03:14:19,706 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-22T03:14:19,706 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-22T03:14:19,706 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-22T03:14:19,706 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-22T03:14:19,706 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-22T03:14:19,706 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-22T03:14:19,706 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@211c1f3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,707 DEBUG [Thread-127 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-22T03:14:19,709 WARN  [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-22T03:14:19,709 INFO  [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-22T03:14:19,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,714 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33531
2024-11-22T03:14:19,721 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90 (was 88) - Thread LEAK? -, OpenFileDescriptor=556 (was 547) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=5880 (was 5882)
2024-11-22T03:14:19,729 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90, OpenFileDescriptor=556, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=5878
2024-11-22T03:14:19,739 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:19,739 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:19,741 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245259741,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:19,743 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,744 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:19,747 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34173
2024-11-22T03:14:19,747 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5681b81a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,748 DEBUG [Thread-128 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:19,749 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,752 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245259752,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,756 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,756 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,757 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:19,758 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,758 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:19,759 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:19,759 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,759 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,759 DEBUG [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:19,759 INFO  [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:19,760 INFO  [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54232, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:19,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,767 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34173
2024-11-22T03:14:19,774 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92 (was 90) - Thread LEAK? -, OpenFileDescriptor=565 (was 556) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=5874 (was 5878)
2024-11-22T03:14:19,779 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92, OpenFileDescriptor=565, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=5873
2024-11-22T03:14:19,788 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:19,788 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:19,790 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245259790,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:19,791 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,792 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:19,792 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34093
2024-11-22T03:14:19,792 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c90c619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,793 DEBUG [Thread-129 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:19,794 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,797 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245259797,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,798 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,798 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,799 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:19,799 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,800 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:19,800 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,800 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,800 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:19,800 DEBUG [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:19,800 INFO  [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:19,801 INFO  [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:34664, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:19,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:19,808 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34093
2024-11-22T03:14:19,809 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:19,809 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:19,810 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45247
2024-11-22T03:14:19,810 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:19,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0df70d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:19,811 DEBUG [Thread-130 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:19,811 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,813 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,813 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,814 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:19,815 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,815 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:19,815 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:19,815 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:19,815 DEBUG [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-22T03:14:19,815 INFO  [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:19,816 INFO  [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60040, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:21,816 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-22T03:14:21,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:21,817 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:45247 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException
2024-11-22T03:14:21,817 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45247
2024-11-22T03:14:21,821 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1732245259791
2024-11-22T03:14:21,823 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96 (was 92)
Potentially hanging thread: NettyRpcServer-30-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-31-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-31-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-30-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=583 (was 565) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=5880 (was 5873) - AvailableMemoryMB LEAK? -
2024-11-22T03:14:21,829 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96, OpenFileDescriptor=583, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=5880
2024-11-22T03:14:21,839 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:21,839 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:21,841 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245261841,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:21,843 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:21,843 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:21,844 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45941
2024-11-22T03:14:21,844 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:21,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62bd6455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:21,845 DEBUG [Thread-131 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:21,846 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:21,849 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245261849,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:21,850 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:21,850 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:21,852 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:21,852 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:21,852 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:21,853 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:21,853 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:21,853 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:21,853 DEBUG [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:21,853 INFO  [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:21,853 INFO  [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:57828, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:21,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:21,859 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45941
2024-11-22T03:14:21,865 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98 (was 96) - Thread LEAK? -, OpenFileDescriptor=592 (was 583) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=5879 (was 5880)
2024-11-22T03:14:21,871 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98, OpenFileDescriptor=592, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=5879
2024-11-22T03:14:21,881 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:21,881 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:21,883 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245261883,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:21,885 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:21,885 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:21,886 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42165
2024-11-22T03:14:21,886 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:21,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f494305, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:21,887 DEBUG [Thread-132 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:21,887 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:21,890 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245261889,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:21,891 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:21,891 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:21,892 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-22T03:14:21,893 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:21,893 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-22T03:14:21,893 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-22T03:14:21,894 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:21,894 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:21,894 DEBUG [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-22T03:14:21,894 INFO  [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-22T03:14:21,894 INFO  [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53150, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-22T03:14:21,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:21,899 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42165
2024-11-22T03:14:21,906 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100 (was 98) - Thread LEAK? -, OpenFileDescriptor=601 (was 592) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=5878 (was 5879)
2024-11-22T03:14:21,912 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100, OpenFileDescriptor=601, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=5877
2024-11-22T03:14:21,921 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-22T03:14:21,921 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-22T03:14:21,922 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1732245261922,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-22T03:14:21,924 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-22T03:14:21,924 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T03:14:21,925 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35731
2024-11-22T03:14:21,925 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-22T03:14:21,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53739854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T03:14:21,926 DEBUG [Thread-133 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-22T03:14:21,927 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:21,930 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1732245261929,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-22T03:14:21,931 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-22T03:14:21,932 INFO  [NettyRpcServer-34-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-22T03:14:21,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T03:14:21,937 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35731
2024-11-22T03:14:21,943 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=102 (was 100) - Thread LEAK? -, OpenFileDescriptor=610 (was 601) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=5876 (was 5877)
2024-11-22T03:14:21,944 INFO  [Time-limited test {}] impl.DefaultInternalKdcServerImpl(102): Default Internal kdc server stopped.