2024-11-15 14:13:05,913 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116
2024-11-15 14:13:05,931 main DEBUG Took 0.015168 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-15 14:13:05,932 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-15 14:13:05,933 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-15 14:13:05,936 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-15 14:13:05,938 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-15 14:13:05,963 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-15 14:13:05,985 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-15 14:13:05,988 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-15 14:13:05,989 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-15 14:13:05,989 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-15 14:13:05,990 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-15 14:13:05,991 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-15 14:13:05,992 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-15 14:13:05,993 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-15 14:13:05,994 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-15 14:13:05,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-15 14:13:05,996 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-15 14:13:05,996 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-15 14:13:05,997 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-15 14:13:05,998 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-15 14:13:05,999 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-15 14:13:05,999 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-15 14:13:06,000 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-15 14:13:06,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-15 14:13:06,001 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-15 14:13:06,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-15 14:13:06,003 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-15 14:13:06,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-15 14:13:06,004 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-15 14:13:06,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-15 14:13:06,005 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-15 14:13:06,006 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-15 14:13:06,010 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-15 14:13:06,013 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-15 14:13:06,019 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-15 14:13:06,020 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-15 14:13:06,024 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-15 14:13:06,024 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-15 14:13:06,039 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-15 14:13:06,049 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-15 14:13:06,053 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-15 14:13:06,054 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-15 14:13:06,057 main DEBUG createAppenders(={Console})
2024-11-15 14:13:06,059 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 initialized
2024-11-15 14:13:06,061 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116
2024-11-15 14:13:06,061 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@38467116 OK.
2024-11-15 14:13:06,063 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-15 14:13:06,067 main DEBUG OutputStream closed
2024-11-15 14:13:06,069 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-15 14:13:06,070 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-15 14:13:06,070 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5d0a1059 OK
2024-11-15 14:13:06,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-15 14:13:06,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-15 14:13:06,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-15 14:13:06,198 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-15 14:13:06,201 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-15 14:13:06,202 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-15 14:13:06,203 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-15 14:13:06,203 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-15 14:13:06,204 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-15 14:13:06,205 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-15 14:13:06,205 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-15 14:13:06,206 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-15 14:13:06,206 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-15 14:13:06,207 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-15 14:13:06,207 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-15 14:13:06,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-15 14:13:06,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-15 14:13:06,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-15 14:13:06,212 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-15 14:13:06,212 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5812f68b) with optional ClassLoader: null
2024-11-15 14:13:06,213 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-15 14:13:06,214 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5812f68b] started OK.
2024-11-15T14:13:06,729 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/945f70aa-634f-a245-a70c-31bc396d0d57
2024-11-15 14:13:06,733 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-15 14:13:06,735 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-15T14:13:06,764 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.security.TestSecureIPC timeout: 52 mins
2024-11-15T14:13:07,143 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=13, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=594, ProcessCount=11, AvailableMemoryMB=1022
2024-11-15T14:13:07,408 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-15T14:13:07,608 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:07,613 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:07,727 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679987713,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:08,925 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:09,018 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-15T14:13:09,055 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-15T14:13:09,073 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-15T14:13:09,097 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44903
2024-11-15T14:13:09,104 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:09,105 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:09,105 INFO  [Listener,port=44903 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44903: starting
2024-11-15T14:13:09,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c1f4227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:09,293 DEBUG [Thread-6 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:09,361 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44903
2024-11-15T14:13:09,378 DEBUG [Thread-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:09,482 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679989478,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:09,501 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:09,508 DEBUG [Reader=1,bindAddress=localhost,port=44903 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:09,520 DEBUG [Reader=1,bindAddress=localhost,port=44903 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:09,521 DEBUG [Reader=1,bindAddress=localhost,port=44903 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:09,542 DEBUG [Reader=1,bindAddress=localhost,port=44903 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:09,545 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:09,547 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:09,552 DEBUG [Reader=1,bindAddress=localhost,port=44903 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:09,552 DEBUG [Reader=1,bindAddress=localhost,port=44903 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:09,554 DEBUG [Reader=1,bindAddress=localhost,port=44903 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:09,555 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:09,556 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:09,556 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-15T14:13:09,557 DEBUG [Reader=1,bindAddress=localhost,port=44903 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:09,557 DEBUG [Thread-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:09,559 DEBUG [Reader=1,bindAddress=localhost,port=44903 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:09,560 DEBUG [Reader=1,bindAddress=localhost,port=44903 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:09,560 DEBUG [Reader=1,bindAddress=localhost,port=44903 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:09,561 INFO  [Reader=1,bindAddress=localhost,port=44903 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:09,575 INFO  [Reader=1,bindAddress=localhost,port=44903 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:47258, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:09,575 DEBUG [Thread-6 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93
2024-11-15T14:13:09,582 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:247
2024-11-15T14:13:09,587 DEBUG [BRPC Connection (1322466407) to localhost:44903 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-15T14:13:09,624 DEBUG [BRPC Connection (1322466407) to localhost:44903 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:233
2024-11-15T14:13:09,629 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2264
2024-11-15T14:13:09,631 DEBUG [BRPC Connection (1322466407) to localhost:44903 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-15T14:13:09,636 DEBUG [BRPC Connection (1322466407) to localhost:44903 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2250
2024-11-15T14:13:09,655 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117
2024-11-15T14:13:09,657 DEBUG [Thread-6 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18180
2024-11-15T14:13:09,672 DEBUG [BRPC Connection (1322466407) to localhost:44903 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-15T14:13:09,698 DEBUG [BRPC Connection (1322466407) to localhost:44903 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22283
2024-11-15T14:13:09,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:09,711 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44903
2024-11-15T14:13:09,712 INFO  [Listener,port=44903 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44903: stopping
2024-11-15T14:13:09,713 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:09,713 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:09,726 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=21 (was 13)
Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=1,bindAddress=localhost,port=44903
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: pool-3-thread-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=44903
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=594 (was 594), ProcessCount=11 (was 11), AvailableMemoryMB=771 (was 1022)
2024-11-15T14:13:09,736 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=594, ProcessCount=11, AvailableMemoryMB=770
2024-11-15T14:13:09,752 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:09,752 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:09,767 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679989766,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:09,785 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:09,794 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33413
2024-11-15T14:13:09,798 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:09,798 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:09,798 INFO  [Listener,port=33413 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33413: starting
2024-11-15T14:13:09,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28de070a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-15T14:13:09,815 DEBUG [Thread-11 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:09,816 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33413
2024-11-15T14:13:09,819 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:09,833 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679989832,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:09,838 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:09,839 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-15T14:13:09,840 DEBUG [Thread-11 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:33413
org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-15T14:13:09,845 WARN  [Thread-11 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731679989771
2024-11-15T14:13:09,940 DEBUG [Thread-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:09,945 DEBUG [Thread-11 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:09,945 DEBUG [Thread-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-15T14:13:09,946 DEBUG [Thread-11 {}] ipc.FailedServers(52): Added failed server with address localhost:33413 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:09,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:09,948 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33413
2024-11-15T14:13:09,949 INFO  [Listener,port=33413 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33413: stopping
2024-11-15T14:13:09,951 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:09,951 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:09,965 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 594) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=748 (was 770)
2024-11-15T14:13:09,978 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=748
2024-11-15T14:13:10,000 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:10,000 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:10,015 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679990013,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:10,033 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:10,039 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45001
2024-11-15T14:13:10,043 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:10,043 INFO  [Listener,port=45001 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45001: starting
2024-11-15T14:13:10,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20ba6ba4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:10,043 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:10,045 DEBUG [Thread-14 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:10,046 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45001
2024-11-15T14:13:10,048 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,060 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679990059,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,063 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:10,063 DEBUG [Reader=1,bindAddress=localhost,port=45001 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,065 DEBUG [Reader=1,bindAddress=localhost,port=45001 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,066 DEBUG [Reader=1,bindAddress=localhost,port=45001 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,069 DEBUG [Reader=1,bindAddress=localhost,port=45001 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:10,071 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:10,072 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:10,072 DEBUG [Reader=1,bindAddress=localhost,port=45001 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,072 DEBUG [Reader=1,bindAddress=localhost,port=45001 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,073 DEBUG [Reader=1,bindAddress=localhost,port=45001 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:10,073 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:10,075 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-15T14:13:10,077 DEBUG [Thread-14 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:45001
javax.security.sasl.SaslException: No common protection layer between client and server
	at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?]
	at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?]
	at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-15T14:13:10,078 WARN  [Thread-14 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731679990030
2024-11-15T14:13:10,110 DEBUG [Thread-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,113 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:10,114 DEBUG [Reader=2,bindAddress=localhost,port=45001 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,116 DEBUG [Reader=2,bindAddress=localhost,port=45001 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,117 DEBUG [Reader=2,bindAddress=localhost,port=45001 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,120 DEBUG [Reader=2,bindAddress=localhost,port=45001 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:10,120 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:10,121 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:10,121 DEBUG [Reader=2,bindAddress=localhost,port=45001 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,121 DEBUG [Reader=2,bindAddress=localhost,port=45001 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,122 DEBUG [Reader=2,bindAddress=localhost,port=45001 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:10,122 DEBUG [Thread-14 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:10,123 DEBUG [Thread-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-15T14:13:10,123 DEBUG [Thread-14 {}] ipc.FailedServers(52): Added failed server with address localhost:45001 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:10,126 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45001
2024-11-15T14:13:10,127 INFO  [Listener,port=45001 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45001: stopping
2024-11-15T14:13:10,128 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:10,128 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:10,139 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=23 (was 19)
Potentially hanging thread: Reader=1,bindAddress=localhost,port=45001
	java.base@17.0.11/sun.nio.ch.EventFD.set0(Native Method)
	java.base@17.0.11/sun.nio.ch.EventFD.set(EventFD.java:48)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:251)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:183)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:151)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=45001
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer idle connection scanner for port 45001
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:371)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243)
	java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499)
	app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97)
	app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121)
	app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=696 (was 748)
2024-11-15T14:13:10,152 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=694
2024-11-15T14:13:10,169 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:10,170 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:10,178 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679990177,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:10,183 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:10,189 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40943
2024-11-15T14:13:10,191 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:10,191 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:10,192 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45de5980, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:10,191 INFO  [Listener,port=40943 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40943: starting
2024-11-15T14:13:10,196 DEBUG [Thread-17 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:10,197 DEBUG [Thread-17 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40943
2024-11-15T14:13:10,198 DEBUG [Thread-17 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,210 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679990209,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,214 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:10,219 DEBUG [Reader=1,bindAddress=localhost,port=40943 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,221 DEBUG [Reader=1,bindAddress=localhost,port=40943 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,221 DEBUG [Reader=1,bindAddress=localhost,port=40943 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,225 DEBUG [Reader=1,bindAddress=localhost,port=40943 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:10,226 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:10,227 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:10,227 DEBUG [Reader=1,bindAddress=localhost,port=40943 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,228 DEBUG [Reader=1,bindAddress=localhost,port=40943 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,228 DEBUG [Reader=1,bindAddress=localhost,port=40943 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:10,229 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:10,230 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:10,230 DEBUG [Thread-17 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-15T14:13:10,230 DEBUG [Thread-17 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:10,231 DEBUG [Reader=1,bindAddress=localhost,port=40943 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,231 DEBUG [Reader=1,bindAddress=localhost,port=40943 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,231 DEBUG [Reader=1,bindAddress=localhost,port=40943 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,232 DEBUG [Reader=1,bindAddress=localhost,port=40943 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:10,232 INFO  [Reader=1,bindAddress=localhost,port=40943 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:10,232 INFO  [Reader=1,bindAddress=localhost,port=40943 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36800, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:10,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:10,290 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40943
2024-11-15T14:13:10,290 INFO  [Listener,port=40943 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40943: stopping
2024-11-15T14:13:10,291 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:10,291 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:10,292 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:10,306 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40101
2024-11-15T14:13:10,311 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:10,315 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:10,315 INFO  [Listener,port=40101 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40101: starting
2024-11-15T14:13:10,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e4e4104, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:10,320 DEBUG [Thread-20 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:10,321 DEBUG [Thread-20 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40101
2024-11-15T14:13:10,323 DEBUG [Thread-20 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,327 DEBUG [Reader=1,bindAddress=localhost,port=40101 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,328 DEBUG [Reader=1,bindAddress=localhost,port=40101 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,328 DEBUG [Reader=1,bindAddress=localhost,port=40101 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,330 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:10,332 DEBUG [Reader=1,bindAddress=localhost,port=40101 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:10,335 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:10,336 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:10,336 DEBUG [Reader=1,bindAddress=localhost,port=40101 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,337 DEBUG [Reader=1,bindAddress=localhost,port=40101 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,337 DEBUG [Reader=1,bindAddress=localhost,port=40101 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:10,338 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:10,340 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:10,341 DEBUG [Thread-20 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-15T14:13:10,341 DEBUG [Thread-20 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:10,341 DEBUG [Reader=1,bindAddress=localhost,port=40101 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,341 DEBUG [Reader=1,bindAddress=localhost,port=40101 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,341 DEBUG [Reader=1,bindAddress=localhost,port=40101 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,342 DEBUG [Reader=1,bindAddress=localhost,port=40101 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:10,342 INFO  [Reader=1,bindAddress=localhost,port=40101 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:10,342 INFO  [Reader=1,bindAddress=localhost,port=40101 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50508, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:10,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:10,411 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40101
2024-11-15T14:13:10,412 INFO  [Listener,port=40101 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40101: stopping
2024-11-15T14:13:10,412 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:10,412 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:10,412 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:10,422 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44761
2024-11-15T14:13:10,437 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:10,438 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:10,438 INFO  [Listener,port=44761 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44761: starting
2024-11-15T14:13:10,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b54c2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:10,442 DEBUG [Thread-23 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:10,442 DEBUG [Thread-23 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44761
2024-11-15T14:13:10,444 DEBUG [Thread-23 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,447 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:10,447 DEBUG [Reader=1,bindAddress=localhost,port=44761 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,449 DEBUG [Reader=1,bindAddress=localhost,port=44761 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,449 DEBUG [Reader=1,bindAddress=localhost,port=44761 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,453 DEBUG [Reader=1,bindAddress=localhost,port=44761 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:10,458 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:10,459 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:10,459 DEBUG [Reader=1,bindAddress=localhost,port=44761 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,459 DEBUG [Reader=1,bindAddress=localhost,port=44761 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,460 DEBUG [Reader=1,bindAddress=localhost,port=44761 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:10,461 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:10,462 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:10,462 DEBUG [Reader=1,bindAddress=localhost,port=44761 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,462 DEBUG [Reader=1,bindAddress=localhost,port=44761 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,463 DEBUG [Reader=1,bindAddress=localhost,port=44761 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,463 DEBUG [Reader=1,bindAddress=localhost,port=44761 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:10,463 INFO  [Reader=1,bindAddress=localhost,port=44761 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:10,464 DEBUG [Thread-23 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-15T14:13:10,464 DEBUG [Thread-23 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:10,467 INFO  [Reader=1,bindAddress=localhost,port=44761 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41444, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:10,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:10,507 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44761
2024-11-15T14:13:10,509 INFO  [Listener,port=44761 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44761: stopping
2024-11-15T14:13:10,509 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:10,510 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:10,511 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:10,522 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43793
2024-11-15T14:13:10,529 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:10,540 INFO  [Listener,port=43793 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43793: starting
2024-11-15T14:13:10,540 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:10,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4902eac6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:10,559 DEBUG [Thread-26 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:10,563 DEBUG [Thread-26 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43793
2024-11-15T14:13:10,565 DEBUG [Thread-26 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,567 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:10,568 DEBUG [Reader=1,bindAddress=localhost,port=43793 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,570 DEBUG [Reader=1,bindAddress=localhost,port=43793 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,570 DEBUG [Reader=1,bindAddress=localhost,port=43793 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,572 DEBUG [Reader=1,bindAddress=localhost,port=43793 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:10,573 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:10,573 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:10,574 DEBUG [Reader=1,bindAddress=localhost,port=43793 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,574 DEBUG [Reader=1,bindAddress=localhost,port=43793 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,575 DEBUG [Reader=1,bindAddress=localhost,port=43793 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:10,576 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:10,576 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:10,576 DEBUG [Thread-26 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int
2024-11-15T14:13:10,577 DEBUG [Reader=1,bindAddress=localhost,port=43793 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,577 DEBUG [Thread-26 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:10,577 DEBUG [Reader=1,bindAddress=localhost,port=43793 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,577 DEBUG [Reader=1,bindAddress=localhost,port=43793 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,577 DEBUG [Reader=1,bindAddress=localhost,port=43793 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-15T14:13:10,577 INFO  [Reader=1,bindAddress=localhost,port=43793 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:10,578 INFO  [Reader=1,bindAddress=localhost,port=43793 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55106, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:10,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:10,608 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43793
2024-11-15T14:13:10,609 INFO  [Listener,port=43793 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43793: stopping
2024-11-15T14:13:10,610 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:10,611 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:10,611 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:10,619 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44575
2024-11-15T14:13:10,625 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:10,629 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:10,629 INFO  [Listener,port=44575 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44575: starting
2024-11-15T14:13:10,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d35dd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:10,637 DEBUG [Thread-29 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:10,637 DEBUG [Thread-29 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44575
2024-11-15T14:13:10,639 DEBUG [Thread-29 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,642 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:10,642 DEBUG [Reader=1,bindAddress=localhost,port=44575 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,643 DEBUG [Reader=1,bindAddress=localhost,port=44575 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,643 DEBUG [Reader=1,bindAddress=localhost,port=44575 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,646 DEBUG [Reader=1,bindAddress=localhost,port=44575 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:10,647 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:10,648 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:10,648 DEBUG [Reader=1,bindAddress=localhost,port=44575 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,648 DEBUG [Reader=1,bindAddress=localhost,port=44575 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,652 DEBUG [Reader=1,bindAddress=localhost,port=44575 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:10,653 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:10,653 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:10,654 DEBUG [Thread-29 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-15T14:13:10,654 DEBUG [Thread-29 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:10,654 DEBUG [Reader=1,bindAddress=localhost,port=44575 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:10,655 DEBUG [Reader=1,bindAddress=localhost,port=44575 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:10,661 DEBUG [Reader=1,bindAddress=localhost,port=44575 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:10,661 DEBUG [Reader=1,bindAddress=localhost,port=44575 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:10,662 INFO  [Reader=1,bindAddress=localhost,port=44575 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:10,662 INFO  [Reader=1,bindAddress=localhost,port=44575 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54680, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:10,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:10,697 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44575
2024-11-15T14:13:10,698 INFO  [Listener,port=44575 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44575: stopping
2024-11-15T14:13:10,701 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:10,701 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:10,723 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=23 (was 19)
Potentially hanging thread: RpcServer idle connection scanner for port 44575
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=44575
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=1,bindAddress=localhost,port=44575
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=657 (was 694)
2024-11-15T14:13:10,737 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=657
2024-11-15T14:13:10,757 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:10,758 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:10,769 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679990768,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:10,782 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:10,795 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34671
2024-11-15T14:13:10,798 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:10,800 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:10,800 INFO  [Listener,port=34671 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34671: starting
2024-11-15T14:13:10,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@426ede81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:10,809 DEBUG [Thread-32 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-15T14:13:10,809 DEBUG [Thread-32 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34671
2024-11-15T14:13:10,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:10,818 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34671
2024-11-15T14:13:10,819 INFO  [Listener,port=34671 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34671: stopping
2024-11-15T14:13:10,819 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:10,819 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:10,839 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=22 (was 19)
Potentially hanging thread: Reader=0,bindAddress=localhost,port=34671
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=2,bindAddress=localhost,port=34671
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=1,bindAddress=localhost,port=34671
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.workerCountOf(ThreadPoolExecutor.java:398)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.tryTerminate(ThreadPoolExecutor.java:715)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.processWorkerExit(ThreadPoolExecutor.java:1002)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1150)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=292 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=645 (was 657)
2024-11-15T14:13:10,848 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=645
2024-11-15T14:13:10,871 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:10,872 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:10,883 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679990882,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:10,889 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:10,904 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35353
2024-11-15T14:13:10,904 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-15T14:13:10,905 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-15T14:13:10,905 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-15T14:13:10,905 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-15T14:13:10,905 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-15T14:13:10,905 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-15T14:13:10,905 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-15T14:13:10,905 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-15T14:13:10,906 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:10,906 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:10,906 INFO  [Listener,port=35353 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35353: starting
2024-11-15T14:13:10,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@465b0e50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:10,911 DEBUG [Thread-35 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-15T14:13:10,912 DEBUG [Thread-35 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35353
2024-11-15T14:13:10,913 WARN  [Reader=1,bindAddress=localhost,port=35353 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-15T14:13:10,914 INFO  [Reader=1,bindAddress=localhost,port=35353 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-15T14:13:10,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:10,954 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35353
2024-11-15T14:13:10,955 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:10,957 INFO  [Listener,port=35353 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35353: stopping
2024-11-15T14:13:10,957 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:10,968 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=640 (was 645)
2024-11-15T14:13:10,980 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=639
2024-11-15T14:13:11,000 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:11,001 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:11,009 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679991008,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:11,013 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:11,028 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38879
2024-11-15T14:13:11,031 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:11,032 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:11,032 INFO  [Listener,port=38879 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38879: starting
2024-11-15T14:13:11,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa1b204, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:11,034 DEBUG [Thread-38 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:11,034 DEBUG [Thread-38 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38879
2024-11-15T14:13:11,036 DEBUG [Thread-38 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:11,049 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679991048,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:11,052 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(103): Have sent token of size 498 from initSASLContext.
2024-11-15T14:13:11,052 DEBUG [Reader=1,bindAddress=localhost,port=38879 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:11,053 DEBUG [Reader=1,bindAddress=localhost,port=38879 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:11,053 DEBUG [Reader=1,bindAddress=localhost,port=38879 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=498 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:11,055 DEBUG [Reader=1,bindAddress=localhost,port=38879 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:11,056 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:11,057 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:11,058 DEBUG [Reader=1,bindAddress=localhost,port=38879 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:11,058 DEBUG [Reader=1,bindAddress=localhost,port=38879 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:11,059 DEBUG [Reader=1,bindAddress=localhost,port=38879 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:11,060 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:11,060 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:11,061 DEBUG [Thread-38 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-15T14:13:11,061 DEBUG [Reader=1,bindAddress=localhost,port=38879 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:11,061 DEBUG [Reader=1,bindAddress=localhost,port=38879 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:11,061 DEBUG [Thread-38 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:11,061 DEBUG [Reader=1,bindAddress=localhost,port=38879 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:11,062 DEBUG [Reader=1,bindAddress=localhost,port=38879 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:11,062 INFO  [Reader=1,bindAddress=localhost,port=38879 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:11,062 INFO  [Reader=1,bindAddress=localhost,port=38879 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36932, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:11,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:11,082 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38879
2024-11-15T14:13:11,083 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:11,084 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:11,084 INFO  [Listener,port=38879 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38879: stopping
2024-11-15T14:13:11,094 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=23 (was 19)
Potentially hanging thread: RpcServer idle connection scanner for port 38879
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Listener,port=38879
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:213)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243)
	java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499)
	app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97)
	app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121)
	app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:65)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=623 (was 639)
2024-11-15T14:13:11,110 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=622
2024-11-15T14:13:11,136 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:11,136 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:11,143 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679991143,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:11,147 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:11,151 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43117
2024-11-15T14:13:11,155 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:11,155 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:11,155 INFO  [Listener,port=43117 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43117: starting
2024-11-15T14:13:11,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d59984, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:11,157 DEBUG [Thread-41 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:11,157 DEBUG [Thread-41 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43117
2024-11-15T14:13:11,159 DEBUG [Thread-41 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:11,166 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679991165,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:11,169 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:11,169 DEBUG [Reader=1,bindAddress=localhost,port=43117 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:11,170 DEBUG [Reader=1,bindAddress=localhost,port=43117 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:11,170 DEBUG [Reader=1,bindAddress=localhost,port=43117 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:11,172 DEBUG [Reader=1,bindAddress=localhost,port=43117 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:11,172 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:11,173 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:11,173 DEBUG [Reader=1,bindAddress=localhost,port=43117 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:11,173 DEBUG [Reader=1,bindAddress=localhost,port=43117 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:11,173 DEBUG [Reader=1,bindAddress=localhost,port=43117 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:11,174 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:11,174 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:11,174 DEBUG [Thread-41 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-15T14:13:11,174 DEBUG [Reader=1,bindAddress=localhost,port=43117 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:11,174 DEBUG [Thread-41 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:11,174 DEBUG [Reader=1,bindAddress=localhost,port=43117 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:11,175 DEBUG [Reader=1,bindAddress=localhost,port=43117 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:11,175 DEBUG [Reader=1,bindAddress=localhost,port=43117 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:11,175 INFO  [Reader=1,bindAddress=localhost,port=43117 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:11,176 INFO  [Reader=1,bindAddress=localhost,port=43117 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44938, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:11,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:11,195 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43117
2024-11-15T14:13:11,196 INFO  [Listener,port=43117 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43117: stopping
2024-11-15T14:13:11,196 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:11,196 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:11,197 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:11,199 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=33141
2024-11-15T14:13:11,207 INFO  [Listener,port=33141 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=33141: starting
2024-11-15T14:13:11,207 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:11,207 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:11,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4341f3b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:11,211 DEBUG [Thread-44 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:11,212 DEBUG [Thread-44 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33141
2024-11-15T14:13:11,213 DEBUG [Thread-44 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:11,215 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:11,215 DEBUG [Reader=1,bindAddress=localhost,port=33141 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:11,217 DEBUG [Reader=1,bindAddress=localhost,port=33141 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:11,217 DEBUG [Reader=1,bindAddress=localhost,port=33141 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:11,218 DEBUG [Reader=1,bindAddress=localhost,port=33141 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:11,219 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:11,219 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:11,220 DEBUG [Reader=1,bindAddress=localhost,port=33141 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:11,220 DEBUG [Reader=1,bindAddress=localhost,port=33141 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:11,220 DEBUG [Reader=1,bindAddress=localhost,port=33141 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:11,220 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:11,221 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:11,221 DEBUG [Thread-44 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-15T14:13:11,221 DEBUG [Reader=1,bindAddress=localhost,port=33141 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:11,221 DEBUG [Thread-44 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:11,221 DEBUG [Reader=1,bindAddress=localhost,port=33141 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:11,221 DEBUG [Reader=1,bindAddress=localhost,port=33141 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:11,221 DEBUG [Reader=1,bindAddress=localhost,port=33141 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:11,222 INFO  [Reader=1,bindAddress=localhost,port=33141 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:11,222 INFO  [Reader=1,bindAddress=localhost,port=33141 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48392, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:13,223 ERROR [Thread-44 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function.
java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:48392 remote=localhost/127.0.0.1:33141]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?]
	at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?]
	at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-15T14:13:13,224 DEBUG [Thread-44 {}] ipc.FailedServers(52): Added failed server with address localhost:33141 to list caused by java.io.IOException: Timeout while waiting connection header response
2024-11-15T14:13:13,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:13,225 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 33141
2024-11-15T14:13:13,225 INFO  [Listener,port=33141 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=33141: stopping
2024-11-15T14:13:13,225 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:13,225 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:13,232 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19 (was 19), OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=423 (was 622)
2024-11-15T14:13:13,238 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=422
2024-11-15T14:13:13,254 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:13,255 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:13,260 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679993259,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:13,263 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:13,264 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45753
2024-11-15T14:13:13,265 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:13,265 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:13,265 INFO  [Listener,port=45753 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45753: starting
2024-11-15T14:13:13,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3552636a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:13,267 DEBUG [Thread-47 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:13,267 DEBUG [Thread-47 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45753
2024-11-15T14:13:13,268 DEBUG [Thread-47 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,276 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679993276,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,278 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:13,279 DEBUG [Reader=1,bindAddress=localhost,port=45753 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,280 DEBUG [Reader=1,bindAddress=localhost,port=45753 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:13,280 DEBUG [Reader=1,bindAddress=localhost,port=45753 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:13,282 DEBUG [Reader=1,bindAddress=localhost,port=45753 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:13,282 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:13,283 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:13,283 DEBUG [Reader=1,bindAddress=localhost,port=45753 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:13,283 DEBUG [Reader=1,bindAddress=localhost,port=45753 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:13,283 DEBUG [Reader=1,bindAddress=localhost,port=45753 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:13,284 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:13,284 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:13,284 DEBUG [Thread-47 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-15T14:13:13,285 DEBUG [Reader=1,bindAddress=localhost,port=45753 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:13,285 DEBUG [Thread-47 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:13,285 DEBUG [Reader=1,bindAddress=localhost,port=45753 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:13,285 DEBUG [Reader=1,bindAddress=localhost,port=45753 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,285 DEBUG [Reader=1,bindAddress=localhost,port=45753 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:13,285 INFO  [Reader=1,bindAddress=localhost,port=45753 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:13,286 INFO  [Reader=1,bindAddress=localhost,port=45753 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42092, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:13,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:13,295 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45753
2024-11-15T14:13:13,295 INFO  [Listener,port=45753 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45753: stopping
2024-11-15T14:13:13,295 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:13,296 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:13,301 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=22 (was 19)
Potentially hanging thread: Reader=1,bindAddress=localhost,port=45753
	java.base@17.0.11/java.lang.ThreadGroup.threadTerminated(ThreadGroup.java:967)
	java.base@17.0.11/java.lang.Thread.exit(Thread.java:853)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:212)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.tryTerminate(ThreadPoolExecutor.java:711)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.processWorkerExit(ThreadPoolExecutor.java:1002)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1150)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=404 (was 422)
2024-11-15T14:13:13,307 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=403
2024-11-15T14:13:13,322 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:13,322 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:13,341 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679993335,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:13,345 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:13,355 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38065
2024-11-15T14:13:13,357 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:13,357 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:13,357 INFO  [Listener,port=38065 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38065: starting
2024-11-15T14:13:13,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a903243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:13,367 DEBUG [Thread-50 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:13,368 DEBUG [Thread-50 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:38065
2024-11-15T14:13:13,369 DEBUG [Thread-50 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,376 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679993376,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,379 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:13,379 DEBUG [Reader=1,bindAddress=localhost,port=38065 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,380 DEBUG [Reader=1,bindAddress=localhost,port=38065 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:13,380 DEBUG [Reader=1,bindAddress=localhost,port=38065 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:13,382 DEBUG [Reader=1,bindAddress=localhost,port=38065 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:13,383 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:13,383 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:13,384 DEBUG [Reader=1,bindAddress=localhost,port=38065 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:13,384 DEBUG [Reader=1,bindAddress=localhost,port=38065 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:13,384 DEBUG [Reader=1,bindAddress=localhost,port=38065 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:13,385 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:13,385 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:13,385 DEBUG [Thread-50 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-15T14:13:13,386 DEBUG [Reader=1,bindAddress=localhost,port=38065 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:13,386 DEBUG [Thread-50 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:13,386 DEBUG [Reader=1,bindAddress=localhost,port=38065 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:13,386 DEBUG [Reader=1,bindAddress=localhost,port=38065 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,386 DEBUG [Reader=1,bindAddress=localhost,port=38065 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:13,386 INFO  [Reader=1,bindAddress=localhost,port=38065 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:13,387 INFO  [Reader=1,bindAddress=localhost,port=38065 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44418, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:13,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:13,398 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38065
2024-11-15T14:13:13,399 INFO  [Listener,port=38065 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38065: stopping
2024-11-15T14:13:13,400 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:13,401 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:13,407 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=23 (was 19)
Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=38065
	java.base@17.0.11/jdk.internal.misc.Unsafe.checkPointer(Unsafe.java:544)
	java.base@17.0.11/jdk.internal.misc.Unsafe.freeMemoryChecks(Unsafe.java:936)
	java.base@17.0.11/jdk.internal.misc.Unsafe.freeMemory(Unsafe.java:918)
	java.base@17.0.11/sun.nio.ch.EPoll.freePollArray(EPoll.java:80)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:217)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:151)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=1,bindAddress=localhost,port=38065
	java.base@17.0.11/jdk.internal.misc.Unsafe.freeMemory(Unsafe.java:918)
	java.base@17.0.11/sun.nio.ch.EPoll.freePollArray(EPoll.java:80)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.implClose(EPollSelectorImpl.java:217)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:185)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:151)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=357 (was 403)
2024-11-15T14:13:13,415 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=19, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=356
2024-11-15T14:13:13,433 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:13,434 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:13,441 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679993439,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:13,445 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:13,449 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43967
2024-11-15T14:13:13,449 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:13,449 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:13,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5eb7be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:13,451 INFO  [Listener,port=43967 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43967: starting
2024-11-15T14:13:13,459 DEBUG [Thread-53 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:13,460 DEBUG [Thread-53 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43967
2024-11-15T14:13:13,461 DEBUG [Thread-53 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,475 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679993475,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,478 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:13,478 DEBUG [Thread-53 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth.
2024-11-15T14:13:13,478 DEBUG [Thread-53 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:13,479 INFO  [Reader=1,bindAddress=localhost,port=43967 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-15T14:13:13,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:13,500 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43967
2024-11-15T14:13:13,501 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:13,501 INFO  [Listener,port=43967 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43967: stopping
2024-11-15T14:13:13,501 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:13,508 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[0: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=24 (was 19)
Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=43967
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer idle connection scanner for port 43967
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Listener,port=43967
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243)
	java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499)
	app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97)
	app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121)
	app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:252)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 290), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=338 (was 356)
2024-11-15T14:13:13,515 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=19, OpenFileDescriptor=292, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=338
2024-11-15T14:13:13,531 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:13,531 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:13,539 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679993538,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:13,549 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:13,569 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:13,574 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-15T14:13:13,641 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1006 (auto-detected)
2024-11-15T14:13:13,643 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected)
2024-11-15T14:13:13,692 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42059
2024-11-15T14:13:13,692 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:13,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@397948c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:13,694 DEBUG [Thread-54 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:13,696 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:42059
2024-11-15T14:13:13,698 DEBUG [Thread-54 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,705 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679993704,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,707 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:13,745 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:13,745 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,748 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:13,753 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:13,754 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:13,755 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:13,755 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:13,755 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:13,756 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:13,756 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-15T14:13:13,756 DEBUG [Thread-54 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:13,756 DEBUG [NettyRpcServer-2-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:13,756 DEBUG [NettyRpcServer-2-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,757 DEBUG [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:13,757 INFO  [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:13,763 INFO  [NettyRpcServer-2-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43412, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:13,764 DEBUG [Thread-54 {}] ipc.BlockingRpcConnection(634): Length of response for connection header:93
2024-11-15T14:13:13,764 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:247
2024-11-15T14:13:13,765 DEBUG [BRPC Connection (1322466407) to localhost:42059 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-15T14:13:13,767 DEBUG [BRPC Connection (1322466407) to localhost:42059 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:233
2024-11-15T14:13:13,768 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:2243
2024-11-15T14:13:13,770 DEBUG [BRPC Connection (1322466407) to localhost:42059 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-15T14:13:13,771 DEBUG [BRPC Connection (1322466407) to localhost:42059 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:2229
2024-11-15T14:13:13,773 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:4117
2024-11-15T14:13:13,773 DEBUG [Thread-54 {}] security.HBaseSaslRpcClient$WrappedOutputStream(270): wrapping token of length:18193
2024-11-15T14:13:13,775 DEBUG [BRPC Connection (1322466407) to localhost:42059 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(232): reading next wrapped RPC packet
2024-11-15T14:13:13,783 DEBUG [BRPC Connection (1322466407) to localhost:42059 from hbase/localhost@EXAMPLE.COM {}] security.HBaseSaslRpcClient$WrappedInputStream(241): unwrapping token of length:22296
2024-11-15T14:13:13,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:13,784 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42059
2024-11-15T14:13:13,793 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21 (was 19)
Potentially hanging thread: NettyRpcServer-2-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-2-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=299 (was 292) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=300 (was 338)
2024-11-15T14:13:13,799 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=21, OpenFileDescriptor=299, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=299
2024-11-15T14:13:13,810 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:13,810 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:13,815 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679993814,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:13,817 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:13,817 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:13,818 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45797
2024-11-15T14:13:13,818 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:13,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61499d5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-15T14:13:13,819 DEBUG [Thread-55 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:13,820 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45797
2024-11-15T14:13:13,821 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,826 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679993826,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,828 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:13,828 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-15T14:13:13,828 DEBUG [Thread-55 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:45797
org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:111) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-15T14:13:13,829 WARN  [Thread-55 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731679993816
2024-11-15T14:13:13,836 DEBUG [Thread-55 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,838 DEBUG [Thread-55 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:13,838 DEBUG [Thread-55 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-15T14:13:13,838 DEBUG [Thread-55 {}] ipc.FailedServers(52): Added failed server with address localhost:45797 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:13,839 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45797
2024-11-15T14:13:13,846 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24 (was 21)
Potentially hanging thread: NettyRpcServer-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=308 (was 299) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=274 (was 299)
2024-11-15T14:13:13,851 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=24, OpenFileDescriptor=308, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=273
2024-11-15T14:13:13,862 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:13,862 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:13,866 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679993866,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:13,868 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:13,869 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:13,870 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37881
2024-11-15T14:13:13,870 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:13,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9432ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:13,871 DEBUG [Thread-56 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:13,872 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37881
2024-11-15T14:13:13,873 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,879 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679993878,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,881 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:13,881 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:13,881 DEBUG [NettyRpcServer-4-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,884 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:13,885 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:13,885 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:13,885 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:13,886 DEBUG [NettyRpcServer-4-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:13,886 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:13,886 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-15T14:13:13,887 DEBUG [Thread-56 {}] ipc.BlockingRpcConnection$1(409): Exception encountered while connecting to the server localhost:37881
javax.security.sasl.SaslException: No common protection layer between client and server
	at com.sun.security.sasl.gsskerb.GssKrb5Client.doFinalHandshake(GssKrb5Client.java:270) ~[jdk.security.jgss:?]
	at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:203) ~[jdk.security.jgss:?]
	at org.apache.hadoop.hbase.security.AbstractHBaseSaslRpcClient.evaluateChallenge(AbstractHBaseSaslRpcClient.java:105) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.security.HBaseSaslRpcClient.saslConnect(HBaseSaslRpcClient.java:126) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupSaslConnection(BlockingRpcConnection.java:367) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:541) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$2.run(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?]
	at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?]
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:538) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-15T14:13:13,887 WARN  [Thread-56 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731679993867
2024-11-15T14:13:13,963 DEBUG [Thread-56 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,965 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:13,965 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:13,965 DEBUG [NettyRpcServer-4-3 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,967 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:13,968 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:13,968 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:13,968 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:13,969 DEBUG [NettyRpcServer-4-3 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:13,969 DEBUG [Thread-56 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:13,969 DEBUG [Thread-56 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-15T14:13:13,970 DEBUG [Thread-56 {}] ipc.FailedServers(52): Added failed server with address localhost:37881 to list caused by java.io.IOException: Failed to initiate connection for hbase/localhost@EXAMPLE.COM to hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:13,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:13,970 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37881
2024-11-15T14:13:13,978 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27 (was 24)
Potentially hanging thread: NettyRpcServer-4-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-4-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-4-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=317 (was 308) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=255 (was 273)
2024-11-15T14:13:13,985 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=27, OpenFileDescriptor=317, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=254
2024-11-15T14:13:13,997 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:13,997 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:14,002 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679994001,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:14,005 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:14,006 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:14,008 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45959
2024-11-15T14:13:14,008 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:14,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a46dcff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:14,011 DEBUG [Thread-57 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:14,011 DEBUG [Thread-57 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45959
2024-11-15T14:13:14,012 DEBUG [Thread-57 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,019 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679994019,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,021 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(103): Have sent token of size 498 from initSASLContext.
2024-11-15T14:13:14,022 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=498 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,022 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,024 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 107 from saslServer.
2024-11-15T14:13:14,024 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(119): Will read input token of size 107 for processing by initSASLContext
2024-11-15T14:13:14,025 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:14,025 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,025 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:14,026 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:14,026 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:14,026 DEBUG [Thread-57 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-15T14:13:14,026 DEBUG [Thread-57 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:14,026 DEBUG [NettyRpcServer-5-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,027 DEBUG [NettyRpcServer-5-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,027 DEBUG [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:14,027 INFO  [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:14,028 INFO  [NettyRpcServer-5-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33032, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:14,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:14,040 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45959
2024-11-15T14:13:14,041 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:14,041 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:14,042 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45733
2024-11-15T14:13:14,042 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:14,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b7a751f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:14,043 DEBUG [Thread-58 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:14,043 DEBUG [Thread-58 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45733
2024-11-15T14:13:14,044 DEBUG [Thread-58 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,046 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:14,046 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,046 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,048 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:14,048 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:14,049 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:14,049 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,049 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:14,049 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:14,049 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:14,049 DEBUG [Thread-58 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-15T14:13:14,050 DEBUG [Thread-58 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:14,050 DEBUG [NettyRpcServer-6-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,050 DEBUG [NettyRpcServer-6-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,050 DEBUG [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:14,050 INFO  [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:14,051 INFO  [NettyRpcServer-6-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42220, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:14,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:14,060 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45733
2024-11-15T14:13:14,061 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:14,062 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:14,063 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43019
2024-11-15T14:13:14,063 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:14,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d3f2f39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:14,064 DEBUG [Thread-59 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:14,064 DEBUG [Thread-59 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43019
2024-11-15T14:13:14,065 DEBUG [Thread-59 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,067 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:14,067 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,067 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,069 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:14,070 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:14,071 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:14,073 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,073 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:14,074 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:14,074 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:14,074 DEBUG [Thread-59 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-15T14:13:14,074 DEBUG [Thread-59 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:14,074 DEBUG [NettyRpcServer-7-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,075 DEBUG [NettyRpcServer-7-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,075 DEBUG [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:14,075 INFO  [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:14,076 INFO  [NettyRpcServer-7-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44532, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:14,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:14,083 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43019
2024-11-15T14:13:14,084 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:14,085 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:14,086 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40025
2024-11-15T14:13:14,086 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:14,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d195958, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:14,087 DEBUG [Thread-60 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:14,088 DEBUG [Thread-60 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:40025
2024-11-15T14:13:14,089 DEBUG [Thread-60 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,091 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:14,091 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,091 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,093 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:14,094 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:14,094 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:14,094 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,095 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:14,095 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:14,095 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:14,095 DEBUG [Thread-60 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-int
2024-11-15T14:13:14,095 DEBUG [Thread-60 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:14,095 DEBUG [NettyRpcServer-8-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,096 DEBUG [NettyRpcServer-8-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,096 DEBUG [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-15T14:13:14,096 INFO  [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:14,097 INFO  [NettyRpcServer-8-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36866, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:14,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:14,108 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40025
2024-11-15T14:13:14,108 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:14,109 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:14,110 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:37339
2024-11-15T14:13:14,110 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:14,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794fd366, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:14,111 DEBUG [Thread-61 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:14,112 DEBUG [Thread-61 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:37339
2024-11-15T14:13:14,113 DEBUG [Thread-61 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,115 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:14,115 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,115 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,116 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:14,117 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:14,117 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:14,117 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,117 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:14,118 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:14,118 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:14,118 DEBUG [Thread-61 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-15T14:13:14,118 DEBUG [Thread-61 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:14,118 DEBUG [NettyRpcServer-9-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,118 DEBUG [NettyRpcServer-9-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,118 DEBUG [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:14,118 INFO  [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:14,119 INFO  [NettyRpcServer-9-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55010, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:14,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:14,130 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:37339
2024-11-15T14:13:14,136 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37 (was 27)
Potentially hanging thread: NettyRpcServer-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=362 (was 317) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=230 (was 254)
2024-11-15T14:13:14,142 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=37, OpenFileDescriptor=362, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=229
2024-11-15T14:13:14,156 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:14,156 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:14,160 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679994160,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:14,163 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:14,163 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:14,165 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43719
2024-11-15T14:13:14,165 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:14,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52df71a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:14,166 DEBUG [Thread-62 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-15T14:13:14,166 DEBUG [Thread-62 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:43719
2024-11-15T14:13:14,169 DEBUG [BRPC Connection (1322466407) to localhost:43719 from testuser {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-15T14:13:14,169 WARN  [BRPC Connection (1322466407) to localhost:43719 from testuser {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-15T14:13:14,171 DEBUG [BRPC Connection (1322466407) to localhost:43719 from testuser {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-15T14:13:14,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:14,171 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43719
2024-11-15T14:13:14,265 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39 (was 37)
Potentially hanging thread: NettyRpcServer-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=371 (was 362) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=277 (was 229) - AvailableMemoryMB LEAK? -
2024-11-15T14:13:14,273 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=39, OpenFileDescriptor=371, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=276
2024-11-15T14:13:14,287 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:14,287 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:14,291 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679994291,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:14,293 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:14,294 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:14,295 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:32947
2024-11-15T14:13:14,295 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-15T14:13:14,295 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-15T14:13:14,295 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-15T14:13:14,295 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-15T14:13:14,295 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-15T14:13:14,295 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-15T14:13:14,295 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-15T14:13:14,295 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-15T14:13:14,295 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:14,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@490b87f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:14,296 DEBUG [Thread-63 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-15T14:13:14,297 DEBUG [Thread-63 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:32947
2024-11-15T14:13:14,298 WARN  [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-15T14:13:14,299 INFO  [NettyRpcServer-11-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:36590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-15T14:13:14,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:14,307 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:32947
2024-11-15T14:13:14,315 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41 (was 39)
Potentially hanging thread: NettyRpcServer-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=380 (was 371) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=263 (was 276)
2024-11-15T14:13:14,320 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=41, OpenFileDescriptor=380, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=263
2024-11-15T14:13:14,334 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:14,334 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:14,338 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679994337,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:14,339 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:14,340 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:14,341 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34861
2024-11-15T14:13:14,341 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:14,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ca091b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:14,342 DEBUG [Thread-64 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:14,343 DEBUG [Thread-64 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:34861
2024-11-15T14:13:14,344 DEBUG [Thread-64 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,349 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679994349,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,350 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:14,351 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,351 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,354 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:14,354 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:14,355 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:14,355 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,355 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:14,356 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:14,356 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:14,356 DEBUG [Thread-64 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-15T14:13:14,356 DEBUG [NettyRpcServer-12-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,356 DEBUG [Thread-64 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:14,357 DEBUG [NettyRpcServer-12-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,357 DEBUG [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:14,357 INFO  [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:14,358 INFO  [NettyRpcServer-12-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:33036, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:14,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:14,365 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34861
2024-11-15T14:13:14,372 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43 (was 41)
Potentially hanging thread: NettyRpcServer-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=389 (was 380) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=246 (was 263)
2024-11-15T14:13:14,378 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=43, OpenFileDescriptor=389, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=245
2024-11-15T14:13:14,395 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:14,396 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:14,401 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679994400,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:14,404 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:14,404 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:14,405 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:33483
2024-11-15T14:13:14,406 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:14,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b03f1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:14,408 DEBUG [Thread-65 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:14,409 DEBUG [Thread-65 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:33483
2024-11-15T14:13:14,410 DEBUG [Thread-65 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,416 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679994416,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,418 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:14,418 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,418 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,420 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:14,421 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:14,422 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:14,422 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,422 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:14,423 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:14,423 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:14,423 DEBUG [Thread-65 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-15T14:13:14,423 DEBUG [NettyRpcServer-13-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,423 DEBUG [Thread-65 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:14,423 DEBUG [NettyRpcServer-13-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,424 DEBUG [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:14,424 INFO  [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:14,424 INFO  [NettyRpcServer-13-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48474, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:14,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:14,451 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:33483
2024-11-15T14:13:14,452 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:14,452 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:14,453 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45123
2024-11-15T14:13:14,454 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:14,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dde6bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:14,455 DEBUG [Thread-66 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:14,455 DEBUG [Thread-66 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:45123
2024-11-15T14:13:14,456 DEBUG [Thread-66 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,458 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:14,458 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,458 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,460 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:14,461 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:14,461 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:14,462 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,462 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:14,462 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:14,462 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:14,462 DEBUG [Thread-66 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth-conf
2024-11-15T14:13:14,462 DEBUG [Thread-66 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:14,462 DEBUG [NettyRpcServer-14-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:14,463 DEBUG [NettyRpcServer-14-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:14,463 DEBUG [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:14,463 INFO  [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:14,464 INFO  [NettyRpcServer-14-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55622, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:16,463 ERROR [Thread-66 {}] ipc.BlockingRpcConnection(646): Can't get the connection header response for rpc timeout, please check if server has the correct configuration to support the additional function.
java.net.SocketTimeoutException: 2000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:55622 remote=localhost/127.0.0.1:45123]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:163) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:132) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:201) ~[?:?]
	at java.io.DataInputStream.readFully(DataInputStream.java:172) ~[?:?]
	at org.apache.hadoop.security.SaslInputStream.readMoreData(SaslInputStream.java:97) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.security.SaslInputStream.read(SaslInputStream.java:260) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:263) ~[?:?]
	at java.io.DataInputStream.readInt(DataInputStream.java:381) ~[?:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.processResponseForConnectionHeader(BlockingRpcConnection.java:630) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.setupIOstreams(BlockingRpcConnection.java:570) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.writeRequest(BlockingRpcConnection.java:685) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection$4.run(BlockingRpcConnection.java:819) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.HBaseRpcControllerImpl.notifyOnCancel(HBaseRpcControllerImpl.java:277) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.BlockingRpcConnection.sendRequest(BlockingRpcConnection.java:792) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:449) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestRpcServiceProtos$TestProtobufRpcProto$BlockingStub.echo(TestRpcServiceProtos.java:501) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.security.AbstractTestSecureIPC$TestThread.run(AbstractTestSecureIPC.java:451) ~[test-classes/:?]
2024-11-15T14:13:16,464 DEBUG [Thread-66 {}] ipc.FailedServers(52): Added failed server with address localhost:45123 to list caused by java.io.IOException: Timeout while waiting connection header response
2024-11-15T14:13:16,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:16,464 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45123
2024-11-15T14:13:16,472 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47 (was 43)
Potentially hanging thread: NettyRpcServer-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=407 (was 389) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=287 (was 245) - AvailableMemoryMB LEAK? -
2024-11-15T14:13:16,479 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=47, OpenFileDescriptor=407, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=287
2024-11-15T14:13:16,490 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:16,490 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:16,494 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679996493,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:16,496 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:16,496 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:16,498 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41507
2024-11-15T14:13:16,498 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:16,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9410812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:16,499 DEBUG [Thread-67 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:16,500 DEBUG [Thread-67 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:41507
2024-11-15T14:13:16,501 DEBUG [Thread-67 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,509 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679996508,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,510 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:16,510 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,511 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,512 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:16,512 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:16,513 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:16,513 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,513 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:16,513 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:16,513 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:16,514 DEBUG [Thread-67 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-15T14:13:16,514 DEBUG [Thread-67 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:16,514 DEBUG [NettyRpcServer-15-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,514 DEBUG [NettyRpcServer-15-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,514 DEBUG [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:16,514 INFO  [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:16,515 INFO  [NettyRpcServer-15-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48342, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:16,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:16,525 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41507
2024-11-15T14:13:16,532 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49 (was 47)
Potentially hanging thread: NettyRpcServer-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=416 (was 407) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=270 (was 287)
2024-11-15T14:13:16,538 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=49, OpenFileDescriptor=416, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=269
2024-11-15T14:13:16,548 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:16,548 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:16,552 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679996551,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:16,554 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:16,554 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:16,555 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35091
2024-11-15T14:13:16,555 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:16,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42b1eeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:16,556 DEBUG [Thread-68 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:16,557 DEBUG [Thread-68 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:35091
2024-11-15T14:13:16,558 DEBUG [Thread-68 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,563 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679996562,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,564 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:16,564 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,564 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,566 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:16,566 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(119): Will read input token of size 108 for processing by initSASLContext
2024-11-15T14:13:16,566 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 0 from initSASLContext.
2024-11-15T14:13:16,567 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,567 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:16,567 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(139): Will read input token of size 32 for processing by initSASLContext
2024-11-15T14:13:16,567 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(129): Will send token of size 32 from initSASLContext.
2024-11-15T14:13:16,567 DEBUG [Thread-68 {}] security.HBaseSaslRpcClient(147): SASL client context established. Negotiated QoP: auth
2024-11-15T14:13:16,568 DEBUG [NettyRpcServer-16-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,568 DEBUG [Thread-68 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:16,568 DEBUG [NettyRpcServer-16-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,568 DEBUG [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:16,568 INFO  [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:16,568 INFO  [NettyRpcServer-16-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:49600, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:16,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:16,575 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35091
2024-11-15T14:13:16,582 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51 (was 49)
Potentially hanging thread: NettyRpcServer-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=425 (was 416) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=256 (was 269)
2024-11-15T14:13:16,590 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=51, OpenFileDescriptor=425, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=255
2024-11-15T14:13:16,602 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:16,602 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:16,606 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679996606,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:16,609 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:16,610 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:16,611 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44105
2024-11-15T14:13:16,611 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:16,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4521cb37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:16,614 DEBUG [Thread-69 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:16,614 DEBUG [Thread-69 {}] ipc.BlockingRpcConnection(510): Connecting to localhost:44105
2024-11-15T14:13:16,616 DEBUG [Thread-69 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,621 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679996621,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,623 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(103): Have sent token of size 499 from initSASLContext.
2024-11-15T14:13:16,623 DEBUG [Thread-69 {}] security.HBaseSaslRpcClient(113): Server asks us to fall back to simple auth.
2024-11-15T14:13:16,623 DEBUG [Thread-69 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:16,623 INFO  [NettyRpcServer-17-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-15T14:13:16,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:16,631 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44105
2024-11-15T14:13:16,644 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[1: rpcClientImpl=org.apache.hadoop.hbase.ipc.BlockingRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=53 (was 51)
Potentially hanging thread: NettyRpcServer-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=434 (was 425) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=238 (was 255)
2024-11-15T14:13:16,654 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=53, OpenFileDescriptor=434, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=238
2024-11-15T14:13:16,669 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:16,669 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:16,674 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679996674,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:16,677 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:16,683 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45185
2024-11-15T14:13:16,683 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:16,683 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:16,683 INFO  [Listener,port=45185 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45185: starting
2024-11-15T14:13:16,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cafdb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:16,688 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-15T14:13:16,688 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-15T14:13:16,694 DEBUG [Thread-72 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:16,716 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,724 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679996723,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,726 DEBUG [Reader=1,bindAddress=localhost,port=45185 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,727 DEBUG [Reader=1,bindAddress=localhost,port=45185 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,727 DEBUG [Reader=1,bindAddress=localhost,port=45185 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,728 DEBUG [Reader=1,bindAddress=localhost,port=45185 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:16,729 DEBUG [Reader=1,bindAddress=localhost,port=45185 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,729 DEBUG [Reader=1,bindAddress=localhost,port=45185 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,730 DEBUG [Reader=1,bindAddress=localhost,port=45185 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:16,730 DEBUG [Reader=1,bindAddress=localhost,port=45185 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,730 DEBUG [Reader=1,bindAddress=localhost,port=45185 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,731 DEBUG [Reader=1,bindAddress=localhost,port=45185 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,731 DEBUG [Reader=1,bindAddress=localhost,port=45185 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:16,731 INFO  [Reader=1,bindAddress=localhost,port=45185 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:16,737 INFO  [Reader=1,bindAddress=localhost,port=45185 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:48010, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:16,739 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:16,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:16,754 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45185
2024-11-15T14:13:16,754 INFO  [Listener,port=45185 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45185: stopping
2024-11-15T14:13:16,755 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:16,755 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:16,761 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=55 (was 53)
Potentially hanging thread: Reader=0,bindAddress=localhost,port=45185
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.signalAll(AbstractQueuedSynchronizer.java:1487)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.tryTerminate(ThreadPoolExecutor.java:728)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.processWorkerExit(ThreadPoolExecutor.java:1002)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1150)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-1
	app//org.apache.hadoop.hbase.ipc.IPCUtil$1.initialValue(IPCUtil.java:294)
	app//org.apache.hadoop.hbase.ipc.IPCUtil$1.initialValue(IPCUtil.java:290)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.initialize(FastThreadLocal.java:177)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocal.get(FastThreadLocal.java:142)
	app//org.apache.hadoop.hbase.ipc.IPCUtil.execute(IPCUtil.java:304)
	app//org.apache.hadoop.hbase.ipc.NettyRpcConnection.shutdown(NettyRpcConnection.java:146)
	app//org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelInactive(NettyRpcDuplexHandler.java:161)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274)
	app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412)
	app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274)
	app//org.apache.hbase.thirdparty.io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81)
	app//org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:280)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274)
	app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412)
	app//org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274)
	app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1352)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)
	app//org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:850)
	app//org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:811)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 434) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=246 (was 238) - AvailableMemoryMB LEAK? -
2024-11-15T14:13:16,768 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=54, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=245
2024-11-15T14:13:16,779 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:16,779 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:16,782 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679996782,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:16,784 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:16,786 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=37593
2024-11-15T14:13:16,786 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:16,786 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:16,786 INFO  [Listener,port=37593 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=37593: starting
2024-11-15T14:13:16,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2398c1c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-15T14:13:16,787 DEBUG [Thread-75 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:16,790 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,795 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679996795,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,797 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-15T14:13:16,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:16,798 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:37593 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
2024-11-15T14:13:16,798 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 37593
2024-11-15T14:13:16,798 INFO  [Listener,port=37593 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=37593: stopping
2024-11-15T14:13:16,798 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:16,798 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:16,804 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=55 (was 54)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=236 (was 245)
2024-11-15T14:13:16,810 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=55, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=247
2024-11-15T14:13:16,822 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:16,822 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:16,826 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679996826,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:16,829 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:16,831 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=39089
2024-11-15T14:13:16,831 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:16,831 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:16,831 INFO  [Listener,port=39089 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=39089: starting
2024-11-15T14:13:16,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@741464a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:16,832 DEBUG [Thread-78 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:16,835 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,842 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679996842,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,844 DEBUG [Reader=1,bindAddress=localhost,port=39089 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,845 DEBUG [Reader=1,bindAddress=localhost,port=39089 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,845 DEBUG [Reader=1,bindAddress=localhost,port=39089 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,846 DEBUG [Reader=1,bindAddress=localhost,port=39089 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:16,847 DEBUG [Reader=1,bindAddress=localhost,port=39089 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,847 DEBUG [Reader=1,bindAddress=localhost,port=39089 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,848 DEBUG [Reader=1,bindAddress=localhost,port=39089 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:16,849 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-15T14:13:16,849 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:39089 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server
2024-11-15T14:13:16,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:16,850 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 39089
2024-11-15T14:13:16,850 INFO  [Listener,port=39089 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=39089: stopping
2024-11-15T14:13:16,850 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:16,850 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:16,856 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=59 (was 55)
Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/java.lang.Thread.currentThread(Native Method)
	java.base@17.0.11/java.lang.ThreadLocal.isPresent(ThreadLocal.java:183)
	java.base@17.0.11/java.lang.Thread.exit(Thread.java:849)

Potentially hanging thread: Relogin-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer idle connection scanner for port 39089
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=264 (was 247) - AvailableMemoryMB LEAK? -
2024-11-15T14:13:16,861 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=57, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=264
2024-11-15T14:13:16,871 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:16,872 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:16,882 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679996882,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:16,884 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:16,886 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46803
2024-11-15T14:13:16,886 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:16,886 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:16,886 INFO  [Listener,port=46803 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46803: starting
2024-11-15T14:13:16,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c913c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:16,887 DEBUG [Thread-81 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:16,889 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,892 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679996892,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,894 DEBUG [Reader=1,bindAddress=localhost,port=46803 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,895 DEBUG [Reader=1,bindAddress=localhost,port=46803 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,895 DEBUG [Reader=1,bindAddress=localhost,port=46803 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,896 DEBUG [Reader=1,bindAddress=localhost,port=46803 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:16,897 DEBUG [Reader=1,bindAddress=localhost,port=46803 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,897 DEBUG [Reader=1,bindAddress=localhost,port=46803 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,897 DEBUG [Reader=1,bindAddress=localhost,port=46803 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:16,897 DEBUG [Reader=1,bindAddress=localhost,port=46803 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,898 DEBUG [Reader=1,bindAddress=localhost,port=46803 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,898 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:16,898 DEBUG [Reader=1,bindAddress=localhost,port=46803 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,898 DEBUG [Reader=1,bindAddress=localhost,port=46803 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:16,898 INFO  [Reader=1,bindAddress=localhost,port=46803 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:16,899 INFO  [Reader=1,bindAddress=localhost,port=46803 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38588, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:16,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:16,905 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46803
2024-11-15T14:13:16,905 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:16,905 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:16,906 INFO  [Listener,port=46803 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46803: stopping
2024-11-15T14:13:16,906 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:16,907 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=37211
2024-11-15T14:13:16,907 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:16,908 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:16,908 INFO  [Listener,port=37211 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=37211: starting
2024-11-15T14:13:16,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11bb4a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:16,909 DEBUG [Thread-84 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:16,911 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,913 DEBUG [Reader=1,bindAddress=localhost,port=37211 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,914 DEBUG [Reader=1,bindAddress=localhost,port=37211 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,914 DEBUG [Reader=1,bindAddress=localhost,port=37211 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,915 DEBUG [Reader=1,bindAddress=localhost,port=37211 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:16,916 DEBUG [Reader=1,bindAddress=localhost,port=37211 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,916 DEBUG [Reader=1,bindAddress=localhost,port=37211 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,916 DEBUG [Reader=1,bindAddress=localhost,port=37211 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:16,916 DEBUG [Reader=1,bindAddress=localhost,port=37211 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,916 DEBUG [Reader=1,bindAddress=localhost,port=37211 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,916 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:16,916 DEBUG [Reader=1,bindAddress=localhost,port=37211 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,917 DEBUG [Reader=1,bindAddress=localhost,port=37211 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:16,917 INFO  [Reader=1,bindAddress=localhost,port=37211 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:16,917 INFO  [Reader=1,bindAddress=localhost,port=37211 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54704, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:16,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:16,924 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 37211
2024-11-15T14:13:16,924 INFO  [Listener,port=37211 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=37211: stopping
2024-11-15T14:13:16,924 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:16,924 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:16,924 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:16,926 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=38623
2024-11-15T14:13:16,926 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:16,926 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:16,926 INFO  [Listener,port=38623 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=38623: starting
2024-11-15T14:13:16,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f4c2e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:16,930 DEBUG [Thread-87 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:16,932 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731679996883
2024-11-15T14:13:16,932 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,934 DEBUG [Reader=1,bindAddress=localhost,port=38623 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,935 DEBUG [Reader=1,bindAddress=localhost,port=38623 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,935 DEBUG [Reader=1,bindAddress=localhost,port=38623 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,936 DEBUG [Reader=1,bindAddress=localhost,port=38623 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:16,937 DEBUG [Reader=1,bindAddress=localhost,port=38623 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,937 DEBUG [Reader=1,bindAddress=localhost,port=38623 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,937 DEBUG [Reader=1,bindAddress=localhost,port=38623 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:16,938 DEBUG [Reader=1,bindAddress=localhost,port=38623 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,938 DEBUG [Reader=1,bindAddress=localhost,port=38623 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,938 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:16,938 DEBUG [Reader=1,bindAddress=localhost,port=38623 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,938 DEBUG [Reader=1,bindAddress=localhost,port=38623 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:16,938 INFO  [Reader=1,bindAddress=localhost,port=38623 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:16,939 INFO  [Reader=1,bindAddress=localhost,port=38623 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:35388, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:16,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:16,947 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 38623
2024-11-15T14:13:16,947 INFO  [Listener,port=38623 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=38623: stopping
2024-11-15T14:13:16,947 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:16,947 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:16,948 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:16,949 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=35931
2024-11-15T14:13:16,949 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:16,949 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:16,949 INFO  [Listener,port=35931 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=35931: starting
2024-11-15T14:13:16,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4899f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:16,951 DEBUG [Thread-90 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:16,953 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,954 DEBUG [Reader=1,bindAddress=localhost,port=35931 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,955 DEBUG [Reader=1,bindAddress=localhost,port=35931 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,955 DEBUG [Reader=1,bindAddress=localhost,port=35931 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,956 DEBUG [Reader=1,bindAddress=localhost,port=35931 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:16,957 DEBUG [Reader=1,bindAddress=localhost,port=35931 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,957 DEBUG [Reader=1,bindAddress=localhost,port=35931 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,957 DEBUG [Reader=1,bindAddress=localhost,port=35931 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:16,957 DEBUG [Reader=1,bindAddress=localhost,port=35931 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,957 DEBUG [Reader=1,bindAddress=localhost,port=35931 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,958 DEBUG [Reader=1,bindAddress=localhost,port=35931 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,958 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:16,958 DEBUG [Reader=1,bindAddress=localhost,port=35931 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-15T14:13:16,958 INFO  [Reader=1,bindAddress=localhost,port=35931 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:16,959 INFO  [Reader=1,bindAddress=localhost,port=35931 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:40112, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:16,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:16,966 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 35931
2024-11-15T14:13:16,967 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:16,967 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:16,967 INFO  [Listener,port=35931 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=35931: stopping
2024-11-15T14:13:16,967 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:16,968 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=36837
2024-11-15T14:13:16,970 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:16,970 INFO  [Listener,port=36837 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=36837: starting
2024-11-15T14:13:16,970 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:16,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a1e5b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:16,972 DEBUG [Thread-93 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:16,973 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,975 DEBUG [Reader=1,bindAddress=localhost,port=36837 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,976 DEBUG [Reader=1,bindAddress=localhost,port=36837 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,976 DEBUG [Reader=1,bindAddress=localhost,port=36837 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,977 DEBUG [Reader=1,bindAddress=localhost,port=36837 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:16,978 DEBUG [Reader=1,bindAddress=localhost,port=36837 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,979 DEBUG [Reader=1,bindAddress=localhost,port=36837 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,979 DEBUG [Reader=1,bindAddress=localhost,port=36837 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:16,979 DEBUG [Reader=1,bindAddress=localhost,port=36837 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:16,979 DEBUG [Reader=1,bindAddress=localhost,port=36837 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:16,979 DEBUG [Reader=1,bindAddress=localhost,port=36837 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:16,980 DEBUG [Reader=1,bindAddress=localhost,port=36837 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:16,980 INFO  [Reader=1,bindAddress=localhost,port=36837 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:16,980 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:16,981 INFO  [Reader=1,bindAddress=localhost,port=36837 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41438, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:16,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:16,990 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 36837
2024-11-15T14:13:16,991 INFO  [Listener,port=36837 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=36837: stopping
2024-11-15T14:13:16,991 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:16,991 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:16,999 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64 (was 57)
Potentially hanging thread: Reader=1,bindAddress=localhost,port=36837
	java.base@17.0.11/java.lang.Throwable.fillInStackTrace(Native Method)
	java.base@17.0.11/java.lang.Throwable.fillInStackTrace(Throwable.java:798)
	java.base@17.0.11/java.lang.Throwable.<init>(Throwable.java:256)
	java.base@17.0.11/java.lang.Exception.<init>(Exception.java:55)
	java.base@17.0.11/java.lang.InterruptedException.<init>(InterruptedException.java:57)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1640)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=36837
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=238 (was 264)
2024-11-15T14:13:17,004 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=62, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=238
2024-11-15T14:13:17,015 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:17,015 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:17,018 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679997017,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:17,020 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:17,021 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=40469
2024-11-15T14:13:17,021 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:17,021 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:17,021 INFO  [Listener,port=40469 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=40469: starting
2024-11-15T14:13:17,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4244d5c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:17,022 DEBUG [Thread-96 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-15T14:13:17,033 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-15T14:13:17,033 WARN  [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-15T14:13:17,033 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-15T14:13:17,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:17,034 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 40469
2024-11-15T14:13:17,034 INFO  [Listener,port=40469 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=40469: stopping
2024-11-15T14:13:17,036 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:17,036 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:17,041 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65 (was 62)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Reader=0,bindAddress=localhost,port=40469
	java.base@17.0.11/sun.nio.ch.EventFD.set0(Native Method)
	java.base@17.0.11/sun.nio.ch.EventFD.set(EventFD.java:48)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:251)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:183)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:151)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:115)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=237 (was 238)
2024-11-15T14:13:17,053 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=63, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=237
2024-11-15T14:13:17,065 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:17,065 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:17,069 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679997068,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:17,071 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:17,073 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=46821
2024-11-15T14:13:17,073 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-15T14:13:17,073 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-15T14:13:17,073 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-15T14:13:17,073 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-15T14:13:17,073 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-15T14:13:17,073 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-15T14:13:17,074 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-15T14:13:17,074 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-15T14:13:17,074 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:17,074 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:17,074 INFO  [Listener,port=46821 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=46821: starting
2024-11-15T14:13:17,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c57707c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:17,079 DEBUG [Thread-99 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-15T14:13:17,085 WARN  [Reader=1,bindAddress=localhost,port=46821 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-15T14:13:17,085 INFO  [Reader=1,bindAddress=localhost,port=46821 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-15T14:13:17,093 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:17,093 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 46821
2024-11-15T14:13:17,093 INFO  [Listener,port=46821 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=46821: stopping
2024-11-15T14:13:17,093 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:17,093 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:17,102 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=66 (was 63)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-10
	java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.close0(Native Method)
	java.base@17.0.11/sun.nio.ch.SocketDispatcher.close(SocketDispatcher.java:70)
	java.base@17.0.11/sun.nio.ch.SocketChannelImpl.tryClose(SocketChannelImpl.java:980)
	java.base@17.0.11/sun.nio.ch.SocketChannelImpl.tryFinishClose(SocketChannelImpl.java:994)
	java.base@17.0.11/sun.nio.ch.SocketChannelImpl.kill(SocketChannelImpl.java:1088)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:277)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:112)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer idle connection scanner for port 46821
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: FifoRpcScheduler.handler-pool-0
	java.base@17.0.11/java.lang.ThreadLocal$ThreadLocalMap.getEntryAfterMiss(ThreadLocal.java:462)
	java.base@17.0.11/java.lang.ThreadLocal$ThreadLocalMap.getEntry(ThreadLocal.java:439)
	java.base@17.0.11/java.lang.ThreadLocal.get(ThreadLocal.java:165)
	java.base@17.0.11/jdk.internal.misc.TerminatingThreadLocal._threadTerminated(TerminatingThreadLocal.java:74)
	java.base@17.0.11/jdk.internal.misc.TerminatingThreadLocal.threadTerminated(TerminatingThreadLocal.java:70)
	java.base@17.0.11/java.lang.Thread.exit(Thread.java:850)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=235 (was 237)
2024-11-15T14:13:17,109 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=64, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=234
2024-11-15T14:13:17,122 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:17,123 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:17,125 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679997125,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:17,127 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:17,128 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44295
2024-11-15T14:13:17,128 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:17,128 INFO  [Listener,port=44295 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44295: starting
2024-11-15T14:13:17,128 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:17,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c1656d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:17,129 DEBUG [Thread-102 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:17,134 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:17,138 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679997137,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:17,139 DEBUG [Reader=1,bindAddress=localhost,port=44295 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:17,140 DEBUG [Reader=1,bindAddress=localhost,port=44295 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:17,140 DEBUG [Reader=1,bindAddress=localhost,port=44295 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:17,141 DEBUG [Reader=1,bindAddress=localhost,port=44295 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:17,142 DEBUG [Reader=1,bindAddress=localhost,port=44295 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:17,142 DEBUG [Reader=1,bindAddress=localhost,port=44295 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:17,142 DEBUG [Reader=1,bindAddress=localhost,port=44295 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:17,142 DEBUG [Reader=1,bindAddress=localhost,port=44295 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:17,142 DEBUG [Reader=1,bindAddress=localhost,port=44295 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:17,142 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:17,143 DEBUG [Reader=1,bindAddress=localhost,port=44295 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:17,143 DEBUG [Reader=1,bindAddress=localhost,port=44295 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:17,143 INFO  [Reader=1,bindAddress=localhost,port=44295 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:17,143 INFO  [Reader=1,bindAddress=localhost,port=44295 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:41688, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:17,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:17,151 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44295
2024-11-15T14:13:17,151 INFO  [Listener,port=44295 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44295: stopping
2024-11-15T14:13:17,151 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:17,151 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:17,157 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65 (was 64)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=233 (was 234)
2024-11-15T14:13:17,163 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=65, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=232
2024-11-15T14:13:17,173 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:17,173 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:17,176 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679997175,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:17,178 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:17,179 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=43189
2024-11-15T14:13:17,179 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:17,179 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:17,179 INFO  [Listener,port=43189 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=43189: starting
2024-11-15T14:13:17,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b57c1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:17,182 DEBUG [Thread-105 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:17,184 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:17,188 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679997188,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:17,190 DEBUG [Reader=1,bindAddress=localhost,port=43189 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:17,191 DEBUG [Reader=1,bindAddress=localhost,port=43189 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:17,191 DEBUG [Reader=1,bindAddress=localhost,port=43189 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:17,192 DEBUG [Reader=1,bindAddress=localhost,port=43189 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:17,193 DEBUG [Reader=1,bindAddress=localhost,port=43189 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:17,193 DEBUG [Reader=1,bindAddress=localhost,port=43189 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:17,193 DEBUG [Reader=1,bindAddress=localhost,port=43189 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:17,193 DEBUG [Reader=1,bindAddress=localhost,port=43189 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:17,193 DEBUG [Reader=1,bindAddress=localhost,port=43189 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:17,193 DEBUG [Reader=1,bindAddress=localhost,port=43189 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:17,194 DEBUG [Reader=1,bindAddress=localhost,port=43189 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:17,194 INFO  [Reader=1,bindAddress=localhost,port=43189 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:17,194 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:17,195 INFO  [Reader=1,bindAddress=localhost,port=43189 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60224, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:17,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:17,204 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 43189
2024-11-15T14:13:17,205 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:17,205 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:17,205 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:17,206 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=45861
2024-11-15T14:13:17,207 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:17,210 INFO  [Listener,port=43189 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=43189: stopping
2024-11-15T14:13:17,213 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:17,213 INFO  [Listener,port=45861 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=45861: starting
2024-11-15T14:13:17,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@236006c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:17,214 DEBUG [Thread-108 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:17,216 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:17,217 DEBUG [Reader=1,bindAddress=localhost,port=45861 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:17,218 DEBUG [Reader=1,bindAddress=localhost,port=45861 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:17,218 DEBUG [Reader=1,bindAddress=localhost,port=45861 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:17,219 DEBUG [Reader=1,bindAddress=localhost,port=45861 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:17,220 DEBUG [Reader=1,bindAddress=localhost,port=45861 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:17,220 DEBUG [Reader=1,bindAddress=localhost,port=45861 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:17,220 DEBUG [Reader=1,bindAddress=localhost,port=45861 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:17,220 DEBUG [Reader=1,bindAddress=localhost,port=45861 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:17,220 DEBUG [Reader=1,bindAddress=localhost,port=45861 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:17,220 DEBUG [Reader=1,bindAddress=localhost,port=45861 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:17,221 DEBUG [Reader=1,bindAddress=localhost,port=45861 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:17,221 INFO  [Reader=1,bindAddress=localhost,port=45861 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:17,221 INFO  [Reader=1,bindAddress=localhost,port=45861 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:38652, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:19,224 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-15T14:13:19,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,225 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 45861
2024-11-15T14:13:19,225 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:45861 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException
2024-11-15T14:13:19,225 INFO  [Listener,port=45861 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=45861: stopping
2024-11-15T14:13:19,225 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:19,225 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:19,244 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69 (was 65)
Potentially hanging thread: Reader=1,bindAddress=localhost,port=45861
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.doRunLoop(SimpleRpcServer.java:168)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener$Reader.run(SimpleRpcServer.java:148)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer idle connection scanner for port 45861
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=253 (was 232) - AvailableMemoryMB LEAK? -
2024-11-15T14:13:19,255 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=67, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=254
2024-11-15T14:13:19,265 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:19,265 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:19,267 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679999267,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:19,269 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,271 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=34089
2024-11-15T14:13:19,271 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:19,271 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,271 INFO  [Listener,port=34089 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=34089: starting
2024-11-15T14:13:19,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a779ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,272 DEBUG [Thread-111 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:19,274 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,277 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679999276,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,278 DEBUG [Reader=1,bindAddress=localhost,port=34089 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,279 DEBUG [Reader=1,bindAddress=localhost,port=34089 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:19,279 DEBUG [Reader=1,bindAddress=localhost,port=34089 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,280 DEBUG [Reader=1,bindAddress=localhost,port=34089 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:19,280 DEBUG [Reader=1,bindAddress=localhost,port=34089 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:19,280 DEBUG [Reader=1,bindAddress=localhost,port=34089 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,280 DEBUG [Reader=1,bindAddress=localhost,port=34089 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:19,281 DEBUG [Reader=1,bindAddress=localhost,port=34089 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:19,281 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:19,281 DEBUG [Reader=1,bindAddress=localhost,port=34089 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,281 DEBUG [Reader=1,bindAddress=localhost,port=34089 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,281 DEBUG [Reader=1,bindAddress=localhost,port=34089 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:19,281 INFO  [Reader=1,bindAddress=localhost,port=34089 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:19,281 INFO  [Reader=1,bindAddress=localhost,port=34089 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50952, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:19,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,287 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 34089
2024-11-15T14:13:19,287 INFO  [Listener,port=34089 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=34089: stopping
2024-11-15T14:13:19,288 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:19,288 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:19,293 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69 (was 67)
Potentially hanging thread: RPCClient-NioEventLoopGroup-18-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/java.util.stream.Sink$ChainedReference.cancellationRequested(Sink.java:263)
	java.base@17.0.11/java.util.stream.Sink$ChainedReference.cancellationRequested(Sink.java:263)
	java.base@17.0.11/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129)
	java.base@17.0.11/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527)
	java.base@17.0.11/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513)
	java.base@17.0.11/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)
	java.base@17.0.11/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150)
	java.base@17.0.11/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
	java.base@17.0.11/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647)
	app//org.apache.logging.log4j.util.StackLocator.lambda$calcLocation$15(StackLocator.java:100)
	app//org.apache.logging.log4j.util.StackLocator$$Lambda$125/0x00007fa80418c260.apply(Unknown Source)
	java.base@17.0.11/java.lang.StackStreamFactory$StackFrameTraverser.consumeFrames(StackStreamFactory.java:534)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.doStackWalk(StackStreamFactory.java:306)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.callStackWalk(Native Method)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.beginStackWalk(StackStreamFactory.java:370)
	java.base@17.0.11/java.lang.StackStreamFactory$AbstractStackWalker.walk(StackStreamFactory.java:243)
	java.base@17.0.11/java.lang.StackWalker.walk(StackWalker.java:499)
	app//org.apache.logging.log4j.util.StackLocator.calcLocation(StackLocator.java:97)
	app//org.apache.logging.log4j.util.StackLocatorUtil.calcLocation(StackLocatorUtil.java:121)
	app//org.apache.logging.log4j.spi.AbstractLogger.getLocation(AbstractLogger.java:2216)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.doRunLoop(SimpleRpcServerResponder.java:161)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:63)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=255 (was 254) - AvailableMemoryMB LEAK? -
2024-11-15T14:13:19,299 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=68, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=260
2024-11-15T14:13:19,300 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731679999268
2024-11-15T14:13:19,309 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:19,309 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:19,311 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679999311,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:19,313 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,314 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=41669
2024-11-15T14:13:19,316 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:19,316 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,316 INFO  [Listener,port=41669 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=41669: starting
2024-11-15T14:13:19,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@313b636b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,318 DEBUG [Thread-114 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:19,319 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,322 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679999322,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,323 DEBUG [Reader=1,bindAddress=localhost,port=41669 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,324 DEBUG [Reader=1,bindAddress=localhost,port=41669 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:19,324 DEBUG [Reader=1,bindAddress=localhost,port=41669 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,325 DEBUG [Reader=1,bindAddress=localhost,port=41669 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 108 from saslServer.
2024-11-15T14:13:19,326 DEBUG [Reader=1,bindAddress=localhost,port=41669 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:19,326 DEBUG [Reader=1,bindAddress=localhost,port=41669 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,326 DEBUG [Reader=1,bindAddress=localhost,port=41669 {}] ipc.SimpleServerRpcConnection(250): Will send token of size 32 from saslServer.
2024-11-15T14:13:19,327 DEBUG [Reader=1,bindAddress=localhost,port=41669 {}] ipc.SimpleServerRpcConnection(226): Created SASL server with mechanism=KERBEROS
2024-11-15T14:13:19,327 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:19,327 DEBUG [Reader=1,bindAddress=localhost,port=41669 {}] ipc.SimpleServerRpcConnection(228): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,327 DEBUG [Reader=1,bindAddress=localhost,port=41669 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,327 DEBUG [Reader=1,bindAddress=localhost,port=41669 {}] ipc.SimpleServerRpcConnection(259): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:19,327 INFO  [Reader=1,bindAddress=localhost,port=41669 {}] ipc.SimpleServerRpcConnection(263): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:19,327 INFO  [Reader=1,bindAddress=localhost,port=41669 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50904, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:19,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,334 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 41669
2024-11-15T14:13:19,334 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:19,334 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:19,335 INFO  [Listener,port=41669 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=41669: stopping
2024-11-15T14:13:19,340 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=71 (was 68)
Potentially hanging thread: RpcServer idle connection scanner for port 41669
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Listener,port=41669
	app//org.apache.logging.log4j.spi.DefaultThreadContextMap.getImmutableMapOrNull(DefaultThreadContextMap.java:194)
	app//org.apache.logging.log4j.ThreadContext.getImmutableContext(ThreadContext.java:369)
	app//org.apache.logging.log4j.core.impl.ThreadContextDataProvider.supplyContextData(ThreadContextDataProvider.java:32)
	app//org.apache.logging.log4j.core.impl.ThreadContextDataInjector$ForDefaultThreadContextMap.injectContextData(ThreadContextDataInjector.java:119)
	app//org.apache.logging.log4j.core.impl.Log4jLogEvent.createContextData(Log4jLogEvent.java:497)
	app//org.apache.logging.log4j.core.impl.Log4jLogEvent.<init>(Log4jLogEvent.java:355)
	app//org.apache.logging.log4j.core.impl.DefaultLogEventFactory.createEvent(DefaultLogEventFactory.java:74)
	app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:550)
	app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
	app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
	app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
	app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2022)
	app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1875)
	app//org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:252)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-15
	java.base@17.0.11/sun.nio.ch.FileDispatcherImpl.close0(Native Method)
	java.base@17.0.11/sun.nio.ch.SocketDispatcher.close(SocketDispatcher.java:70)
	java.base@17.0.11/sun.nio.ch.SocketChannelImpl.tryClose(SocketChannelImpl.java:980)
	java.base@17.0.11/sun.nio.ch.SocketChannelImpl.tryFinishClose(SocketChannelImpl.java:994)
	java.base@17.0.11/sun.nio.ch.SocketChannelImpl.kill(SocketChannelImpl.java:1088)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:277)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:112)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=261 (was 260) - AvailableMemoryMB LEAK? -
2024-11-15T14:13:19,346 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=69, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=261
2024-11-15T14:13:19,354 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:19,355 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:19,356 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679999356,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:19,359 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.SimpleRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,360 INFO  [Time-limited test {}] ipc.SimpleRpcServer$Listener(128): AbstractTestSecureIPC: started 3 reader(s) listening on port=44831
2024-11-15T14:13:19,360 DEBUG [RpcServer.responder {}] ipc.SimpleRpcServerResponder(61): RpcServer.responder: starting
2024-11-15T14:13:19,361 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,361 INFO  [Listener,port=44831 {}] ipc.SimpleRpcServer$Listener(208): Listener,port=44831: starting
2024-11-15T14:13:19,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7181b682, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,362 DEBUG [Thread-117 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:19,364 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,366 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679999366,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,368 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:19,368 INFO  [Reader=1,bindAddress=localhost,port=44831 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:53988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-15T14:13:19,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,374 INFO  [Time-limited test {}] ipc.SimpleRpcServer(432): Stopping server on 44831
2024-11-15T14:13:19,374 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(161): RpcServer.responder: stopped
2024-11-15T14:13:19,374 INFO  [Listener,port=44831 {}] ipc.SimpleRpcServer$Listener(252): Listener,port=44831: stopping
2024-11-15T14:13:19,374 INFO  [RpcServer.responder {}] ipc.SimpleRpcServerResponder(65): RpcServer.responder: stopping
2024-11-15T14:13:19,380 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[2: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.SimpleRpcServer] Thread=73 (was 69)
Potentially hanging thread: RpcServer.responder
	java.base@17.0.11/sun.nio.ch.EventFD.set0(Native Method)
	java.base@17.0.11/sun.nio.ch.EventFD.set(EventFD.java:48)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:251)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.implCloseSelector(SelectorImpl.java:183)
	java.base@17.0.11/java.nio.channels.spi.AbstractSelector.close(AbstractSelector.java:127)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServerResponder.run(SimpleRpcServerResponder.java:67)

Potentially hanging thread: RPCClient-NioEventLoopGroup-18-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Listener,port=44831
	java.base@17.0.11/java.util.concurrent.ConcurrentHashMap$Traverser.advance(ConcurrentHashMap.java:3383)
	java.base@17.0.11/java.util.concurrent.ConcurrentHashMap$BaseIterator.<init>(ConcurrentHashMap.java:3435)
	java.base@17.0.11/java.util.concurrent.ConcurrentHashMap$KeyIterator.<init>(ConcurrentHashMap.java:3454)
	java.base@17.0.11/java.util.concurrent.ConcurrentHashMap$KeySetView.iterator(ConcurrentHashMap.java:4635)
	java.base@17.0.11/java.util.concurrent.ConcurrentHashMap$CollectionView.toArray(ConcurrentHashMap.java:4489)
	java.base@17.0.11/java.util.Collections$SetFromMap.toArray(Collections.java:5688)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$ConnectionManager.toArray(SimpleRpcServer.java:566)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$ConnectionManager.closeAll(SimpleRpcServer.java:623)
	app//org.apache.hadoop.hbase.ipc.SimpleRpcServer$Listener.run(SimpleRpcServer.java:266)

Potentially hanging thread: RpcServer idle connection scanner for port 44831
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
 - Thread LEAK? -, OpenFileDescriptor=466 (was 466), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=260 (was 261)
2024-11-15T14:13:19,387 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=70, OpenFileDescriptor=466, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=259
2024-11-15T14:13:19,397 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:19,397 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:19,399 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679999398,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:19,400 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,400 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:19,401 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43169
2024-11-15T14:13:19,401 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eb420dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,402 DEBUG [Thread-118 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:19,403 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,406 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679999406,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,407 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,407 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,409 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:19,410 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,410 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:19,410 DEBUG [NettyRpcServer-19-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,411 DEBUG [NettyRpcServer-19-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,411 DEBUG [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:19,411 INFO  [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:19,412 INFO  [NettyRpcServer-19-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44158, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:19,413 DEBUG [RPCClient-NioEventLoopGroup-18-1 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:19,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,421 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43169
2024-11-15T14:13:19,426 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72 (was 70)
Potentially hanging thread: NettyRpcServer-19-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-19-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=475 (was 466) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=259 (was 259)
2024-11-15T14:13:19,432 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=72, OpenFileDescriptor=475, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=259
2024-11-15T14:13:19,442 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:19,442 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:19,444 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679999444,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:19,446 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,446 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:19,447 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35895
2024-11-15T14:13:19,448 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1221de48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null
2024-11-15T14:13:19,449 DEBUG [Thread-119 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:19,449 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,452 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679999452,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,453 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-15T14:13:19,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,454 DEBUG [RPCClient-NioEventLoopGroup-18-2 {}] ipc.FailedServers(52): Added failed server with address localhost:35895 to list caused by org.apache.hadoop.hbase.ipc.FallbackDisallowedException: Server asks us to fall back to SIMPLE auth, but this client is configured to only allow secure connections.
2024-11-15T14:13:19,454 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35895
2024-11-15T14:13:19,461 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74 (was 72)
Potentially hanging thread: NettyRpcServer-20-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-20-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=484 (was 475) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=256 (was 259)
2024-11-15T14:13:19,467 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=74, OpenFileDescriptor=484, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=256
2024-11-15T14:13:19,476 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:19,477 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:19,478 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679999478,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:19,481 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,481 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:19,482 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45979
2024-11-15T14:13:19,482 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3db34e7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,483 DEBUG [Thread-120 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:19,485 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,488 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679999487,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,489 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,489 DEBUG [NettyRpcServer-21-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,491 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:19,491 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,492 DEBUG [NettyRpcServer-21-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:19,492 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-15T14:13:19,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,492 DEBUG [RPCClient-NioEventLoopGroup-18-3 {}] ipc.FailedServers(52): Added failed server with address localhost:45979 to list caused by javax.security.sasl.SaslException: No common protection layer between client and server
2024-11-15T14:13:19,492 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45979
2024-11-15T14:13:19,499 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslNoCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76 (was 74)
Potentially hanging thread: NettyRpcServer-21-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-21-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=493 (was 484) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=253 (was 256)
2024-11-15T14:13:19,505 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=76, OpenFileDescriptor=493, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=252
2024-11-15T14:13:19,515 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:19,515 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:19,518 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679999517,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:19,519 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,520 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:19,521 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42525
2024-11-15T14:13:19,521 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73ebb301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,522 DEBUG [Thread-121 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:19,523 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,528 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679999527,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,530 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,530 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,532 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:19,533 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,533 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:19,534 DEBUG [NettyRpcServer-22-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,534 DEBUG [RPCClient-NioEventLoopGroup-18-4 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:19,534 DEBUG [NettyRpcServer-22-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,534 DEBUG [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:19,534 INFO  [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:19,535 INFO  [NettyRpcServer-22-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:54688, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:19,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,546 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42525
2024-11-15T14:13:19,547 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,547 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:19,548 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:45679
2024-11-15T14:13:19,548 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447b4654, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,582 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731679999518
2024-11-15T14:13:19,603 DEBUG [Thread-122 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:19,607 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,608 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,608 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,610 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:19,610 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,611 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:19,611 DEBUG [NettyRpcServer-23-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,611 DEBUG [RPCClient-NioEventLoopGroup-18-5 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:19,611 DEBUG [NettyRpcServer-23-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,611 DEBUG [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:19,611 INFO  [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:19,612 INFO  [NettyRpcServer-23-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:52934, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:19,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,619 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:45679
2024-11-15T14:13:19,620 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,620 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:19,621 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44841
2024-11-15T14:13:19,621 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e92d32b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,622 DEBUG [Thread-123 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:19,623 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,625 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,625 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,627 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:19,628 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,628 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:19,628 DEBUG [NettyRpcServer-24-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,628 DEBUG [RPCClient-NioEventLoopGroup-18-6 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:19,628 DEBUG [NettyRpcServer-24-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,628 DEBUG [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:19,629 INFO  [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:19,629 INFO  [NettyRpcServer-24-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:37778, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:19,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,636 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44841
2024-11-15T14:13:19,637 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,637 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:19,638 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44631
2024-11-15T14:13:19,638 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f76d5d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,639 DEBUG [Thread-124 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:19,641 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,642 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,642 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,644 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:19,644 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,645 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:19,645 DEBUG [NettyRpcServer-25-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,645 DEBUG [NettyRpcServer-25-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,645 DEBUG [RPCClient-NioEventLoopGroup-18-7 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:19,645 DEBUG [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-int
2024-11-15T14:13:19,645 INFO  [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:19,646 INFO  [NettyRpcServer-25-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:59272, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:19,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,654 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44631
2024-11-15T14:13:19,655 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,655 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:19,656 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42521
2024-11-15T14:13:19,656 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38c34ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,657 DEBUG [Thread-125 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:19,658 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,659 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,659 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,661 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:19,661 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,661 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:19,662 DEBUG [NettyRpcServer-26-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,662 DEBUG [NettyRpcServer-26-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,662 DEBUG [RPCClient-NioEventLoopGroup-18-8 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:19,662 DEBUG [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:19,662 INFO  [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:19,663 INFO  [NettyRpcServer-26-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50750, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:19,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,720 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42521
2024-11-15T14:13:19,726 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testSaslWithCommonQop[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86 (was 76)
Potentially hanging thread: NettyRpcServer-22-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-22-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-26-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-26-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-24-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-24-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-25-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-25-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-23-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-23-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=538 (was 493) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=244 (was 252)
2024-11-15T14:13:19,733 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=86, OpenFileDescriptor=538, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=243
2024-11-15T14:13:19,743 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:19,743 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:19,746 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679999746,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:19,748 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,748 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:19,749 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:44283
2024-11-15T14:13:19,749 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5625db0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,751 DEBUG [Thread-126 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-15T14:13:19,753 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(424): process preamble call response with response type null
2024-11-15T14:13:19,753 WARN  [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(430): Got a negative call id -1 but there is no preamble call
2024-11-15T14:13:19,753 DEBUG [RPCClient-NioEventLoopGroup-18-9 {}] ipc.RpcConnection(450): Unknown callId: -1, skipping over this response of 0 bytes
2024-11-15T14:13:19,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,753 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:44283
2024-11-15T14:13:19,760 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerDisallowFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88 (was 86)
Potentially hanging thread: NettyRpcServer-27-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-27-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=547 (was 538) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=241 (was 243)
2024-11-15T14:13:19,768 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=88, OpenFileDescriptor=547, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=241
2024-11-15T14:13:19,780 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:19,780 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:19,782 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679999782,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:19,785 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,785 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:19,790 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:40841
2024-11-15T14:13:19,790 WARN  [Time-limited test {}] ipc.RpcServer(376): ********* WARNING! *********
2024-11-15T14:13:19,790 WARN  [Time-limited test {}] ipc.RpcServer(377): This server is configured to allow connections from INSECURE clients
2024-11-15T14:13:19,790 WARN  [Time-limited test {}] ipc.RpcServer(378): (hbase.ipc.server.fallback-to-simple-auth-allowed = true).
2024-11-15T14:13:19,790 WARN  [Time-limited test {}] ipc.RpcServer(379): While this option is enabled, client identities cannot be secured, and user
2024-11-15T14:13:19,791 WARN  [Time-limited test {}] ipc.RpcServer(380): impersonation is possible!
2024-11-15T14:13:19,791 WARN  [Time-limited test {}] ipc.RpcServer(381): For secure operation, please disable SIMPLE authentication as soon as possible,
2024-11-15T14:13:19,791 WARN  [Time-limited test {}] ipc.RpcServer(382): by setting hbase.ipc.server.fallback-to-simple-auth-allowed = false in hbase-site.xml
2024-11-15T14:13:19,791 WARN  [Time-limited test {}] ipc.RpcServer(383): ****************************
2024-11-15T14:13:19,791 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,791 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3944cbd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,793 DEBUG [Thread-127 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=TestProtobufRpcProto, sasl=false
2024-11-15T14:13:19,796 WARN  [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(451): Allowed fallback to SIMPLE auth for testuser (auth:SIMPLE) connecting from 127.0.0.1
2024-11-15T14:13:19,796 INFO  [NettyRpcServer-28-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:60204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=testuser (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-15T14:13:19,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,805 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:40841
2024-11-15T14:13:19,813 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcServerFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90 (was 88)
Potentially hanging thread: NettyRpcServer-28-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-28-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=556 (was 547) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=238 (was 241)
2024-11-15T14:13:19,819 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=90, OpenFileDescriptor=556, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=236
2024-11-15T14:13:19,829 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:19,829 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:19,832 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679999831,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:19,834 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,834 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:19,835 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:35841
2024-11-15T14:13:19,835 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@670996db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,836 DEBUG [Thread-128 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:19,838 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,841 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679999841,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,844 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,844 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,846 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:19,847 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,847 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:19,848 DEBUG [RPCClient-NioEventLoopGroup-18-11 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:19,848 DEBUG [NettyRpcServer-29-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,848 DEBUG [NettyRpcServer-29-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,848 DEBUG [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:19,848 INFO  [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:19,848 INFO  [NettyRpcServer-29-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:50624, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:19,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,854 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:35841
2024-11-15T14:13:19,860 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92 (was 90)
Potentially hanging thread: NettyRpcServer-29-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-29-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=565 (was 556) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=230 (was 236)
2024-11-15T14:13:19,867 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=92, OpenFileDescriptor=565, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=230
2024-11-15T14:13:19,878 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:19,878 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:19,882 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731679999882,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:19,884 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,884 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:19,885 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:41161
2024-11-15T14:13:19,885 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5160a93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,886 DEBUG [Thread-129 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:19,887 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,890 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731679999890,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,891 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,891 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,892 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:19,893 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,893 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:19,894 DEBUG [NettyRpcServer-30-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,894 DEBUG [NettyRpcServer-30-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,894 DEBUG [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:19,894 INFO  [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:19,894 DEBUG [RPCClient-NioEventLoopGroup-18-12 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:19,895 INFO  [NettyRpcServer-30-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:55012, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:19,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:19,907 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:41161
2024-11-15T14:13:19,908 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:19,909 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:19,910 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:43641
2024-11-15T14:13:19,910 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:19,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@406e6f76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:19,919 DEBUG [Thread-130 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:19,920 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,923 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,923 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,925 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:19,926 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,926 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:19,926 DEBUG [NettyRpcServer-31-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:19,927 DEBUG [NettyRpcServer-31-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:19,927 DEBUG [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth-conf
2024-11-15T14:13:19,927 INFO  [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:19,927 INFO  [NettyRpcServer-31-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:43780, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:21,929 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = false
2024-11-15T14:13:21,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:21,930 DEBUG [RPCClient-NioEventLoopGroup-18-13 {}] ipc.FailedServers(52): Added failed server with address localhost:43641 to list caused by java.io.IOException: org.apache.hbase.thirdparty.io.netty.handler.timeout.ReadTimeoutException
2024-11-15T14:13:21,931 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:43641
2024-11-15T14:13:21,942 WARN  [Relogin-pool-0 {}] security.UserGroupInformation(1404): Not attempting to re-login since the last re-login was attempted less than 60 seconds before. Last Login=1731679999883
2024-11-15T14:13:21,942 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testDifferentConfWithCryptoAES[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96 (was 92)
Potentially hanging thread: NettyRpcServer-31-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-30-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-30-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-31-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=583 (was 565) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=643 (was 630) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=333 (was 230) - AvailableMemoryMB LEAK? -
2024-11-15T14:13:21,950 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=96, OpenFileDescriptor=583, MaxFileDescriptor=1048576, SystemLoadAverage=643, ProcessCount=11, AvailableMemoryMB=333
2024-11-15T14:13:21,963 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:21,963 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:21,965 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731680001965,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:21,967 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:21,967 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:21,968 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:42227
2024-11-15T14:13:21,968 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:21,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1899ef98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:21,969 DEBUG [Thread-131 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:21,970 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:21,972 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731680001972,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:21,974 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=499 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:21,974 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:21,975 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 108 from saslServer.
2024-11-15T14:13:21,976 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:21,976 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:21,976 DEBUG [NettyRpcServer-32-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:21,976 DEBUG [RPCClient-NioEventLoopGroup-18-14 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:21,977 DEBUG [NettyRpcServer-32-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:21,977 DEBUG [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:21,977 INFO  [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:21,977 INFO  [NettyRpcServer-32-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:42934, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:21,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:21,983 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:42227
2024-11-15T14:13:21,989 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthNoCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98 (was 96)
Potentially hanging thread: NettyRpcServer-32-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-32-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=592 (was 583) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=643 (was 643), ProcessCount=11 (was 11), AvailableMemoryMB=332 (was 333)
2024-11-15T14:13:21,995 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=98, OpenFileDescriptor=592, MaxFileDescriptor=1048576, SystemLoadAverage=643, ProcessCount=11, AvailableMemoryMB=331
2024-11-15T14:13:22,004 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:22,004 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:22,006 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731680002006,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:22,008 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:22,008 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:22,009 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:34545
2024-11-15T14:13:22,009 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:22,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604fd61b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:22,010 DEBUG [Thread-132 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:22,011 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:22,013 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731680002013,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:22,014 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=498 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:22,014 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider(50): Server's Kerberos principal name is hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:22,016 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 107 from saslServer.
2024-11-15T14:13:22,016 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=0 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:22,016 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(62): Will send token of size 32 from saslServer.
2024-11-15T14:13:22,017 DEBUG [NettyRpcServer-33-2 {}] ipc.NettyHBaseSaslRpcServerHandler(55): Read input token of size=32 for processing by saslServer.evaluateResponse()
2024-11-15T14:13:22,017 DEBUG [RPCClient-NioEventLoopGroup-18-15 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:22,017 DEBUG [NettyRpcServer-33-2 {}] provider.GssSaslServerAuthenticationProvider$SaslGssCallbackHandler(95): SASL server GSSAPI callback: setting canonicalized client ID: hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:22,017 DEBUG [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(356): SASL server context established. Authenticated client: hbase/localhost@EXAMPLE.COM (auth:KERBEROS). Negotiated QoP is auth
2024-11-15T14:13:22,017 INFO  [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(359): Auth successful for hbase/localhost@EXAMPLE.COM (auth:KERBEROS)
2024-11-15T14:13:22,017 INFO  [NettyRpcServer-33-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:51042, version=3.0.0-beta-2-SNAPSHOT, sasl=true, ugi=hbase/localhost@EXAMPLE.COM (auth:KERBEROS), service=TestProtobufRpcProto
2024-11-15T14:13:22,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:22,023 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:34545
2024-11-15T14:13:22,029 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcCallWithEnabledKerberosSaslAuthCanonicalHostname[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100 (was 98)
Potentially hanging thread: NettyRpcServer-33-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-33-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=601 (was 592) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=643 (was 643), ProcessCount=11 (was 11), AvailableMemoryMB=330 (was 331)
2024-11-15T14:13:22,035 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=100, OpenFileDescriptor=601, MaxFileDescriptor=1048576, SystemLoadAverage=643, ProcessCount=11, AvailableMemoryMB=330
2024-11-15T14:13:22,043 INFO  [pool-3-thread-1 {}] request.KdcRequest(651): The preauth data is empty.
2024-11-15T14:13:22,043 INFO  [pool-3-thread-1 {}] server.KdcHandler(177): KRB error occurred while processing request:Additional pre-authentication required
2024-11-15T14:13:22,045 INFO  [pool-3-thread-1 {}] request.AsRequest(112): AS_REQ ISSUE: authtime 1731680002044,hbase/localhost@EXAMPLE.COM for krbtgt/EXAMPLE.COM@EXAMPLE.COM
2024-11-15T14:13:22,046 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.test.pb.TestProtobufRpcProto
2024-11-15T14:13:22,046 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-15T14:13:22,047 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /127.0.0.1:46567
2024-11-15T14:13:22,047 INFO  [Time-limited test {}] ipc.FifoRpcScheduler(60): Using FifoRpcScheduler as user call queue; handlerCount=1; maxQueueLength=10
2024-11-15T14:13:22,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c097538, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=2000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-15T14:13:22,048 DEBUG [Thread-133 {}] ipc.RpcConnection(159): Using KERBEROS authentication for service=TestProtobufRpcProto, sasl=true
2024-11-15T14:13:22,049 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] provider.GssSaslClientAuthenticationProvider(47): Setting up Kerberos RPC to server=hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:22,052 INFO  [pool-3-thread-1 {}] request.TgsRequest(115): TGS_REQ ISSUE: authtime 1731680002051,hbase/localhost for hbase/localhost@EXAMPLE.COM
2024-11-15T14:13:22,053 DEBUG [RPCClient-NioEventLoopGroup-18-16 {}] ipc.RpcConnection(322): sasl negotiation done with serverPrincipal = hbase/localhost@EXAMPLE.COM, succeed = true
2024-11-15T14:13:22,054 INFO  [NettyRpcServer-34-2 {}] ipc.ServerRpcConnection(484): Connection from 127.0.0.1:44368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=hbase/localhost@EXAMPLE.COM (auth:SIMPLE), service=TestProtobufRpcProto
2024-11-15T14:13:22,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-15T14:13:22,060 INFO  [Time-limited test {}] ipc.NettyRpcServer(345): Stopping server on localhost/127.0.0.1:46567
2024-11-15T14:13:22,067 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: security.TestSecureIPC#testRpcClientFallbackToSimpleAuth[3: rpcClientImpl=org.apache.hadoop.hbase.ipc.NettyRpcClient, rpcServerImpl=org.apache.hadoop.hbase.ipc.NettyRpcServer] Thread=102 (was 100)
Potentially hanging thread: NettyRpcServer-34-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: NettyRpcServer-34-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=610 (was 601) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=643 (was 643), ProcessCount=11 (was 11), AvailableMemoryMB=328 (was 330)
2024-11-15T14:13:22,067 INFO  [Time-limited test {}] impl.DefaultInternalKdcServerImpl(102): Default Internal kdc server stopped.